From 3fa82c5168f5859a02cebe2a7b5e12425f7d51d9 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Tue, 13 Jun 2023 17:54:41 -0400 Subject: [PATCH 001/147] Initial commit moving towards stages plugin --- pyproject.toml | 1 + src/_nebari/schema.py | 2 +- src/_nebari/stages/base.py | 75 ++++ src/_nebari/stages/infrastructure/__init__.py | 214 ++++++++++ .../infrastructure/template}/aws/locals.tf | 0 .../infrastructure/template}/aws/main.tf | 0 .../template}/aws/modules/accounting/main.tf | 0 .../aws/modules/accounting/variables.tf | 0 .../template}/aws/modules/efs/main.tf | 0 .../template}/aws/modules/efs/outputs.tf | 0 .../template}/aws/modules/efs/variables.tf | 0 .../template}/aws/modules/kafka/main.tf | 0 .../template}/aws/modules/kafka/outputs.tf | 0 .../template}/aws/modules/kafka/variables.tf | 0 .../aws/modules/kubernetes/autoscaling.tf | 0 .../aws/modules/kubernetes/locals.tf | 0 .../template}/aws/modules/kubernetes/main.tf | 0 .../aws/modules/kubernetes/outputs.tf | 0 .../aws/modules/kubernetes/policy.tf | 0 .../aws/modules/kubernetes/variables.tf | 0 .../template}/aws/modules/network/main.tf | 0 .../template}/aws/modules/network/outputs.tf | 0 .../aws/modules/network/variables.tf | 0 .../template}/aws/modules/permissions/main.tf | 0 .../aws/modules/permissions/outputs.tf | 0 .../aws/modules/permissions/variables.tf | 0 .../template}/aws/modules/rds/main.tf | 0 .../template}/aws/modules/rds/outputs.tf | 0 .../template}/aws/modules/rds/users.tf | 0 .../template}/aws/modules/rds/variables.tf | 0 .../template}/aws/modules/registry/main.tf | 0 .../template}/aws/modules/registry/outputs.tf | 0 .../aws/modules/registry/variables.tf | 0 .../template}/aws/modules/s3/main.tf | 0 .../template}/aws/modules/s3/outputs.tf | 0 .../template}/aws/modules/s3/variables.tf | 0 .../infrastructure/template}/aws/outputs.tf | 0 .../infrastructure/template}/aws/variables.tf | 0 .../infrastructure/template}/aws/versions.tf | 0 .../infrastructure/template}/azure/main.tf | 0 .../azure/modules/kubernetes/main.tf | 0 .../azure/modules/kubernetes/outputs.tf | 0 .../azure/modules/kubernetes/variables.tf | 0 .../template}/azure/modules/registry/main.tf | 0 .../azure/modules/registry/variables.tf | 0 .../infrastructure/template}/azure/outputs.tf | 0 .../template}/azure/providers.tf | 0 .../template}/azure/variables.tf | 0 .../template}/azure/versions.tf | 0 .../infrastructure/template}/do/main.tf | 0 .../template}/do/modules/kubernetes/locals.tf | 0 .../template}/do/modules/kubernetes/main.tf | 0 .../do/modules/kubernetes/outputs.tf | 0 .../do/modules/kubernetes/variables.tf | 0 .../do/modules/kubernetes}/versions.tf | 0 .../template}/do/modules/registry/main.tf | 0 .../template}/do/modules/registry/variable.tf | 0 .../template/do/modules/registry}/versions.tf | 0 .../infrastructure/template}/do/outputs.tf | 0 .../infrastructure/template}/do/providers.tf | 0 .../infrastructure/template}/do/variables.tf | 0 .../infrastructure/template/do}/versions.tf | 0 .../infrastructure/template}/existing/main.tf | 0 .../infrastructure/template}/gcp/main.tf | 0 .../gcp/modules/kubernetes/locals.tf | 0 .../template}/gcp/modules/kubernetes/main.tf | 0 .../gcp/modules/kubernetes/outputs.tf | 0 .../gcp/modules/kubernetes/service_account.tf | 0 .../kubernetes/templates/kubeconfig.yaml | 0 .../gcp/modules/kubernetes/variables.tf | 0 .../template}/gcp/modules/network/main.tf | 0 .../gcp/modules/network/variables.tf | 0 .../template}/gcp/modules/registry/main.tf | 0 .../gcp/modules/registry/variables.tf | 0 .../infrastructure/template}/gcp/outputs.tf | 0 .../infrastructure/template}/gcp/provider.tf | 0 .../infrastructure/template}/gcp/variables.tf | 0 .../infrastructure/template}/gcp/versions.tf | 0 .../infrastructure/template}/local/main.tf | 0 .../template}/local/metallb.yaml | 0 .../infrastructure/template}/local/outputs.tf | 0 .../template}/local/variables.tf | 0 src/_nebari/stages/input_vars.py | 403 ------------------ .../stages/kubernetes_ingress/__init__.py | 190 +++++++++ .../kubernetes_ingress/template}/locals.tf | 0 .../kubernetes_ingress/template}/main.tf | 0 .../modules/kubernetes/ingress/main.tf | 0 .../modules/kubernetes/ingress/outputs.tf | 0 .../modules/kubernetes/ingress/variables.tf | 0 .../kubernetes_ingress/template}/outputs.tf | 0 .../kubernetes_ingress/template}/variables.tf | 0 .../kubernetes_ingress/template}/versions.tf | 0 .../stages/kubernetes_initialize/__init__.py | 99 +++++ .../template}/external-container-registry.tf | 0 .../kubernetes_initialize/template}/locals.tf | 0 .../kubernetes_initialize/template}/main.tf | 0 .../modules/cluster-autoscaler/main.tf | 0 .../modules/cluster-autoscaler/variables.tf | 0 .../template}/modules/extcr/main.tf | 0 .../template}/modules/extcr/variables.tf | 0 .../template}/modules/initialization/main.tf | 0 .../modules/initialization/variables.tf | 0 .../nvidia-installer/aws-nvidia-installer.tf | 0 .../nvidia-installer/gcp-nvidia-installer.tf | 0 .../modules/nvidia-installer/variables.tf | 0 .../template}/modules/traefik_crds/main.tf | 0 .../template}/variables.tf | 0 .../template}/versions.tf | 0 .../stages/kubernetes_keycloak/__init__.py | 165 +++++++ .../kubernetes_keycloak/template}/main.tf | 0 .../modules/kubernetes/keycloak-helm/main.tf | 0 .../kubernetes/keycloak-helm/outputs.tf | 0 .../kubernetes/keycloak-helm/values.yaml | 0 .../kubernetes/keycloak-helm/variables.tf | 0 .../kubernetes_keycloak/template}/outputs.tf | 0 .../template}/variables.tf | 0 .../kubernetes_keycloak/template}/versions.tf | 0 .../__init__.py | 116 +++++ .../template}/main.tf | 0 .../template}/outputs.tf | 0 .../template}/permissions.tf | 0 .../template}/providers.tf | 0 .../template}/social_auth.tf | 0 .../template}/variables.tf | 0 .../template}/versions.tf | 0 .../stages/kubernetes_services/__init__.py | 208 +++++++++ .../template}/argo-workflows.tf | 0 .../kubernetes_services/template}/clearml.tf | 0 .../template}/conda-store.tf | 0 .../template}/dask_gateway.tf | 0 .../template}/forward-auth.tf | 0 .../template}/jupyterhub.tf | 0 .../template}/jupyterhub_ssh.tf | 0 .../kubernetes_services/template}/kbatch.tf | 0 .../kubernetes_services/template}/locals.tf | 0 .../template/modules}/__init__.py | 0 .../template/modules/kubernetes}/__init__.py | 0 .../modules/kubernetes/forwardauth/main.tf | 0 .../kubernetes/forwardauth/variables.tf | 0 .../modules/kubernetes/nfs-mount/main.tf | 0 .../modules/kubernetes/nfs-mount/outputs.tf | 0 .../modules/kubernetes/nfs-mount/variables.tf | 0 .../modules/kubernetes/nfs-server/main.tf | 0 .../modules/kubernetes/nfs-server/output.tf | 0 .../kubernetes/nfs-server/variables.tf | 0 .../modules/kubernetes/services}/__init__.py | 0 .../services/argo-workflows/main.tf | 0 .../services/argo-workflows/values.yaml | 0 .../services/argo-workflows/variables.tf | 0 .../services/argo-workflows/versions.tf | 0 .../services/clearml/chart/Chart.yaml | 0 .../kubernetes/services/clearml/chart/LICENSE | 0 .../services/clearml/chart/README.md | 0 .../clearml/chart/charts/mongodb-10.3.7.tgz | Bin .../clearml/chart/charts/redis-10.9.0.tgz | Bin .../clearml/chart/templates/NOTES.txt | 0 .../clearml/chart/templates/_helpers.tpl | 0 .../chart/templates/deployment-agent.yaml | 0 .../templates/deployment-agentservices.yaml | 0 .../chart/templates/deployment-apiserver.yaml | 0 .../chart/templates/deployment-elastic.yaml | 0 .../templates/deployment-fileserver.yaml | 0 .../chart/templates/deployment-webserver.yaml | 0 .../clearml/chart/templates/ingress.yaml | 0 .../chart/templates/pvc-agentservices.yaml | 0 .../chart/templates/pvc-apiserver.yaml | 0 .../chart/templates/pvc-fileserver.yaml | 0 .../clearml/chart/templates/secret-agent.yaml | 0 .../clearml/chart/templates/secrets.yaml | 0 .../chart/templates/service-apiserver.yaml | 0 .../chart/templates/service-fileserver.yaml | 0 .../chart/templates/service-webserver.yaml | 0 .../services/clearml/chart/values.yaml | 0 .../kubernetes/services/clearml/ingress.tf | 0 .../kubernetes/services/clearml/main.tf | 0 .../kubernetes/services/clearml/variables.tf | 0 .../services/conda-store}/__init__.py | 0 .../services/conda-store/config}/__init__.py | 0 .../conda-store/config/conda_store_config.py | 0 .../kubernetes/services/conda-store/output.tf | 0 .../kubernetes/services/conda-store/server.tf | 0 .../services/conda-store/storage.tf | 0 .../services/conda-store/variables.tf | 0 .../kubernetes/services/conda-store/worker.tf | 0 .../services/dask-gateway}/__init__.py | 0 .../services/dask-gateway/controler.tf | 0 .../kubernetes/services/dask-gateway/crds.tf | 0 .../services/dask-gateway/files}/__init__.py | 0 .../dask-gateway/files/controller_config.py | 0 .../dask-gateway/files/gateway_config.py | 0 .../services/dask-gateway/gateway.tf | 0 .../kubernetes/services/dask-gateway/main.tf | 0 .../services/dask-gateway/middleware.tf | 0 .../services/dask-gateway/outputs.tf | 0 .../services/dask-gateway/variables.tf | 0 .../services/jupyterhub-ssh/main.tf | 0 .../services/jupyterhub-ssh/sftp.tf | 0 .../kubernetes/services/jupyterhub-ssh/ssh.tf | 0 .../services/jupyterhub-ssh/variables.tf | 0 .../services/jupyterhub}/__init__.py | 0 .../services/jupyterhub/configmaps.tf | 0 .../services/jupyterhub}/files/__init__.py | 0 .../jupyterhub/files/ipython}/__init__.py | 0 .../files/ipython/ipython_config.py | 0 .../jupyter/jupyter_server_config.py.tpl | 0 .../jupyterhub/files/jupyterhub/01-theme.py | 0 .../jupyterhub/files/jupyterhub/02-spawner.py | 0 .../files/jupyterhub/03-profiles.py | 0 .../jupyterhub/files/jupyterhub}/__init__.py | 0 .../files/jupyterlab/overrides.json | 0 .../jupyterhub/files/skel/.bash_logout | 0 .../services/jupyterhub/files/skel/.bashrc | 0 .../services/jupyterhub/files/skel/.profile | 0 .../kubernetes/services/jupyterhub/main.tf | 0 .../kubernetes/services/jupyterhub/outputs.tf | 0 .../services/jupyterhub/values.yaml | 0 .../services/jupyterhub/variables.tf | 0 .../kubernetes/services/kbatch/main.tf | 0 .../kubernetes/services/kbatch/values.yaml | 0 .../kubernetes/services/kbatch/variables.tf | 0 .../kubernetes/services/kbatch/versions.tf | 0 .../services/keycloak-client/main.tf | 0 .../services/keycloak-client/outputs.tf | 0 .../services/keycloak-client/variables.tf | 0 .../services/keycloak-client/versions.tf | 0 .../kubernetes/services/minio/ingress.tf | 0 .../modules/kubernetes/services/minio/main.tf | 0 .../kubernetes/services/minio/outputs.tf | 0 .../kubernetes/services/minio/values.yaml | 0 .../kubernetes/services/minio/variables.tf | 0 .../dashboards/Main/cluster_information.json | 0 .../dashboards/Main/conda_store.json | 0 .../dashboards/Main/jupyterhub_dashboard.json | 0 .../monitoring/dashboards/Main/keycloak.json | 0 .../monitoring/dashboards/Main/traefik.json | 0 .../dashboards/Main/usage_report.json | 0 .../kubernetes/services/monitoring/main.tf | 0 .../services/monitoring/values.yaml | 0 .../services/monitoring/variables.tf | 0 .../services/monitoring/versions.tf | 0 .../kubernetes/services/postgresql/main.tf | 0 .../kubernetes/services/postgresql/outputs.tf | 0 .../services/postgresql/values.yaml | 0 .../services/postgresql/variables.tf | 0 .../services/prefect/chart/.helmignore | 0 .../services/prefect/chart/Chart.yaml | 0 .../prefect/chart/templates/_helpers.tpl | 0 .../prefect/chart/templates/prefect.yaml | 0 .../prefect/chart/templates/secret.yaml | 0 .../services/prefect/chart/values.yaml | 0 .../kubernetes/services/prefect/main.tf | 0 .../kubernetes/services/prefect/values.yaml | 0 .../kubernetes/services/prefect/variables.tf | 0 .../modules/kubernetes/services/redis/main.tf | 0 .../kubernetes/services/redis/outputs.tf | 0 .../kubernetes/services/redis/values.yaml | 0 .../kubernetes/services/redis/variables.tf | 0 .../template}/monitoring.tf | 0 .../kubernetes_services/template}/outputs.tf | 0 .../kubernetes_services/template}/prefect.tf | 0 .../template}/providers.tf | 0 .../template}/variables.tf | 0 .../kubernetes_services/template}/versions.tf | 0 .../stages/nebari_tf_extensions/__init__.py | 53 +++ .../template}/helm-extension.tf | 0 .../template}/modules/helm-extensions/main.tf | 0 .../modules/helm-extensions/variables.tf | 0 .../modules/nebariextension/ingress.tf | 0 .../nebariextension/keycloak-config.tf | 0 .../modules/nebariextension/locals.tf | 0 .../template}/modules/nebariextension/main.tf | 0 .../modules/nebariextension/variables.tf | 0 .../template}/nebari-config.tf | 0 .../template}/providers.tf | 0 .../template}/tf-extensions.tf | 0 .../template}/variables.tf | 0 .../template}/versions.tf | 0 src/_nebari/stages/state_imports.py | 50 --- .../stages/terraform_state/__init__.py | 122 ++++++ .../terraform_state/template}/aws/main.tf | 0 .../aws/modules/terraform-state/main.tf | 0 .../aws/modules/terraform-state/output.tf | 0 .../aws/modules/terraform-state/variables.tf | 0 .../terraform_state/template}/azure/main.tf | 0 .../azure/modules/terraform-state/main.tf | 0 .../modules/terraform-state/variables.tf | 0 .../terraform_state/template}/do/main.tf | 0 .../template}/do/modules/spaces/main.tf | 0 .../template}/do/modules/spaces/variables.tf | 0 .../template/do/modules/spaces}/versions.tf | 0 .../do/modules/terraform-state/main.tf | 0 .../do/modules/terraform-state/variables.tf | 0 .../do/modules/terraform-state}/versions.tf | 0 .../terraform_state/template}/gcp/main.tf | 0 .../template}/gcp/modules/gcs/main.tf | 0 .../template}/gcp/modules/gcs/variables.tf | 0 .../gcp/modules/terraform-state/main.tf | 0 .../gcp/modules/terraform-state/variables.tf | 0 src/_nebari/stages/tf_objects.py | 160 ------- .../jupyterhub/files/ipython/__init__.py | 0 .../jupyterhub/files/jupyterhub/__init__.py | 0 src/_nebari/template/stages/__init__.py | 0 src/_nebari/utils.py | 38 -- src/nebari/hookspecs.py | 50 +++ src/nebari/plugins.py | 21 + 305 files changed, 1315 insertions(+), 652 deletions(-) create mode 100644 src/_nebari/stages/base.py create mode 100644 src/_nebari/stages/infrastructure/__init__.py rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/locals.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/accounting/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/accounting/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/efs/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/efs/outputs.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/efs/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/kafka/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/kafka/outputs.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/kafka/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/kubernetes/autoscaling.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/kubernetes/locals.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/kubernetes/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/kubernetes/outputs.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/kubernetes/policy.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/kubernetes/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/network/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/network/outputs.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/network/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/permissions/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/permissions/outputs.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/permissions/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/rds/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/rds/outputs.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/rds/users.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/rds/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/registry/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/registry/outputs.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/registry/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/s3/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/s3/outputs.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/modules/s3/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/outputs.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/aws/versions.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/azure/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/azure/modules/kubernetes/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/azure/modules/kubernetes/outputs.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/azure/modules/kubernetes/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/azure/modules/registry/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/azure/modules/registry/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/azure/outputs.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/azure/providers.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/azure/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/azure/versions.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/do/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/do/modules/kubernetes/locals.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/do/modules/kubernetes/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/do/modules/kubernetes/outputs.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/do/modules/kubernetes/variables.tf (100%) rename src/_nebari/{template/stages/01-terraform-state/do/modules/spaces => stages/infrastructure/template/do/modules/kubernetes}/versions.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/do/modules/registry/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/do/modules/registry/variable.tf (100%) rename src/_nebari/{template/stages/01-terraform-state/do/modules/terraform-state => stages/infrastructure/template/do/modules/registry}/versions.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/do/outputs.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/do/providers.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/do/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure/do/modules/kubernetes => stages/infrastructure/template/do}/versions.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/existing/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/gcp/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/gcp/modules/kubernetes/locals.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/gcp/modules/kubernetes/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/gcp/modules/kubernetes/outputs.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/gcp/modules/kubernetes/service_account.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/gcp/modules/kubernetes/templates/kubeconfig.yaml (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/gcp/modules/kubernetes/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/gcp/modules/network/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/gcp/modules/network/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/gcp/modules/registry/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/gcp/modules/registry/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/gcp/outputs.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/gcp/provider.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/gcp/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/gcp/versions.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/local/main.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/local/metallb.yaml (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/local/outputs.tf (100%) rename src/_nebari/{template/stages/02-infrastructure => stages/infrastructure/template}/local/variables.tf (100%) delete mode 100644 src/_nebari/stages/input_vars.py create mode 100644 src/_nebari/stages/kubernetes_ingress/__init__.py rename src/_nebari/{template/stages/03-kubernetes-initialize => stages/kubernetes_ingress/template}/locals.tf (100%) rename src/_nebari/{template/stages/04-kubernetes-ingress => stages/kubernetes_ingress/template}/main.tf (100%) rename src/_nebari/{template/stages/04-kubernetes-ingress => stages/kubernetes_ingress/template}/modules/kubernetes/ingress/main.tf (100%) rename src/_nebari/{template/stages/04-kubernetes-ingress => stages/kubernetes_ingress/template}/modules/kubernetes/ingress/outputs.tf (100%) rename src/_nebari/{template/stages/04-kubernetes-ingress => stages/kubernetes_ingress/template}/modules/kubernetes/ingress/variables.tf (100%) rename src/_nebari/{template/stages/04-kubernetes-ingress => stages/kubernetes_ingress/template}/outputs.tf (100%) rename src/_nebari/{template/stages/04-kubernetes-ingress => stages/kubernetes_ingress/template}/variables.tf (100%) rename src/_nebari/{template/stages/03-kubernetes-initialize => stages/kubernetes_ingress/template}/versions.tf (100%) create mode 100644 src/_nebari/stages/kubernetes_initialize/__init__.py rename src/_nebari/{template/stages/03-kubernetes-initialize => stages/kubernetes_initialize/template}/external-container-registry.tf (100%) rename src/_nebari/{template/stages/04-kubernetes-ingress => stages/kubernetes_initialize/template}/locals.tf (100%) rename src/_nebari/{template/stages/03-kubernetes-initialize => stages/kubernetes_initialize/template}/main.tf (100%) rename src/_nebari/{template/stages/03-kubernetes-initialize => stages/kubernetes_initialize/template}/modules/cluster-autoscaler/main.tf (100%) rename src/_nebari/{template/stages/03-kubernetes-initialize => stages/kubernetes_initialize/template}/modules/cluster-autoscaler/variables.tf (100%) rename src/_nebari/{template/stages/03-kubernetes-initialize => stages/kubernetes_initialize/template}/modules/extcr/main.tf (100%) rename src/_nebari/{template/stages/03-kubernetes-initialize => stages/kubernetes_initialize/template}/modules/extcr/variables.tf (100%) rename src/_nebari/{template/stages/03-kubernetes-initialize => stages/kubernetes_initialize/template}/modules/initialization/main.tf (100%) rename src/_nebari/{template/stages/03-kubernetes-initialize => stages/kubernetes_initialize/template}/modules/initialization/variables.tf (100%) rename src/_nebari/{template/stages/03-kubernetes-initialize => stages/kubernetes_initialize/template}/modules/nvidia-installer/aws-nvidia-installer.tf (100%) rename src/_nebari/{template/stages/03-kubernetes-initialize => stages/kubernetes_initialize/template}/modules/nvidia-installer/gcp-nvidia-installer.tf (100%) rename src/_nebari/{template/stages/03-kubernetes-initialize => stages/kubernetes_initialize/template}/modules/nvidia-installer/variables.tf (100%) rename src/_nebari/{template/stages/03-kubernetes-initialize => stages/kubernetes_initialize/template}/modules/traefik_crds/main.tf (100%) rename src/_nebari/{template/stages/03-kubernetes-initialize => stages/kubernetes_initialize/template}/variables.tf (100%) rename src/_nebari/{template/stages/04-kubernetes-ingress => stages/kubernetes_initialize/template}/versions.tf (100%) create mode 100644 src/_nebari/stages/kubernetes_keycloak/__init__.py rename src/_nebari/{template/stages/05-kubernetes-keycloak => stages/kubernetes_keycloak/template}/main.tf (100%) rename src/_nebari/{template/stages/05-kubernetes-keycloak => stages/kubernetes_keycloak/template}/modules/kubernetes/keycloak-helm/main.tf (100%) rename src/_nebari/{template/stages/05-kubernetes-keycloak => stages/kubernetes_keycloak/template}/modules/kubernetes/keycloak-helm/outputs.tf (100%) rename src/_nebari/{template/stages/05-kubernetes-keycloak => stages/kubernetes_keycloak/template}/modules/kubernetes/keycloak-helm/values.yaml (100%) rename src/_nebari/{template/stages/05-kubernetes-keycloak => stages/kubernetes_keycloak/template}/modules/kubernetes/keycloak-helm/variables.tf (100%) rename src/_nebari/{template/stages/05-kubernetes-keycloak => stages/kubernetes_keycloak/template}/outputs.tf (100%) rename src/_nebari/{template/stages/05-kubernetes-keycloak => stages/kubernetes_keycloak/template}/variables.tf (100%) rename src/_nebari/{template/stages/05-kubernetes-keycloak => stages/kubernetes_keycloak/template}/versions.tf (100%) create mode 100644 src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py rename src/_nebari/{template/stages/06-kubernetes-keycloak-configuration => stages/kubernetes_keycloak_configuration/template}/main.tf (100%) rename src/_nebari/{template/stages/06-kubernetes-keycloak-configuration => stages/kubernetes_keycloak_configuration/template}/outputs.tf (100%) rename src/_nebari/{template/stages/06-kubernetes-keycloak-configuration => stages/kubernetes_keycloak_configuration/template}/permissions.tf (100%) rename src/_nebari/{template/stages/06-kubernetes-keycloak-configuration => stages/kubernetes_keycloak_configuration/template}/providers.tf (100%) rename src/_nebari/{template/stages/06-kubernetes-keycloak-configuration => stages/kubernetes_keycloak_configuration/template}/social_auth.tf (100%) rename src/_nebari/{template/stages/06-kubernetes-keycloak-configuration => stages/kubernetes_keycloak_configuration/template}/variables.tf (100%) rename src/_nebari/{template/stages/06-kubernetes-keycloak-configuration => stages/kubernetes_keycloak_configuration/template}/versions.tf (100%) create mode 100644 src/_nebari/stages/kubernetes_services/__init__.py rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/argo-workflows.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/clearml.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/conda-store.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/dask_gateway.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/forward-auth.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/jupyterhub.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/jupyterhub_ssh.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/kbatch.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/locals.tf (100%) rename src/_nebari/{template => stages/kubernetes_services/template/modules}/__init__.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template/modules/kubernetes}/__init__.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/forwardauth/main.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/forwardauth/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/nfs-mount/main.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/nfs-mount/outputs.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/nfs-mount/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/nfs-server/main.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/nfs-server/output.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/nfs-server/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services/modules => stages/kubernetes_services/template/modules/kubernetes/services}/__init__.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/argo-workflows/main.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/argo-workflows/values.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/argo-workflows/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/argo-workflows/versions.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/Chart.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/LICENSE (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/README.md (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/charts/mongodb-10.3.7.tgz (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/charts/redis-10.9.0.tgz (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/NOTES.txt (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/_helpers.tpl (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/deployment-agent.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/deployment-agentservices.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/deployment-apiserver.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/deployment-elastic.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/deployment-fileserver.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/deployment-webserver.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/ingress.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/pvc-agentservices.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/pvc-apiserver.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/pvc-fileserver.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/secret-agent.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/secrets.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/service-apiserver.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/service-fileserver.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/templates/service-webserver.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/chart/values.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/ingress.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/main.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/clearml/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services/modules/kubernetes => stages/kubernetes_services/template/modules/kubernetes/services/conda-store}/__init__.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services/modules/kubernetes/services => stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config}/__init__.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/conda-store/config/conda_store_config.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/conda-store/output.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/conda-store/server.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/conda-store/storage.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/conda-store/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/conda-store/worker.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store => stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway}/__init__.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/dask-gateway/controler.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/dask-gateway/crds.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/config => stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files}/__init__.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/dask-gateway/files/controller_config.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/dask-gateway/files/gateway_config.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/dask-gateway/gateway.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/dask-gateway/main.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/dask-gateway/middleware.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/dask-gateway/outputs.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/dask-gateway/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub-ssh/main.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub-ssh/sftp.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub-ssh/ssh.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub-ssh/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway => stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub}/__init__.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub/configmaps.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway => stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub}/files/__init__.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub => stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/ipython}/__init__.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub/files/ipython/ipython_config.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub/files/jupyterhub/01-theme.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files => stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub}/__init__.py (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub/files/skel/.bash_logout (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub/files/skel/.bashrc (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub/files/skel/.profile (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub/main.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub/outputs.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub/values.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/jupyterhub/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/kbatch/main.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/kbatch/values.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/kbatch/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/kbatch/versions.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/keycloak-client/main.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/keycloak-client/outputs.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/keycloak-client/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/keycloak-client/versions.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/minio/ingress.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/minio/main.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/minio/outputs.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/minio/values.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/minio/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/monitoring/dashboards/Main/cluster_information.json (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/monitoring/dashboards/Main/conda_store.json (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/monitoring/dashboards/Main/jupyterhub_dashboard.json (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/monitoring/dashboards/Main/keycloak.json (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/monitoring/dashboards/Main/traefik.json (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/monitoring/dashboards/Main/usage_report.json (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/monitoring/main.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/monitoring/values.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/monitoring/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/monitoring/versions.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/postgresql/main.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/postgresql/outputs.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/postgresql/values.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/postgresql/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/prefect/chart/.helmignore (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/prefect/chart/Chart.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/prefect/chart/templates/_helpers.tpl (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/prefect/chart/templates/prefect.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/prefect/chart/templates/secret.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/prefect/chart/values.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/prefect/main.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/prefect/values.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/prefect/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/redis/main.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/redis/outputs.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/redis/values.yaml (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/modules/kubernetes/services/redis/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/monitoring.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/outputs.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/prefect.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/providers.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/variables.tf (100%) rename src/_nebari/{template/stages/07-kubernetes-services => stages/kubernetes_services/template}/versions.tf (100%) create mode 100644 src/_nebari/stages/nebari_tf_extensions/__init__.py rename src/_nebari/{template/stages/08-nebari-tf-extensions => stages/nebari_tf_extensions/template}/helm-extension.tf (100%) rename src/_nebari/{template/stages/08-nebari-tf-extensions => stages/nebari_tf_extensions/template}/modules/helm-extensions/main.tf (100%) rename src/_nebari/{template/stages/08-nebari-tf-extensions => stages/nebari_tf_extensions/template}/modules/helm-extensions/variables.tf (100%) rename src/_nebari/{template/stages/08-nebari-tf-extensions => stages/nebari_tf_extensions/template}/modules/nebariextension/ingress.tf (100%) rename src/_nebari/{template/stages/08-nebari-tf-extensions => stages/nebari_tf_extensions/template}/modules/nebariextension/keycloak-config.tf (100%) rename src/_nebari/{template/stages/08-nebari-tf-extensions => stages/nebari_tf_extensions/template}/modules/nebariextension/locals.tf (100%) rename src/_nebari/{template/stages/08-nebari-tf-extensions => stages/nebari_tf_extensions/template}/modules/nebariextension/main.tf (100%) rename src/_nebari/{template/stages/08-nebari-tf-extensions => stages/nebari_tf_extensions/template}/modules/nebariextension/variables.tf (100%) rename src/_nebari/{template/stages/08-nebari-tf-extensions => stages/nebari_tf_extensions/template}/nebari-config.tf (100%) rename src/_nebari/{template/stages/08-nebari-tf-extensions => stages/nebari_tf_extensions/template}/providers.tf (100%) rename src/_nebari/{template/stages/08-nebari-tf-extensions => stages/nebari_tf_extensions/template}/tf-extensions.tf (100%) rename src/_nebari/{template/stages/08-nebari-tf-extensions => stages/nebari_tf_extensions/template}/variables.tf (100%) rename src/_nebari/{template/stages/08-nebari-tf-extensions => stages/nebari_tf_extensions/template}/versions.tf (100%) delete mode 100644 src/_nebari/stages/state_imports.py create mode 100644 src/_nebari/stages/terraform_state/__init__.py rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/aws/main.tf (100%) rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/aws/modules/terraform-state/main.tf (100%) rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/aws/modules/terraform-state/output.tf (100%) rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/aws/modules/terraform-state/variables.tf (100%) rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/azure/main.tf (100%) rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/azure/modules/terraform-state/main.tf (100%) rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/azure/modules/terraform-state/variables.tf (100%) rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/do/main.tf (100%) rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/do/modules/spaces/main.tf (100%) rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/do/modules/spaces/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure/do/modules/registry => stages/terraform_state/template/do/modules/spaces}/versions.tf (100%) rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/do/modules/terraform-state/main.tf (100%) rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/do/modules/terraform-state/variables.tf (100%) rename src/_nebari/{template/stages/02-infrastructure/do => stages/terraform_state/template/do/modules/terraform-state}/versions.tf (100%) rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/gcp/main.tf (100%) rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/gcp/modules/gcs/main.tf (100%) rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/gcp/modules/gcs/variables.tf (100%) rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/gcp/modules/terraform-state/main.tf (100%) rename src/_nebari/{template/stages/01-terraform-state => stages/terraform_state/template}/gcp/modules/terraform-state/variables.tf (100%) delete mode 100644 src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/ipython/__init__.py delete mode 100644 src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/__init__.py delete mode 100644 src/_nebari/template/stages/__init__.py create mode 100644 src/nebari/hookspecs.py create mode 100644 src/nebari/plugins.py diff --git a/pyproject.toml b/pyproject.toml index 3c9b11309..b8666a49f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ classifiers = [ ] dependencies = [ + "pluggy==1.0.0", "auth0-python==4.0.0", "azure-identity==1.12.0", "azure-mgmt-containerservice==19.1.0", diff --git a/src/_nebari/schema.py b/src/_nebari/schema.py index 6966e3b5b..83a627e33 100644 --- a/src/_nebari/schema.py +++ b/src/_nebari/schema.py @@ -580,7 +580,7 @@ class InitInputs(Base): class Main(Base): provider: ProviderEnum project_name: str - namespace: typing.Optional[letter_dash_underscore_pydantic] + namespace: typing.Optional[letter_dash_underscore_pydantic] = "dev" nebari_version: str = "" ci_cd: typing.Optional[CICD] domain: str diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py new file mode 100644 index 000000000..1f3058e7e --- /dev/null +++ b/src/_nebari/stages/base.py @@ -0,0 +1,75 @@ +from typing import List, Dict, Tuple +import pathlib + +from nebari.hookspecs import NebariStage +from _nebari import schema +from _nebari.provider import terraform +from _nebari.stages.tf_objects import NebariTerraformState + + +class NebariTerraformStage(NebariStage): + def __init__( + self, + output_directory: pathlib.Path, + config: schema.Main, + template_directory: pathlib.Path, + stage_prefix: pathlib.Path, + ): + super().__init__(self, output_directory, config) + self.template_directory = pathlib.Path(template_directory) + self.stage_prefix = pathlib.Path(stage_prefix) + + def state_imports(self) -> List[Tuple[str, str]]: + return [] + + def tf_objects(self) -> List[Dict]: + return [ + NebariTerraformState(self.name, config) + ] + + def render(self): + contents = { + str(self.output_directory / stage_prefix / "_nebari.tf.json"): terraform.tf_render_objects(self.tf_objects()) + } + for root, dirs, files in os.walk(self.template_directory): + for filename in filenames: + contents[os.path.join(root, filename)] = open(os.path.join(root, filename)).read() + return contents + + + def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): + return {} + + @contextlib.contextmanager + def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): + deploy_config = dict( + directory=str(output_directory / stage_prefix), + input_vars=self.input_vars(stage_outputs) + ) + state_imports = self.state_imports() + if state_imports: + deploy_config['terraform_import'] = True + deploy_config['state_imports'] = state_imports + + stage_outputs["stages/" + self.name] = terraform.deploy(**deploy_config) + yield + + def check(self, stage_outputs: Dict[str, Dict[str, Any]]): + pass + + @contextlib.contextmanager + def destroy(self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool]): + stage_outputs["stages/" + self.name] = terraform.deploy(( + directory=str(output_directory / stage_prefix), + input_vars=self.input_vars(stage_outputs), + terraform_init=True, + terraform_import=True, + terraform_apply=False, + terraform_destroy=False, + ) + yield + status["stages/" + self.name] = _terraform_destroy( + directory=str(output_directory / stage_prefix), + input_vars=self.input_vars(stage_outputs), + ignore_errors=True, + ) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py new file mode 100644 index 000000000..1c30c7695 --- /dev/null +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -0,0 +1,214 @@ +from typing import List, Dict +import pathlib +import sys +import contextlib + +from nebari.hookspecs import hookimpl, NebariStage +from _nebari.utils import modified_environ +from _nebari.stages.base import NebariTerraformStage +from _nebari.stages.tf_objects import NebariTerraformState, NebariGCPProvider, NebariAWSProvider + +from _nebari import schema + + +@contextlib.contextmanager +def kubernetes_provider_context(kubernetes_credentials: Dict[str, str]): + credential_mapping = { + "config_path": "KUBE_CONFIG_PATH", + "config_context": "KUBE_CTX", + "username": "KUBE_USER", + "password": "KUBE_PASSWORD", + "client_certificate": "KUBE_CLIENT_CERT_DATA", + "client_key": "KUBE_CLIENT_KEY_DATA", + "cluster_ca_certificate": "KUBE_CLUSTER_CA_CERT_DATA", + "host": "KUBE_HOST", + "token": "KUBE_TOKEN", + } + + credentials = { + credential_mapping[k]: v + for k, v in kubernetes_credentials.items() + if v is not None + } + with modified_environ(**credentials): + yield + + +class KubernetesInfrastructureStage(NebariTerraformStage): + @property + def name(self): + return "02-infrastructure" + + @property + def priority(self): + return 20 + + def tf_objects(self) -> List[Dict]: + if self.config.provider == schema.ProviderEnum.gcp: + return [ + NebariGCPProvider(self.config), + NebariTerraformState(self.name, self.config), + ] + elif self.config.provider == schema.ProviderEnum.do: + return [ + NebariTerraformState(self.name, self.config), + ] + elif self.config.provider == schema.ProviderEnum.azure: + return [ + NebariTerraformState(self.name, self.config), + ] + elif self.config.provider == schema.ProviderEnum.aws: + return [ + NebariAWSProvider(self.config), + NebariTerraformState(self.name, self.config), + ] + else: + return [] + + def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): + if self.config.provider == schema.ProviderEnum.local: + return { + "kubeconfig_filename": os.path.join( + tempfile.gettempdir(), "NEBARI_KUBECONFIG" + ), + "kube_context": self.config.local.kube_context, + } + elif self.config.provider == schema.ProviderEnum.existing: + return {"kube_context": self.config.existing.kube_context} + elif self.config.provider == schema.ProviderEnum.do: + return { + "name": self.config.project_name, + "environment": self.config.namespace, + "region": self.config.digital_ocean.region, + "kubernetes_version": self.config.digital_ocean.kubernetes_version, + "node_groups": self.config.digital_ocean.node_groups, + "kubeconfig_filename": os.path.join( + tempfile.gettempdir(), "NEBARI_KUBECONFIG" + ), + **self.config.do.terraform_overrides, + } + elif self.config.provider == schema.ProviderEnum.gcp: + return { + "name": self.config.project_name, + "environment": self.config.namespace, + "region": self.config.google_cloud_platform.region, + "project_id": self.config.google_cloud_platform.project, + "kubernetes_version": self.config.google_cloud_platform.kubernetes_version, + "release_channel": self.config.google_cloud_platform.release_channel, + "node_groups": [ + { + "name": key, + "instance_type": value["instance"], + "min_size": value["min_nodes"], + "max_size": value["max_nodes"], + "guest_accelerators": value["guest_accelerators"] + if "guest_accelerators" in value + else [], + **value, + } + for key, value in self.config.google_cloud_platform.node_groups.items() + ], + "kubeconfig_filename": os.path.join( + tempfile.gettempdir(), "NEBARI_KUBECONFIG" + ), + **self.config.gcp.terraform_overrides, + } + elif self.config.provider == schema.ProviderEnum.azure: + return { + "name": self.config.project_name, + "environment": self.config.namespace, + "region": self.config.azure.region, + "kubernetes_version": self.config.azure.kubernetes_version, + "node_groups": self.config.azure.node_groups, + "kubeconfig_filename": os.path.join( + tempfile.gettempdir(), "NEBARI_KUBECONFIG" + ), + "resource_group_name": f'{self.config.project_name}-{self.config.namespace}', + "node_resource_group_name": f'{self.config.project_name}-{self.config.namespace}-node-resource-group', + **self.config.azure.terraform_overrides, + } + elif self.config.provider == schema.ProviderEnum.aws: + return { + "name": self.config.project_name, + "environment": self.config.namespace, + "region": self.config.amazon_web_services.region, + "kubernetes_version": self.config.amazon_web_services.kubernetes_version, + "node_groups": [ + { + "name": key, + "min_size": value["min_nodes"], + "desired_size": max(value["min_nodes"], 1), + "max_size": value["max_nodes"], + "gpu": value.get("gpu", False), + "instance_type": value["instance"], + "single_subnet": value.get("single_subnet", False), + } + for key, value in self.config.amazon_web_services.node_groups.items() + ], + "kubeconfig_filename": os.path.join( + tempfile.gettempdir(), "NEBARI_KUBECONFIG" + ), + **self.config.amazon_web_services.terraform_overrides, + } + else: + return {} + + def check(self, stage_outputs: stage_outputs: Dict[str, Dict[str, Any]]): + from kubernetes import client, config + from kubernetes.client.rest import ApiException + + directory = "stages/02-infrastructure" + config.load_kube_config( + config_file=stage_outputs["stages/02-infrastructure"]["kubeconfig_filename"][ + "value" + ] + ) + + try: + api_instance = client.CoreV1Api() + result = api_instance.list_namespace() + except ApiException: + self.log.error( + f"ERROR: After stage={self.name} unable to connect to kubernetes cluster" + ) + sys.exit(1) + + if len(result.items) < 1: + self.log.error( + f"ERROR: After stage={self.name} no nodes provisioned within kubernetes cluster" + ) + sys.exit(1) + + self.log.info( + f"After stage={self.name} kubernetes cluster successfully provisioned" + ) + + @contextlib.contextmanager + def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): + with super().deploy(stage_outputs): + with kubernetes_provider_context( + stage_outputs["stages/" + self.name]["kubernetes_credentials"]["value"] + ): + yield + + @contextlib.contextmanager + def destroy(self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool]): + with super().destroy(stage_outputs, status): + with kubernetes_provider_context( + stage_outputs["stages/" + self.name]["kubernetes_credentials"]["value"] + ): + yield + +@hookimpl +def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> List[NebariStage]: + template_directory = pathlib.Path(__file__).parent / "template" / config.provider.value + stage_prefix = pathlib.Path("stages/02-infrastructure") / config.provider.value + + return [ + KubernetesInfrastructureStage( + install_directory, + config, + template_directory=template_directory, + stage_prefix=stage_prefix, + ) + ] diff --git a/src/_nebari/template/stages/02-infrastructure/aws/locals.tf b/src/_nebari/stages/infrastructure/template/aws/locals.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/locals.tf rename to src/_nebari/stages/infrastructure/template/aws/locals.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/main.tf b/src/_nebari/stages/infrastructure/template/aws/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/main.tf rename to src/_nebari/stages/infrastructure/template/aws/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/accounting/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/accounting/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/accounting/main.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/accounting/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/accounting/variables.tf b/src/_nebari/stages/infrastructure/template/aws/modules/accounting/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/accounting/variables.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/accounting/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/efs/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/efs/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/efs/main.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/efs/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/efs/outputs.tf b/src/_nebari/stages/infrastructure/template/aws/modules/efs/outputs.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/efs/outputs.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/efs/outputs.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/efs/variables.tf b/src/_nebari/stages/infrastructure/template/aws/modules/efs/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/efs/variables.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/efs/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/kafka/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kafka/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/kafka/main.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/kafka/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/kafka/outputs.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kafka/outputs.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/kafka/outputs.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/kafka/outputs.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/kafka/variables.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kafka/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/kafka/variables.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/kafka/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/kubernetes/autoscaling.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/autoscaling.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/kubernetes/autoscaling.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/autoscaling.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/kubernetes/locals.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/locals.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/kubernetes/locals.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/locals.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/kubernetes/main.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/kubernetes/outputs.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/outputs.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/kubernetes/outputs.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/outputs.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/kubernetes/policy.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/policy.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/kubernetes/policy.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/policy.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/kubernetes/variables.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/kubernetes/variables.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/network/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/network/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/network/main.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/network/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/network/outputs.tf b/src/_nebari/stages/infrastructure/template/aws/modules/network/outputs.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/network/outputs.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/network/outputs.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/network/variables.tf b/src/_nebari/stages/infrastructure/template/aws/modules/network/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/network/variables.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/network/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/permissions/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/permissions/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/permissions/main.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/permissions/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/permissions/outputs.tf b/src/_nebari/stages/infrastructure/template/aws/modules/permissions/outputs.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/permissions/outputs.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/permissions/outputs.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/permissions/variables.tf b/src/_nebari/stages/infrastructure/template/aws/modules/permissions/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/permissions/variables.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/permissions/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/rds/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/rds/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/rds/main.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/rds/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/rds/outputs.tf b/src/_nebari/stages/infrastructure/template/aws/modules/rds/outputs.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/rds/outputs.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/rds/outputs.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/rds/users.tf b/src/_nebari/stages/infrastructure/template/aws/modules/rds/users.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/rds/users.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/rds/users.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/rds/variables.tf b/src/_nebari/stages/infrastructure/template/aws/modules/rds/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/rds/variables.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/rds/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/registry/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/registry/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/registry/main.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/registry/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/registry/outputs.tf b/src/_nebari/stages/infrastructure/template/aws/modules/registry/outputs.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/registry/outputs.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/registry/outputs.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/registry/variables.tf b/src/_nebari/stages/infrastructure/template/aws/modules/registry/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/registry/variables.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/registry/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/s3/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/s3/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/s3/main.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/s3/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/s3/outputs.tf b/src/_nebari/stages/infrastructure/template/aws/modules/s3/outputs.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/s3/outputs.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/s3/outputs.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/modules/s3/variables.tf b/src/_nebari/stages/infrastructure/template/aws/modules/s3/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/modules/s3/variables.tf rename to src/_nebari/stages/infrastructure/template/aws/modules/s3/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/outputs.tf b/src/_nebari/stages/infrastructure/template/aws/outputs.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/outputs.tf rename to src/_nebari/stages/infrastructure/template/aws/outputs.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/variables.tf b/src/_nebari/stages/infrastructure/template/aws/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/variables.tf rename to src/_nebari/stages/infrastructure/template/aws/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/aws/versions.tf b/src/_nebari/stages/infrastructure/template/aws/versions.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/aws/versions.tf rename to src/_nebari/stages/infrastructure/template/aws/versions.tf diff --git a/src/_nebari/template/stages/02-infrastructure/azure/main.tf b/src/_nebari/stages/infrastructure/template/azure/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/azure/main.tf rename to src/_nebari/stages/infrastructure/template/azure/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/azure/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/azure/modules/kubernetes/main.tf rename to src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/azure/modules/kubernetes/outputs.tf b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/outputs.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/azure/modules/kubernetes/outputs.tf rename to src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/outputs.tf diff --git a/src/_nebari/template/stages/02-infrastructure/azure/modules/kubernetes/variables.tf b/src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/azure/modules/kubernetes/variables.tf rename to src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/azure/modules/registry/main.tf b/src/_nebari/stages/infrastructure/template/azure/modules/registry/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/azure/modules/registry/main.tf rename to src/_nebari/stages/infrastructure/template/azure/modules/registry/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/azure/modules/registry/variables.tf b/src/_nebari/stages/infrastructure/template/azure/modules/registry/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/azure/modules/registry/variables.tf rename to src/_nebari/stages/infrastructure/template/azure/modules/registry/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/azure/outputs.tf b/src/_nebari/stages/infrastructure/template/azure/outputs.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/azure/outputs.tf rename to src/_nebari/stages/infrastructure/template/azure/outputs.tf diff --git a/src/_nebari/template/stages/02-infrastructure/azure/providers.tf b/src/_nebari/stages/infrastructure/template/azure/providers.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/azure/providers.tf rename to src/_nebari/stages/infrastructure/template/azure/providers.tf diff --git a/src/_nebari/template/stages/02-infrastructure/azure/variables.tf b/src/_nebari/stages/infrastructure/template/azure/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/azure/variables.tf rename to src/_nebari/stages/infrastructure/template/azure/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/azure/versions.tf b/src/_nebari/stages/infrastructure/template/azure/versions.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/azure/versions.tf rename to src/_nebari/stages/infrastructure/template/azure/versions.tf diff --git a/src/_nebari/template/stages/02-infrastructure/do/main.tf b/src/_nebari/stages/infrastructure/template/do/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/do/main.tf rename to src/_nebari/stages/infrastructure/template/do/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/do/modules/kubernetes/locals.tf b/src/_nebari/stages/infrastructure/template/do/modules/kubernetes/locals.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/do/modules/kubernetes/locals.tf rename to src/_nebari/stages/infrastructure/template/do/modules/kubernetes/locals.tf diff --git a/src/_nebari/template/stages/02-infrastructure/do/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/do/modules/kubernetes/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/do/modules/kubernetes/main.tf rename to src/_nebari/stages/infrastructure/template/do/modules/kubernetes/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/do/modules/kubernetes/outputs.tf b/src/_nebari/stages/infrastructure/template/do/modules/kubernetes/outputs.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/do/modules/kubernetes/outputs.tf rename to src/_nebari/stages/infrastructure/template/do/modules/kubernetes/outputs.tf diff --git a/src/_nebari/template/stages/02-infrastructure/do/modules/kubernetes/variables.tf b/src/_nebari/stages/infrastructure/template/do/modules/kubernetes/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/do/modules/kubernetes/variables.tf rename to src/_nebari/stages/infrastructure/template/do/modules/kubernetes/variables.tf diff --git a/src/_nebari/template/stages/01-terraform-state/do/modules/spaces/versions.tf b/src/_nebari/stages/infrastructure/template/do/modules/kubernetes/versions.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/do/modules/spaces/versions.tf rename to src/_nebari/stages/infrastructure/template/do/modules/kubernetes/versions.tf diff --git a/src/_nebari/template/stages/02-infrastructure/do/modules/registry/main.tf b/src/_nebari/stages/infrastructure/template/do/modules/registry/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/do/modules/registry/main.tf rename to src/_nebari/stages/infrastructure/template/do/modules/registry/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/do/modules/registry/variable.tf b/src/_nebari/stages/infrastructure/template/do/modules/registry/variable.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/do/modules/registry/variable.tf rename to src/_nebari/stages/infrastructure/template/do/modules/registry/variable.tf diff --git a/src/_nebari/template/stages/01-terraform-state/do/modules/terraform-state/versions.tf b/src/_nebari/stages/infrastructure/template/do/modules/registry/versions.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/do/modules/terraform-state/versions.tf rename to src/_nebari/stages/infrastructure/template/do/modules/registry/versions.tf diff --git a/src/_nebari/template/stages/02-infrastructure/do/outputs.tf b/src/_nebari/stages/infrastructure/template/do/outputs.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/do/outputs.tf rename to src/_nebari/stages/infrastructure/template/do/outputs.tf diff --git a/src/_nebari/template/stages/02-infrastructure/do/providers.tf b/src/_nebari/stages/infrastructure/template/do/providers.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/do/providers.tf rename to src/_nebari/stages/infrastructure/template/do/providers.tf diff --git a/src/_nebari/template/stages/02-infrastructure/do/variables.tf b/src/_nebari/stages/infrastructure/template/do/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/do/variables.tf rename to src/_nebari/stages/infrastructure/template/do/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/do/modules/kubernetes/versions.tf b/src/_nebari/stages/infrastructure/template/do/versions.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/do/modules/kubernetes/versions.tf rename to src/_nebari/stages/infrastructure/template/do/versions.tf diff --git a/src/_nebari/template/stages/02-infrastructure/existing/main.tf b/src/_nebari/stages/infrastructure/template/existing/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/existing/main.tf rename to src/_nebari/stages/infrastructure/template/existing/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/gcp/main.tf b/src/_nebari/stages/infrastructure/template/gcp/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/gcp/main.tf rename to src/_nebari/stages/infrastructure/template/gcp/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/gcp/modules/kubernetes/locals.tf b/src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/locals.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/gcp/modules/kubernetes/locals.tf rename to src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/locals.tf diff --git a/src/_nebari/template/stages/02-infrastructure/gcp/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/gcp/modules/kubernetes/main.tf rename to src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/gcp/modules/kubernetes/outputs.tf b/src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/outputs.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/gcp/modules/kubernetes/outputs.tf rename to src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/outputs.tf diff --git a/src/_nebari/template/stages/02-infrastructure/gcp/modules/kubernetes/service_account.tf b/src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/service_account.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/gcp/modules/kubernetes/service_account.tf rename to src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/service_account.tf diff --git a/src/_nebari/template/stages/02-infrastructure/gcp/modules/kubernetes/templates/kubeconfig.yaml b/src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/templates/kubeconfig.yaml similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/gcp/modules/kubernetes/templates/kubeconfig.yaml rename to src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/templates/kubeconfig.yaml diff --git a/src/_nebari/template/stages/02-infrastructure/gcp/modules/kubernetes/variables.tf b/src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/gcp/modules/kubernetes/variables.tf rename to src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/gcp/modules/network/main.tf b/src/_nebari/stages/infrastructure/template/gcp/modules/network/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/gcp/modules/network/main.tf rename to src/_nebari/stages/infrastructure/template/gcp/modules/network/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/gcp/modules/network/variables.tf b/src/_nebari/stages/infrastructure/template/gcp/modules/network/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/gcp/modules/network/variables.tf rename to src/_nebari/stages/infrastructure/template/gcp/modules/network/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/gcp/modules/registry/main.tf b/src/_nebari/stages/infrastructure/template/gcp/modules/registry/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/gcp/modules/registry/main.tf rename to src/_nebari/stages/infrastructure/template/gcp/modules/registry/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/gcp/modules/registry/variables.tf b/src/_nebari/stages/infrastructure/template/gcp/modules/registry/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/gcp/modules/registry/variables.tf rename to src/_nebari/stages/infrastructure/template/gcp/modules/registry/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/gcp/outputs.tf b/src/_nebari/stages/infrastructure/template/gcp/outputs.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/gcp/outputs.tf rename to src/_nebari/stages/infrastructure/template/gcp/outputs.tf diff --git a/src/_nebari/template/stages/02-infrastructure/gcp/provider.tf b/src/_nebari/stages/infrastructure/template/gcp/provider.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/gcp/provider.tf rename to src/_nebari/stages/infrastructure/template/gcp/provider.tf diff --git a/src/_nebari/template/stages/02-infrastructure/gcp/variables.tf b/src/_nebari/stages/infrastructure/template/gcp/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/gcp/variables.tf rename to src/_nebari/stages/infrastructure/template/gcp/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/gcp/versions.tf b/src/_nebari/stages/infrastructure/template/gcp/versions.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/gcp/versions.tf rename to src/_nebari/stages/infrastructure/template/gcp/versions.tf diff --git a/src/_nebari/template/stages/02-infrastructure/local/main.tf b/src/_nebari/stages/infrastructure/template/local/main.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/local/main.tf rename to src/_nebari/stages/infrastructure/template/local/main.tf diff --git a/src/_nebari/template/stages/02-infrastructure/local/metallb.yaml b/src/_nebari/stages/infrastructure/template/local/metallb.yaml similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/local/metallb.yaml rename to src/_nebari/stages/infrastructure/template/local/metallb.yaml diff --git a/src/_nebari/template/stages/02-infrastructure/local/outputs.tf b/src/_nebari/stages/infrastructure/template/local/outputs.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/local/outputs.tf rename to src/_nebari/stages/infrastructure/template/local/outputs.tf diff --git a/src/_nebari/template/stages/02-infrastructure/local/variables.tf b/src/_nebari/stages/infrastructure/template/local/variables.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/local/variables.tf rename to src/_nebari/stages/infrastructure/template/local/variables.tf diff --git a/src/_nebari/stages/input_vars.py b/src/_nebari/stages/input_vars.py deleted file mode 100644 index 889a81ec8..000000000 --- a/src/_nebari/stages/input_vars.py +++ /dev/null @@ -1,403 +0,0 @@ -import json -import tempfile -from pathlib import Path -from urllib.parse import urlencode - -from _nebari.constants import ( - DEFAULT_CONDA_STORE_IMAGE_TAG, - DEFAULT_GKE_RELEASE_CHANNEL, - DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG, - DEFAULT_TRAEFIK_IMAGE_TAG, -) - - -def stage_01_terraform_state(stage_outputs, config): - if config["provider"] == "do": - return { - "name": config["project_name"], - "namespace": config["namespace"], - "region": config["digital_ocean"]["region"], - } - elif config["provider"] == "gcp": - return { - "name": config["project_name"], - "namespace": config["namespace"], - "region": config["google_cloud_platform"]["region"], - } - elif config["provider"] == "aws": - return { - "name": config["project_name"], - "namespace": config["namespace"], - } - elif config["provider"] == "azure": - return { - "name": config["project_name"], - "namespace": config["namespace"], - "region": config["azure"]["region"], - "storage_account_postfix": config["azure"]["storage_account_postfix"], - "state_resource_group_name": f'{config["project_name"]}-{config["namespace"]}-state', - } - else: - return {} - - -def stage_02_infrastructure(stage_outputs, config): - if config["provider"] == "local": - return { - "kubeconfig_filename": str( - Path(tempfile.gettempdir()) / "NEBARI_KUBECONFIG" - ), - "kube_context": config["local"].get("kube_context"), - } - elif config["provider"] == "existing": - return {"kube_context": config["existing"].get("kube_context")} - elif config["provider"] == "do": - return { - "name": config["project_name"], - "environment": config["namespace"], - "region": config["digital_ocean"]["region"], - "kubernetes_version": config["digital_ocean"]["kubernetes_version"], - "node_groups": config["digital_ocean"]["node_groups"], - "kubeconfig_filename": str( - Path(tempfile.gettempdir()) / "NEBARI_KUBECONFIG" - ), - **config.get("do", {}).get("terraform_overrides", {}), - } - elif config["provider"] == "gcp": - return { - "name": config["project_name"], - "environment": config["namespace"], - "region": config["google_cloud_platform"]["region"], - "project_id": config["google_cloud_platform"]["project"], - "kubernetes_version": config["google_cloud_platform"]["kubernetes_version"], - "release_channel": config.get("google_cloud_platform", {}).get( - "release_channel", DEFAULT_GKE_RELEASE_CHANNEL - ), - "node_groups": [ - { - "name": key, - "instance_type": value["instance"], - "min_size": value["min_nodes"], - "max_size": value["max_nodes"], - "guest_accelerators": value["guest_accelerators"] - if "guest_accelerators" in value - else [], - **value, - } - for key, value in config["google_cloud_platform"]["node_groups"].items() - ], - "kubeconfig_filename": str( - Path(tempfile.gettempdir()) / "NEBARI_KUBECONFIG" - ), - **config.get("gcp", {}).get("terraform_overrides", {}), - } - elif config["provider"] == "azure": - return { - "name": config["project_name"], - "environment": config["namespace"], - "region": config["azure"]["region"], - "kubernetes_version": config["azure"]["kubernetes_version"], - "node_groups": config["azure"]["node_groups"], - "kubeconfig_filename": str( - Path(tempfile.gettempdir()) / "NEBARI_KUBECONFIG" - ), - "resource_group_name": f'{config["project_name"]}-{config["namespace"]}', - "node_resource_group_name": f'{config["project_name"]}-{config["namespace"]}-node-resource-group', - **config.get("azure", {}).get("terraform_overrides", {}), - } - elif config["provider"] == "aws": - return { - "name": config["project_name"], - "environment": config["namespace"], - "region": config["amazon_web_services"]["region"], - "kubernetes_version": config["amazon_web_services"]["kubernetes_version"], - "node_groups": [ - { - "name": key, - "min_size": value["min_nodes"], - "desired_size": max(value["min_nodes"], 1), - "max_size": value["max_nodes"], - "gpu": value.get("gpu", False), - "instance_type": value["instance"], - "single_subnet": value.get("single_subnet", False), - } - for key, value in config["amazon_web_services"]["node_groups"].items() - ], - "kubeconfig_filename": str( - Path(tempfile.gettempdir()) / "NEBARI_KUBECONFIG" - ), - **config.get("amazon_web_services", {}).get("terraform_overrides", {}), - } - else: - return {} - - -def stage_03_kubernetes_initialize(stage_outputs, config): - if config["provider"] == "gcp": - gpu_enabled = any( - node_group.get("guest_accelerators") - for node_group in config["google_cloud_platform"]["node_groups"].values() - ) - gpu_node_group_names = [] - - elif config["provider"] == "aws": - gpu_enabled = any( - node_group.get("gpu") - for node_group in config["amazon_web_services"]["node_groups"].values() - ) - gpu_node_group_names = [ - group for group in config["amazon_web_services"]["node_groups"].keys() - ] - else: - gpu_enabled = False - gpu_node_group_names = [] - - return { - "name": config["project_name"], - "environment": config["namespace"], - "cloud-provider": config["provider"], - "aws-region": config.get("amazon_web_services", {}).get("region"), - "external_container_reg": config.get( - "external_container_reg", {"enabled": False} - ), - "gpu_enabled": gpu_enabled, - "gpu_node_group_names": gpu_node_group_names, - } - - -def _calculate_node_groups(config): - if config["provider"] == "aws": - return { - group: {"key": "eks.amazonaws.com/nodegroup", "value": group} - for group in ["general", "user", "worker"] - } - elif config["provider"] == "gcp": - return { - group: {"key": "cloud.google.com/gke-nodepool", "value": group} - for group in ["general", "user", "worker"] - } - elif config["provider"] == "azure": - return { - group: {"key": "azure-node-pool", "value": group} - for group in ["general", "user", "worker"] - } - elif config["provider"] == "do": - return { - group: {"key": "doks.digitalocean.com/node-pool", "value": group} - for group in ["general", "user", "worker"] - } - elif config["provider"] == "existing": - return config["existing"].get("node_selectors") - else: - return config["local"]["node_selectors"] - - -def stage_04_kubernetes_ingress(stage_outputs, config): - cert_type = config["certificate"]["type"] - cert_details = {"certificate-service": cert_type} - if cert_type == "lets-encrypt": - cert_details["acme-email"] = config["certificate"]["acme_email"] - cert_details["acme-server"] = config["certificate"]["acme_server"] - elif cert_type == "existing": - cert_details["certificate-secret-name"] = config["certificate"]["secret_name"] - - return { - **{ - "traefik-image": { - "image": "traefik", - "tag": DEFAULT_TRAEFIK_IMAGE_TAG, - }, - "name": config["project_name"], - "environment": config["namespace"], - "node_groups": _calculate_node_groups(config), - **config.get("ingress", {}).get("terraform_overrides", {}), - }, - **cert_details, - } - - -def stage_05_kubernetes_keycloak(stage_outputs, config): - initial_root_password = ( - config["security"].get("keycloak", {}).get("initial_root_password", "") - ) - if initial_root_password is None: - initial_root_password = "" - - return { - "name": config["project_name"], - "environment": config["namespace"], - "endpoint": config["domain"], - "initial-root-password": initial_root_password, - "overrides": [ - json.dumps(config["security"].get("keycloak", {}).get("overrides", {})) - ], - "node-group": _calculate_node_groups(config)["general"], - } - - -def stage_06_kubernetes_keycloak_configuration(stage_outputs, config): - realm_id = "nebari" - - users_group = ( - ["users"] if config["security"].get("shared_users_group", False) else [] - ) - - return { - "realm": realm_id, - "realm_display_name": config["security"] - .get("keycloak", {}) - .get("realm_display_name", realm_id), - "authentication": config["security"]["authentication"], - "keycloak_groups": ["superadmin", "admin", "developer", "analyst"] - + users_group, - "default_groups": ["analyst"] + users_group, - } - - -def _split_docker_image_name(image_name): - name, tag = image_name.split(":") - return {"name": name, "tag": tag} - - -def stage_07_kubernetes_services(stage_outputs, config): - final_logout_uri = f"https://{config['domain']}/hub/login" - - # Compound any logout URLs from extensions so they are are logged out in succession - # when Keycloak and JupyterHub are logged out - for ext in config.get("tf_extensions", []): - if ext.get("logout", "") != "": - final_logout_uri = "{}?{}".format( - f"https://{config['domain']}/{ext['urlslug']}{ext['logout']}", - urlencode({"redirect_uri": final_logout_uri}), - ) - jupyterhub_theme = config["theme"]["jupyterhub"] - if config["theme"]["jupyterhub"].get("display_version") and ( - not config["theme"]["jupyterhub"].get("version", False) - ): - jupyterhub_theme.update({"version": f"v{config['nebari_version']}"}) - - return { - "name": config["project_name"], - "environment": config["namespace"], - "endpoint": config["domain"], - "realm_id": stage_outputs["stages/06-kubernetes-keycloak-configuration"][ - "realm_id" - ]["value"], - "node_groups": _calculate_node_groups(config), - # conda-store - "conda-store-environments": config["environments"], - "conda-store-filesystem-storage": config["storage"]["conda_store"], - "conda-store-service-token-scopes": { - "cdsdashboards": { - "primary_namespace": "cdsdashboards", - "role_bindings": { - "*/*": ["viewer"], - }, - }, - "dask-gateway": { - "primary_namespace": "", - "role_bindings": { - "*/*": ["viewer"], - }, - }, - "argo-workflows-jupyter-scheduler": { - "primary_namespace": "", - "role_bindings": { - "*/*": ["viewer"], - }, - }, - }, - "conda-store-default-namespace": config.get("conda_store", {}).get( - "default_namespace", "nebari-git" - ), - "conda-store-extra-settings": config.get("conda_store", {}).get( - "extra_settings", {} - ), - "conda-store-extra-config": config.get("conda_store", {}).get( - "extra_config", "" - ), - "conda-store-image-tag": config.get("conda_store", {}).get( - "image_tag", DEFAULT_CONDA_STORE_IMAGE_TAG - ), - # jupyterhub - "cdsdashboards": config["cdsdashboards"], - "jupyterhub-theme": jupyterhub_theme, - "jupyterhub-image": _split_docker_image_name( - config["default_images"]["jupyterhub"] - ), - "jupyterhub-shared-storage": config["storage"]["shared_filesystem"], - "jupyterhub-shared-endpoint": stage_outputs["stages/02-infrastructure"] - .get("nfs_endpoint", {}) - .get("value"), - "jupyterlab-profiles": config["profiles"]["jupyterlab"], - "jupyterlab-image": _split_docker_image_name( - config["default_images"]["jupyterlab"] - ), - "jupyterhub-overrides": [ - json.dumps(config.get("jupyterhub", {}).get("overrides", {})) - ], - "jupyterhub-hub-extraEnv": json.dumps( - config.get("jupyterhub", {}) - .get("overrides", {}) - .get("hub", {}) - .get("extraEnv", []) - ), - # jupyterlab - "idle-culler-settings": config.get("jupyterlab", {}).get("idle_culler", {}), - # dask-gateway - "dask-worker-image": _split_docker_image_name( - config["default_images"]["dask_worker"] - ), - "dask-gateway-profiles": config["profiles"]["dask_worker"], - # monitoring - "monitoring-enabled": config["monitoring"]["enabled"], - # argo-worfklows - "argo-workflows-enabled": config["argo_workflows"]["enabled"], - "argo-workflows-overrides": [ - json.dumps(config.get("argo_workflows", {}).get("overrides", {})) - ], - "nebari-workflow-controller": config["argo_workflows"] - .get("nebari_workflow_controller", {}) - .get("enabled", True), - "keycloak-read-only-user-credentials": stage_outputs[ - "stages/06-kubernetes-keycloak-configuration" - ]["keycloak-read-only-user-credentials"]["value"], - "workflow-controller-image-tag": config.get("argo_workflows", {}) - .get("nebari_workflow_controller", {}) - .get( - "image_tag", - DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG, - ), - # kbatch - "kbatch-enabled": config["kbatch"]["enabled"], - # prefect - "prefect-enabled": config.get("prefect", {}).get("enabled", False), - "prefect-token": config.get("prefect", {}).get("token", ""), - "prefect-image": config.get("prefect", {}).get("image", ""), - "prefect-overrides": config.get("prefect", {}).get("overrides", {}), - # clearml - "clearml-enabled": config.get("clearml", {}).get("enabled", False), - "clearml-enable-forwardauth": config.get("clearml", {}).get( - "enable_forward_auth", False - ), - "clearml-overrides": [ - json.dumps(config.get("clearml", {}).get("overrides", {})) - ], - "jupyterhub-logout-redirect-url": final_logout_uri, - } - - -def stage_08_nebari_tf_extensions(stage_outputs, config): - return { - "environment": config["namespace"], - "endpoint": config["domain"], - "realm_id": stage_outputs["stages/06-kubernetes-keycloak-configuration"][ - "realm_id" - ]["value"], - "tf_extensions": config.get("tf_extensions", []), - "nebari_config_yaml": config, - "keycloak_nebari_bot_password": stage_outputs["stages/05-kubernetes-keycloak"][ - "keycloak_nebari_bot_password" - ]["value"], - "helm_extensions": config.get("helm_extensions", []), - } diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py new file mode 100644 index 000000000..d61a5d587 --- /dev/null +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -0,0 +1,190 @@ +from typing import List, Dict +import pathlib +import sys +import socket + +from nebari.hookspecs import NebariStage, hookimpl +from _nebari.stages.base import NebariTerraformStage +from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider + +from _nebari import schema, constants + + +# check and retry settings +NUM_ATTEMPTS = 10 +TIMEOUT = 10 # seconds + +def _calculate_node_groups(config: schema.Main): + if config.provider == schema.ProviderEnum.aws: + return { + group: {"key": "eks.amazonaws.com/nodegroup", "value": group} + for group in ["general", "user", "worker"] + } + elif config.provider == schema.ProviderEnum.gcp: + return { + group: {"key": "cloud.google.com/gke-nodepool", "value": group} + for group in ["general", "user", "worker"] + } + elif config.provider == schema.ProviderEnum.azure: + return { + group: {"key": "azure-node-pool", "value": group} + for group in ["general", "user", "worker"] + } + elif config.provider == schema.ProviderEnum.do: + return { + group: {"key": "doks.digitalocean.com/node-pool", "value": group} + for group in ["general", "user", "worker"] + } + elif config.provider == schema.ProviderEnum.existing: + return config.existing.node_selectors + else: + return config.local.node_selectors + + +def check_ingress_dns(stage_outputs, config, disable_prompt): + directory = "stages/04-kubernetes-ingress" + + ip_or_name = stage_outputs[directory]["load_balancer_address"]["value"] + ip = socket.gethostbyname(ip_or_name["hostname"] or ip_or_name["ip"]) + domain_name = config["domain"] + + def _attempt_dns_lookup( + domain_name, ip, num_attempts=NUM_ATTEMPTS, timeout=TIMEOUT + ): + for i in range(num_attempts): + try: + resolved_ip = socket.gethostbyname(domain_name) + if resolved_ip == ip: + print( + f"DNS configured domain={domain_name} matches ingress ip={ip}" + ) + return True + else: + print( + f"Attempt {i+1} polling DNS domain={domain_name} does not match ip={ip} instead got {resolved_ip}" + ) + except socket.gaierror: + print( + f"Attempt {i+1} polling DNS domain={domain_name} record does not exist" + ) + time.sleep(timeout) + return False + + attempt = 0 + while not _attempt_dns_lookup(domain_name, ip): + sleeptime = 60 * (2**attempt) + if not disable_prompt: + input( + f"After attempting to poll the DNS, the record for domain={domain_name} appears not to exist, " + f"has recently been updated, or has yet to fully propagate. This non-deterministic behavior is likely due to " + f"DNS caching and will likely resolve itself in a few minutes.\n\n\tTo poll the DNS again in {sleeptime} seconds " + f"[Press Enter].\n\n...otherwise kill the process and run the deployment again later..." + ) + + print(f"Will attempt to poll DNS again in {sleeptime} seconds...") + time.sleep(sleeptime) + attempt += 1 + if attempt == 5: + print( + f"ERROR: After stage directory={directory} DNS domain={domain_name} does not point to ip={ip}" + ) + sys.exit(1) + + +class KubernetesIngressStage(NebariTerraformStage): + @property + def name(self): + return "04-kubernetes-ingress" + + @property + def priority(self): + return 40 + + def tf_objects(self) -> List[Dict]: + return [ + NebariTerraformState(self.name, self.config), + NebariKubernetesProvider(self.config), + NebariHelmProvider(self.config), + ] + + def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): + cert_type = self.config.certificate.type + cert_details = {"certificate-service": cert_type} + if cert_type == "lets-encrypt": + cert_details["acme-email"] = self.config.certificate.acme_email + cert_details["acme-server"] = self.config.certificate.acme_server + elif cert_type == "existing": + cert_details["certificate-secret-name"] = self.config.certificate.secret_name + + return { + **{ + "traefik-image": { + "image": "traefik", + "tag": constants.DEFAULT_TRAEFIK_IMAGE_TAG, + }, + "name": self.config.project_name, + "environment": self.config.namespace, + "node_groups": _calculate_node_groups(self.config), + **config.ingress.terraform_overrides, + }, + **cert_details, + } + + def check(self, stage_outputs: stage_outputs: Dict[str, Dict[str, Any]]): + def _attempt_tcp_connect(host, port, num_attempts=NUM_ATTEMPTS, timeout=TIMEOUT): + for i in range(num_attempts): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + # normalize hostname to ip address + ip = socket.gethostbyname(host) + s.settimeout(5) + result = s.connect_ex((ip, port)) + if result == 0: + print(f"Attempt {i+1} succeeded to connect to tcp://{ip}:{port}") + return True + print(f"Attempt {i+1} failed to connect to tcp tcp://{ip}:{port}") + except socket.gaierror: + print(f"Attempt {i+1} failed to get IP for {host}...") + finally: + s.close() + + time.sleep(timeout) + + return False + + tcp_ports = { + 80, # http + 443, # https + 8022, # jupyterhub-ssh ssh + 8023, # jupyterhub-ssh sftp + 9080, # minio + 8786, # dask-scheduler + } + ip_or_name = stage_outputs["stages/" + self.name]["load_balancer_address"]["value"] + host = ip_or_name["hostname"] or ip_or_name["ip"] + host = host.strip("\n") + + for port in tcp_ports: + if not _attempt_tcp_connect(host, port): + print( + f"ERROR: After stage={self.name} unable to connect to ingress host={host} port={port}" + ) + sys.exit(1) + + self.log.info( + f"After stage={self.name} kubernetes ingress available on tcp ports={tcp_ports}" + ) + + check_ingress_dns(stage_outputs, self.config, disable_prompt=False) + + +@hookimpl +def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> [NebariStage]: + return [ + KubernetesIngressStage( + install_directory, + config, + template_directory=(pathlib.Path(__file__).parent / "template"), + stage_prefix=pathlib.Path("stages/04-kubernetes-ingress"), + ) + ] diff --git a/src/_nebari/template/stages/03-kubernetes-initialize/locals.tf b/src/_nebari/stages/kubernetes_ingress/template/locals.tf similarity index 100% rename from src/_nebari/template/stages/03-kubernetes-initialize/locals.tf rename to src/_nebari/stages/kubernetes_ingress/template/locals.tf diff --git a/src/_nebari/template/stages/04-kubernetes-ingress/main.tf b/src/_nebari/stages/kubernetes_ingress/template/main.tf similarity index 100% rename from src/_nebari/template/stages/04-kubernetes-ingress/main.tf rename to src/_nebari/stages/kubernetes_ingress/template/main.tf diff --git a/src/_nebari/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/main.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf similarity index 100% rename from src/_nebari/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/main.tf rename to src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf diff --git a/src/_nebari/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/outputs.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/outputs.tf similarity index 100% rename from src/_nebari/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/outputs.tf rename to src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/outputs.tf diff --git a/src/_nebari/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/variables.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf similarity index 100% rename from src/_nebari/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/variables.tf rename to src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf diff --git a/src/_nebari/template/stages/04-kubernetes-ingress/outputs.tf b/src/_nebari/stages/kubernetes_ingress/template/outputs.tf similarity index 100% rename from src/_nebari/template/stages/04-kubernetes-ingress/outputs.tf rename to src/_nebari/stages/kubernetes_ingress/template/outputs.tf diff --git a/src/_nebari/template/stages/04-kubernetes-ingress/variables.tf b/src/_nebari/stages/kubernetes_ingress/template/variables.tf similarity index 100% rename from src/_nebari/template/stages/04-kubernetes-ingress/variables.tf rename to src/_nebari/stages/kubernetes_ingress/template/variables.tf diff --git a/src/_nebari/template/stages/03-kubernetes-initialize/versions.tf b/src/_nebari/stages/kubernetes_ingress/template/versions.tf similarity index 100% rename from src/_nebari/template/stages/03-kubernetes-initialize/versions.tf rename to src/_nebari/stages/kubernetes_ingress/template/versions.tf diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py new file mode 100644 index 000000000..feed5cfaf --- /dev/null +++ b/src/_nebari/stages/kubernetes_initialize/__init__.py @@ -0,0 +1,99 @@ +from typing import List, Dict +import pathlib +import sys + +from nebari.hookspecs import NebariStage, hookimpl +from _nebari.stages.base import NebariTerraformStage +from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider + +from _nebari import schema + + +class KubernetesInitializeStage(NebariTerraformStage): + @property + def name(self): + return "03-kubernetes-initialize" + + @property + def priority(self): + return 30 + + def tf_objects(self) -> List[Dict]: + return [ + NebariTerraformState(self.name, self.config), + NebariKubernetesProvider(self.config), + NebariHelmProvider(self.config), + ] + + def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): + if self.config.provider == schema.ProviderEnum.gcp: + gpu_enabled = any( + node_group.guest_accelerators + for node_group in self.config.google_cloud_platform.node_groups.values() + ) + gpu_node_group_names = [] + + elif self.config.provider == schema.ProvderEnum.aws: + gpu_enabled = any( + node_group.gpu + for node_group in self.config.amazon_web_services.node_groups.values() + ) + gpu_node_group_names = [ + group for group in self.config.amazon_web_services.node_groups.keys() + ] + else: + gpu_enabled = False + gpu_node_group_names = [] + + return { + "name": self.config.project_name, + "environment": self.config.namespace, + "cloud-provider": self.config.provider.value, + "aws-region": self.config.amazon_web_services.region, + "external_container_reg": self.config.external_container_reg.enabled, + "gpu_enabled": gpu_enabled, + "gpu_node_group_names": gpu_node_group_names, + } + + def check(self, stage_outputs: stage_outputs: Dict[str, Dict[str, Any]]): + from kubernetes import client, config + from kubernetes.client.rest import ApiException + + config.load_kube_config( + config_file=stage_outputs["stages/02-infrastructure"]["kubeconfig_filename"][ + "value" + ] + ) + + try: + api_instance = client.CoreV1Api() + result = api_instance.list_namespace() + except ApiException: + self.log.error( + f"ERROR: After stage={self.name} unable to connect to kubernetes cluster" + ) + sys.exit(1) + + namespaces = {_.metadata.name for _ in result.items} + if self.config.namespace not in namespaces: + self.log.error( + f"ERROR: After stage={self.name} namespace={self.config.namespace} not provisioned within kubernetes cluster" + ) + sys.exit(1) + + self.log.info( + f"After stage={self.name} kubernetes initialized successfully" + ) + + + +@hookimpl +def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> List[NebariStage]: + return [ + KubernetesInitializeStage( + install_directory, + config, + template_directory=(pathlib.Path(__file__).parent / "template"), + stage_prefix=pathlib.Path("stages/03-kubernetes-initialize"), + ) + ] diff --git a/src/_nebari/template/stages/03-kubernetes-initialize/external-container-registry.tf b/src/_nebari/stages/kubernetes_initialize/template/external-container-registry.tf similarity index 100% rename from src/_nebari/template/stages/03-kubernetes-initialize/external-container-registry.tf rename to src/_nebari/stages/kubernetes_initialize/template/external-container-registry.tf diff --git a/src/_nebari/template/stages/04-kubernetes-ingress/locals.tf b/src/_nebari/stages/kubernetes_initialize/template/locals.tf similarity index 100% rename from src/_nebari/template/stages/04-kubernetes-ingress/locals.tf rename to src/_nebari/stages/kubernetes_initialize/template/locals.tf diff --git a/src/_nebari/template/stages/03-kubernetes-initialize/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf similarity index 100% rename from src/_nebari/template/stages/03-kubernetes-initialize/main.tf rename to src/_nebari/stages/kubernetes_initialize/template/main.tf diff --git a/src/_nebari/template/stages/03-kubernetes-initialize/modules/cluster-autoscaler/main.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf similarity index 100% rename from src/_nebari/template/stages/03-kubernetes-initialize/modules/cluster-autoscaler/main.tf rename to src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf diff --git a/src/_nebari/template/stages/03-kubernetes-initialize/modules/cluster-autoscaler/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/variables.tf similarity index 100% rename from src/_nebari/template/stages/03-kubernetes-initialize/modules/cluster-autoscaler/variables.tf rename to src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/variables.tf diff --git a/src/_nebari/template/stages/03-kubernetes-initialize/modules/extcr/main.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/extcr/main.tf similarity index 100% rename from src/_nebari/template/stages/03-kubernetes-initialize/modules/extcr/main.tf rename to src/_nebari/stages/kubernetes_initialize/template/modules/extcr/main.tf diff --git a/src/_nebari/template/stages/03-kubernetes-initialize/modules/extcr/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/extcr/variables.tf similarity index 100% rename from src/_nebari/template/stages/03-kubernetes-initialize/modules/extcr/variables.tf rename to src/_nebari/stages/kubernetes_initialize/template/modules/extcr/variables.tf diff --git a/src/_nebari/template/stages/03-kubernetes-initialize/modules/initialization/main.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/initialization/main.tf similarity index 100% rename from src/_nebari/template/stages/03-kubernetes-initialize/modules/initialization/main.tf rename to src/_nebari/stages/kubernetes_initialize/template/modules/initialization/main.tf diff --git a/src/_nebari/template/stages/03-kubernetes-initialize/modules/initialization/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/initialization/variables.tf similarity index 100% rename from src/_nebari/template/stages/03-kubernetes-initialize/modules/initialization/variables.tf rename to src/_nebari/stages/kubernetes_initialize/template/modules/initialization/variables.tf diff --git a/src/_nebari/template/stages/03-kubernetes-initialize/modules/nvidia-installer/aws-nvidia-installer.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/aws-nvidia-installer.tf similarity index 100% rename from src/_nebari/template/stages/03-kubernetes-initialize/modules/nvidia-installer/aws-nvidia-installer.tf rename to src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/aws-nvidia-installer.tf diff --git a/src/_nebari/template/stages/03-kubernetes-initialize/modules/nvidia-installer/gcp-nvidia-installer.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/gcp-nvidia-installer.tf similarity index 100% rename from src/_nebari/template/stages/03-kubernetes-initialize/modules/nvidia-installer/gcp-nvidia-installer.tf rename to src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/gcp-nvidia-installer.tf diff --git a/src/_nebari/template/stages/03-kubernetes-initialize/modules/nvidia-installer/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/variables.tf similarity index 100% rename from src/_nebari/template/stages/03-kubernetes-initialize/modules/nvidia-installer/variables.tf rename to src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/variables.tf diff --git a/src/_nebari/template/stages/03-kubernetes-initialize/modules/traefik_crds/main.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/traefik_crds/main.tf similarity index 100% rename from src/_nebari/template/stages/03-kubernetes-initialize/modules/traefik_crds/main.tf rename to src/_nebari/stages/kubernetes_initialize/template/modules/traefik_crds/main.tf diff --git a/src/_nebari/template/stages/03-kubernetes-initialize/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/variables.tf similarity index 100% rename from src/_nebari/template/stages/03-kubernetes-initialize/variables.tf rename to src/_nebari/stages/kubernetes_initialize/template/variables.tf diff --git a/src/_nebari/template/stages/04-kubernetes-ingress/versions.tf b/src/_nebari/stages/kubernetes_initialize/template/versions.tf similarity index 100% rename from src/_nebari/template/stages/04-kubernetes-ingress/versions.tf rename to src/_nebari/stages/kubernetes_initialize/template/versions.tf diff --git a/src/_nebari/stages/kubernetes_keycloak/__init__.py b/src/_nebari/stages/kubernetes_keycloak/__init__.py new file mode 100644 index 000000000..e24ba97d3 --- /dev/null +++ b/src/_nebari/stages/kubernetes_keycloak/__init__.py @@ -0,0 +1,165 @@ +from typing import List, Dict +import pathlib +import sys +import json + +from nebari.hookspecs import NebariStage, hookimpl +from _nebari.utils import modified_environ +from _nebari.stages.base import NebariTerraformStage +from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider + +from _nebari import schema + + +@contextlib.contextmanager +def keycloak_provider_context(keycloak_credentials: Dict[str, str]): + credential_mapping = { + "client_id": "KEYCLOAK_CLIENT_ID", + "url": "KEYCLOAK_URL", + "username": "KEYCLOAK_USER", + "password": "KEYCLOAK_PASSWORD", + "realm": "KEYCLOAK_REALM", + } + + credentials = {credential_mapping[k]: v for k, v in keycloak_credentials.items()} + with modified_environ(**credentials): + yield + + +def _calculate_node_groups(config: schema.Main): + if config.provider == schema.ProviderEnum.aws: + return { + group: {"key": "eks.amazonaws.com/nodegroup", "value": group} + for group in ["general", "user", "worker"] + } + elif config.provider == schema.ProviderEnum.gcp: + return { + group: {"key": "cloud.google.com/gke-nodepool", "value": group} + for group in ["general", "user", "worker"] + } + elif config.provider == schema.ProviderEnum.azure: + return { + group: {"key": "azure-node-pool", "value": group} + for group in ["general", "user", "worker"] + } + elif config.provider == schema.ProviderEnum.do: + return { + group: {"key": "doks.digitalocean.com/node-pool", "value": group} + for group in ["general", "user", "worker"] + } + elif config.provider == schema.ProviderEnum.existing: + return config.existing.node_selectors + else: + return config.local.node_selectors + + +class KubernetesKeycloakStage(NebariTerraformStage): + @property + def name(self): + return "05-kubernetes-keycloak" + + @property + def priority(self): + return 50 + + def tf_objects(self) -> List[Dict]: + return [ + NebariTerraformState(self.name, self.config), + NebariKubernetesProvider(self.config), + NebariHelmProvider(self.config), + ] + + def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): + return { + "name": self.config.project_name, + "environment": self.config.namespace, + "endpoint": self.config.domain, + "initial-root-password": self.config.security.keycloak.initial_root_password, + "overrides": [ + json.dumps(self.config.security.keycloak.overrides) + ], + "node-group": _calculate_node_groups(self.config)["general"], + } + + + def check(self, stage_outputs: stage_outputs: Dict[str, Dict[str, Any]]): + from keycloak import KeycloakAdmin + from keycloak.exceptions import KeycloakError + + keycloak_url = ( + f"{stage_outputs['stages/' + self.name]['keycloak_credentials']['value']['url']}/auth/" + ) + + def _attempt_keycloak_connection( + keycloak_url, + username, + password, + realm_name, + client_id, + verify=False, + num_attempts=NUM_ATTEMPTS, + timeout=TIMEOUT, + ): + for i in range(num_attempts): + try: + KeycloakAdmin( + keycloak_url, + username=username, + password=password, + realm_name=realm_name, + client_id=client_id, + verify=verify, + ) + self.log.info(f"Attempt {i+1} succeeded connecting to keycloak master realm") + return True + except KeycloakError: + self.log.info(f"Attempt {i+1} failed connecting to keycloak master realm") + time.sleep(timeout) + return False + + if not _attempt_keycloak_connection( + keycloak_url, + stage_outputs['stages/' + self.name]["keycloak_credentials"]["value"]["username"], + stage_outputs['stages/' + self.name]["keycloak_credentials"]["value"]["password"], + stage_outputs['stages/' + self.name]["keycloak_credentials"]["value"]["realm"], + stage_outputs['stages/' + self.name]["keycloak_credentials"]["value"]["client_id"], + verify=False, + ): + self.log.error( + f"ERROR: unable to connect to keycloak master realm at url={keycloak_url} with root credentials" + ) + sys.exit(1) + + self.log.info("Keycloak service successfully started") + + @contextlib.contextmanager + def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): + with super().deploy(stage_outputs): + with keycloak_provider_context( + stage_outputs["stages/" + self.name]["keycloak_credentials"][ + "value" + ] + ): + yield + + @contextlib.contextmanager + def destroy(self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool]): + with super().destroy(stage_outputs, status): + with keycloak_provider_context( + stage_outputs["stages/" + self.name]["keycloak_credentials"][ + "value" + ] + ): + yield + + +@hookimpl +def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> [NebariStage]: + return [ + KubernetesKeycloakStage( + install_directory, + config, + template_directory=(pathlib.Path(__file__).parent / "template"), + stage_prefix=pathlib.Path("stages/05-kubernetes-keycloak"), + ) + ] diff --git a/src/_nebari/template/stages/05-kubernetes-keycloak/main.tf b/src/_nebari/stages/kubernetes_keycloak/template/main.tf similarity index 100% rename from src/_nebari/template/stages/05-kubernetes-keycloak/main.tf rename to src/_nebari/stages/kubernetes_keycloak/template/main.tf diff --git a/src/_nebari/template/stages/05-kubernetes-keycloak/modules/kubernetes/keycloak-helm/main.tf b/src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/main.tf similarity index 100% rename from src/_nebari/template/stages/05-kubernetes-keycloak/modules/kubernetes/keycloak-helm/main.tf rename to src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/main.tf diff --git a/src/_nebari/template/stages/05-kubernetes-keycloak/modules/kubernetes/keycloak-helm/outputs.tf b/src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/outputs.tf similarity index 100% rename from src/_nebari/template/stages/05-kubernetes-keycloak/modules/kubernetes/keycloak-helm/outputs.tf rename to src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/outputs.tf diff --git a/src/_nebari/template/stages/05-kubernetes-keycloak/modules/kubernetes/keycloak-helm/values.yaml b/src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/values.yaml similarity index 100% rename from src/_nebari/template/stages/05-kubernetes-keycloak/modules/kubernetes/keycloak-helm/values.yaml rename to src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/values.yaml diff --git a/src/_nebari/template/stages/05-kubernetes-keycloak/modules/kubernetes/keycloak-helm/variables.tf b/src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/variables.tf similarity index 100% rename from src/_nebari/template/stages/05-kubernetes-keycloak/modules/kubernetes/keycloak-helm/variables.tf rename to src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/variables.tf diff --git a/src/_nebari/template/stages/05-kubernetes-keycloak/outputs.tf b/src/_nebari/stages/kubernetes_keycloak/template/outputs.tf similarity index 100% rename from src/_nebari/template/stages/05-kubernetes-keycloak/outputs.tf rename to src/_nebari/stages/kubernetes_keycloak/template/outputs.tf diff --git a/src/_nebari/template/stages/05-kubernetes-keycloak/variables.tf b/src/_nebari/stages/kubernetes_keycloak/template/variables.tf similarity index 100% rename from src/_nebari/template/stages/05-kubernetes-keycloak/variables.tf rename to src/_nebari/stages/kubernetes_keycloak/template/variables.tf diff --git a/src/_nebari/template/stages/05-kubernetes-keycloak/versions.tf b/src/_nebari/stages/kubernetes_keycloak/template/versions.tf similarity index 100% rename from src/_nebari/template/stages/05-kubernetes-keycloak/versions.tf rename to src/_nebari/stages/kubernetes_keycloak/template/versions.tf diff --git a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py new file mode 100644 index 000000000..c309986eb --- /dev/null +++ b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py @@ -0,0 +1,116 @@ +from typing import List, Dict +import pathlib +import sys + +from nebari.hookspecs import NebariStage, hookimpl +from _nebari.stages.base import NebariTerraformStage +from _nebari.stages.tf_objects import NebariTerraformState + +from _nebari import schema + + +class KubernetesKeycloakConfigurationStage(NebariTerraformStage): + @property + def name(self): + return "06-kubernetes-keycloak-configuration" + + @property + def priority(self): + return 60 + + def tf_objects(self) -> List[Dict]: + return [ + NebariTerraformState(self.name, self.config), + ] + + def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): + realm_id = "nebari" + + users_group = ( + ["users"] if self.config.security.shared_users_group else [] + ) + + return { + "realm": realm_id, + "realm_display_name": self.config.security.keycloak.realm_display_name, + "authentication": self.config.security.authentication, + "keycloak_groups": ["superadmin", "admin", "developer", "analyst"] + + users_group, + "default_groups": ["analyst"] + users_group, + } + + def check(self, stage_outputs: stage_outputs: Dict[str, Dict[str, Any]]): + directory = "stages/05-kubernetes-keycloak" + + from keycloak import KeycloakAdmin + from keycloak.exceptions import KeycloakError + + keycloak_url = ( + f"{stage_outputs[directory]['keycloak_credentials']['value']['url']}/auth/" + ) + + def _attempt_keycloak_connection( + keycloak_url, + username, + password, + realm_name, + client_id, + nebari_realm, + verify=False, + num_attempts=NUM_ATTEMPTS, + timeout=TIMEOUT, + ): + for i in range(num_attempts): + try: + realm_admin = KeycloakAdmin( + keycloak_url, + username=username, + password=password, + realm_name=realm_name, + client_id=client_id, + verify=verify, + ) + existing_realms = {_["id"] for _ in realm_admin.get_realms()} + if nebari_realm in existing_realms: + self.log.info( + f"Attempt {i+1} succeeded connecting to keycloak and nebari realm={nebari_realm} exists" + ) + return True + else: + self.log.info( + f"Attempt {i+1} succeeded connecting to keycloak but nebari realm did not exist" + ) + except KeycloakError: + self.log.info(f"Attempt {i+1} failed connecting to keycloak master realm") + time.sleep(timeout) + return False + + if not _attempt_keycloak_connection( + keycloak_url, + stage_outputs[directory]["keycloak_credentials"]["value"]["username"], + stage_outputs[directory]["keycloak_credentials"]["value"]["password"], + stage_outputs[directory]["keycloak_credentials"]["value"]["realm"], + stage_outputs[directory]["keycloak_credentials"]["value"]["client_id"], + nebari_realm=stage_outputs["stages/06-kubernetes-keycloak-configuration"][ + "realm_id" + ]["value"], + verify=False, + ): + self.log.error( + "ERROR: unable to connect to keycloak master realm and ensure that nebari realm exists" + ) + sys.exit(1) + + self.log.info("Keycloak service successfully started with nebari realm") + + +@hookimpl +def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> List[NebariStage]: + return [ + KubernetesKeycloakConfigurationStage( + install_directory, + config, + template_directory=(pathlib.Path(__file__).parent / "template"), + stage_directory=pathlib.Path("stages/06-kubernetes-keycloak-configuration"), + ) + ] diff --git a/src/_nebari/template/stages/06-kubernetes-keycloak-configuration/main.tf b/src/_nebari/stages/kubernetes_keycloak_configuration/template/main.tf similarity index 100% rename from src/_nebari/template/stages/06-kubernetes-keycloak-configuration/main.tf rename to src/_nebari/stages/kubernetes_keycloak_configuration/template/main.tf diff --git a/src/_nebari/template/stages/06-kubernetes-keycloak-configuration/outputs.tf b/src/_nebari/stages/kubernetes_keycloak_configuration/template/outputs.tf similarity index 100% rename from src/_nebari/template/stages/06-kubernetes-keycloak-configuration/outputs.tf rename to src/_nebari/stages/kubernetes_keycloak_configuration/template/outputs.tf diff --git a/src/_nebari/template/stages/06-kubernetes-keycloak-configuration/permissions.tf b/src/_nebari/stages/kubernetes_keycloak_configuration/template/permissions.tf similarity index 100% rename from src/_nebari/template/stages/06-kubernetes-keycloak-configuration/permissions.tf rename to src/_nebari/stages/kubernetes_keycloak_configuration/template/permissions.tf diff --git a/src/_nebari/template/stages/06-kubernetes-keycloak-configuration/providers.tf b/src/_nebari/stages/kubernetes_keycloak_configuration/template/providers.tf similarity index 100% rename from src/_nebari/template/stages/06-kubernetes-keycloak-configuration/providers.tf rename to src/_nebari/stages/kubernetes_keycloak_configuration/template/providers.tf diff --git a/src/_nebari/template/stages/06-kubernetes-keycloak-configuration/social_auth.tf b/src/_nebari/stages/kubernetes_keycloak_configuration/template/social_auth.tf similarity index 100% rename from src/_nebari/template/stages/06-kubernetes-keycloak-configuration/social_auth.tf rename to src/_nebari/stages/kubernetes_keycloak_configuration/template/social_auth.tf diff --git a/src/_nebari/template/stages/06-kubernetes-keycloak-configuration/variables.tf b/src/_nebari/stages/kubernetes_keycloak_configuration/template/variables.tf similarity index 100% rename from src/_nebari/template/stages/06-kubernetes-keycloak-configuration/variables.tf rename to src/_nebari/stages/kubernetes_keycloak_configuration/template/variables.tf diff --git a/src/_nebari/template/stages/06-kubernetes-keycloak-configuration/versions.tf b/src/_nebari/stages/kubernetes_keycloak_configuration/template/versions.tf similarity index 100% rename from src/_nebari/template/stages/06-kubernetes-keycloak-configuration/versions.tf rename to src/_nebari/stages/kubernetes_keycloak_configuration/template/versions.tf diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py new file mode 100644 index 000000000..86c8c08bc --- /dev/null +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -0,0 +1,208 @@ +from typing import List, Dict +import pathlib +import sys +from urllib.parse import urlencode + +from nebari.hookspecs import NebariStage, hookimpl +from _nebari.stages.base import NebariTerraformStage +from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider + +from _nebari import schema + +# check and retry settings +NUM_ATTEMPTS = 10 +TIMEOUT = 10 # seconds + + +def _split_docker_image_name(image_name): + name, tag = image_name.split(":") + return {"name": name, "tag": tag} + + +def _calculate_node_groups(config: schema.Main): + if config.provider == schema.ProviderEnum.aws: + return { + group: {"key": "eks.amazonaws.com/nodegroup", "value": group} + for group in ["general", "user", "worker"] + } + elif config.provider == schema.ProviderEnum.gcp: + return { + group: {"key": "cloud.google.com/gke-nodepool", "value": group} + for group in ["general", "user", "worker"] + } + elif config.provider == schema.ProviderEnum.azure: + return { + group: {"key": "azure-node-pool", "value": group} + for group in ["general", "user", "worker"] + } + elif config.provider == schema.ProviderEnum.do: + return { + group: {"key": "doks.digitalocean.com/node-pool", "value": group} + for group in ["general", "user", "worker"] + } + elif config.provider == schema.ProviderEnum.existing: + return config.existing.node_selectors + else: + return config.local.node_selectors + + +class KubernetesServicesStage(NebariTerraformStage): + @property + def name(self): + return "07-kubernetes-services" + + @property + def priority(self): + return 70 + + def tf_objects(self) -> List[Dict]: + return [ + NebariTerraformState(self.name, self.config), + NebariKubernetesProvider(self.config), + NebariHelmProvider(self.config), + ] + + def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): + final_logout_uri = f"https://{config['domain']}/hub/login" + + # Compound any logout URLs from extensions so they are are logged out in succession + # when Keycloak and JupyterHub are logged out + for ext in self.config.tf_extensions: + if ext.logout != "": + final_logout_uri = "{}?{}".format( + f"https://{self.config.domain}/{ext.urlslug}{ext.logout}", + urlencode({"redirect_uri": final_logout_uri}), + ) + + jupyterhub_theme = self.config.theme.jupyterhub + if self.config.theme.jupyterhub.display_version and ( + not self.config.theme.jupyterhub.version + ): + jupyterhub_theme.update({"version": f"v{self.config.nebari_version}"}) + + return { + "name": self.config.project_name, + "environment": self.config.namespace, + "endpoint": self.config.domain, + "realm_id": stage_outputs["stages/06-kubernetes-keycloak-configuration"][ + "realm_id" + ]["value"], + "node_groups": _calculate_node_groups(self.config), + # conda-store + "conda-store-environments": self.config.environments, + "conda-store-filesystem-storage": self.config.storage.conda_store, + "conda-store-service-token-scopes": { + "cdsdashboards": { + "primary_namespace": "cdsdashboards", + "role_bindings": { + "*/*": ["viewer"], + }, + }, + "dask-gateway": { + "primary_namespace": "", + "role_bindings": { + "*/*": ["viewer"], + }, + }, + }, + "conda-store-default-namespace": self.config.conda_store.default_namespace, + "conda-store-extra-settings": self.config.conda_store.extra_settings, + "conda-store-extra-config": self.config.conda_store.extra_config, + "conda-store-image-tag": self.config.conda_store.image_tag, + # jupyterhub + "cdsdashboards": self.config.cdsdashboards, + "jupyterhub-theme": jupyterhub_theme, + "jupyterhub-image": _split_docker_image_name( + self.config.default_images.jupyterhub + ), + "jupyterhub-shared-storage": self.config.storage.shared_filesystem, + "jupyterhub-shared-endpoint": stage_outputs["stages/02-infrastructure"] + .get("nfs_endpoint", {}) + .get("value"), + "jupyterlab-profiles": self.config.profiles.jupyterlab, + "jupyterlab-image": _split_docker_image_name( + self.config.default_images.jupyterlab + ), + "jupyterhub-overrides": [ + json.dumps(self.config.jupyterhub.overrides) + ], + "jupyterhub-hub-extraEnv": json.dumps( + self.config.jupyterhub.overrides.hub.extraEnv + ), + # jupyterlab + "idle-culler-settings": self.config.jupyterlab.idle_culler, + # dask-gateway + "dask-worker-image": _split_docker_image_name( + self.config.default_images.dask_worker + ), + "dask-gateway-profiles": self.config.profiles.dask_worker, + # monitoring + "monitoring-enabled": self.config.monitoring.enabled, + # argo-worfklows + "argo-workflows-enabled": self.config.argo_workflows.enabled, + "argo-workflows-overrides": [ + json.dumps(self.config.argo_workflows.overrides) + ], + "nebari-workflow-controller": self.config.argo_workflows.nebari_workflow_controller.enabled, + "keycloak-read-only-user-credentials": stage_outputs[ + "stages/06-kubernetes-keycloak-configuration" + ]["keycloak-read-only-user-credentials"]["value"], + "workflow-controller-image-tag": self.config.argo_workflows.nebari_workflow_controller.image_tag, + # kbatch + "kbatch-enabled": self.config.kbatch.enabled, + # prefect + "prefect-enabled": self.config.prefect.enabled, + "prefect-token": self.config.prefect.token, + "prefect-image": self.config.prefect.image, + "prefect-overrides": self.config.prefect.overrides, + # clearml + "clearml-enabled": self.config.clearml.enabled + "clearml-enable-forwardauth": self.config.clearml.enable_forward_auth, + "clearml-overrides": [ + json.dumps(self.config.clearml.overrides) + ], + "jupyterhub-logout-redirect-url": final_logout_uri, + } + + + def check(self, stage_outputs: stage_outputs: Dict[str, Dict[str, Any]]): + directory = "stages/07-kubernetes-services" + import requests + + # suppress insecure warnings + import urllib3 + + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + def _attempt_connect_url( + url, verify=False, num_attempts=NUM_ATTEMPTS, timeout=TIMEOUT + ): + for i in range(num_attempts): + response = requests.get(url, verify=verify, timeout=timeout) + if response.status_code < 400: + self.log.info(f"Attempt {i+1} health check succeeded for url={url}") + return True + else: + self.log.info(f"Attempt {i+1} health check failed for url={url}") + time.sleep(timeout) + return False + + services = stage_outputs[directory]["service_urls"]["value"] + for service_name, service in services.items(): + service_url = service["health_url"] + if service_url and not _attempt_connect_url(service_url): + self.log.error(f"ERROR: Service {service_name} DOWN when checking url={service_url}") + sys.exit(1) + + + +@hookimpl +def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> List[NebariStage]: + return [ + KubernetesServicesStage( + install_directory, + config, + template_directory=(pathlib.Path(__file__).parent / "template"), + stage_prefix=pathlib.Path("stages/07-kubernetes-services"), + ) + ] diff --git a/src/_nebari/template/stages/07-kubernetes-services/argo-workflows.tf b/src/_nebari/stages/kubernetes_services/template/argo-workflows.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/argo-workflows.tf rename to src/_nebari/stages/kubernetes_services/template/argo-workflows.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/clearml.tf b/src/_nebari/stages/kubernetes_services/template/clearml.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/clearml.tf rename to src/_nebari/stages/kubernetes_services/template/clearml.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/conda-store.tf b/src/_nebari/stages/kubernetes_services/template/conda-store.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/conda-store.tf rename to src/_nebari/stages/kubernetes_services/template/conda-store.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/dask_gateway.tf b/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/dask_gateway.tf rename to src/_nebari/stages/kubernetes_services/template/dask_gateway.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/forward-auth.tf b/src/_nebari/stages/kubernetes_services/template/forward-auth.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/forward-auth.tf rename to src/_nebari/stages/kubernetes_services/template/forward-auth.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/jupyterhub.tf b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/jupyterhub.tf rename to src/_nebari/stages/kubernetes_services/template/jupyterhub.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/jupyterhub_ssh.tf b/src/_nebari/stages/kubernetes_services/template/jupyterhub_ssh.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/jupyterhub_ssh.tf rename to src/_nebari/stages/kubernetes_services/template/jupyterhub_ssh.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/kbatch.tf b/src/_nebari/stages/kubernetes_services/template/kbatch.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/kbatch.tf rename to src/_nebari/stages/kubernetes_services/template/kbatch.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/locals.tf b/src/_nebari/stages/kubernetes_services/template/locals.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/locals.tf rename to src/_nebari/stages/kubernetes_services/template/locals.tf diff --git a/src/_nebari/template/__init__.py b/src/_nebari/stages/kubernetes_services/template/modules/__init__.py similarity index 100% rename from src/_nebari/template/__init__.py rename to src/_nebari/stages/kubernetes_services/template/modules/__init__.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/__init__.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/__init__.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/__init__.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/__init__.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/forwardauth/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/main.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/forwardauth/main.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/main.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/forwardauth/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/forwardauth/variables.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/nfs-mount/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/nfs-mount/main.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/nfs-mount/main.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/nfs-mount/main.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/nfs-mount/outputs.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/nfs-mount/outputs.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/nfs-mount/outputs.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/nfs-mount/outputs.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/nfs-mount/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/nfs-mount/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/nfs-mount/variables.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/nfs-mount/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/nfs-server/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/nfs-server/main.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/nfs-server/main.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/nfs-server/main.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/nfs-server/output.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/nfs-server/output.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/nfs-server/output.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/nfs-server/output.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/nfs-server/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/nfs-server/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/nfs-server/variables.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/nfs-server/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/__init__.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/__init__.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/__init__.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/__init__.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/main.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/main.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/values.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/values.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/values.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/values.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/variables.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/versions.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/versions.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/versions.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/argo-workflows/versions.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/Chart.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/Chart.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/Chart.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/Chart.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/LICENSE b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/LICENSE similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/LICENSE rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/LICENSE diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/README.md b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/README.md similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/README.md rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/README.md diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/charts/mongodb-10.3.7.tgz b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/charts/mongodb-10.3.7.tgz similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/charts/mongodb-10.3.7.tgz rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/charts/mongodb-10.3.7.tgz diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/charts/redis-10.9.0.tgz b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/charts/redis-10.9.0.tgz similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/charts/redis-10.9.0.tgz rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/charts/redis-10.9.0.tgz diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/NOTES.txt b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/NOTES.txt similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/NOTES.txt rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/NOTES.txt diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/_helpers.tpl b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/_helpers.tpl similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/_helpers.tpl rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/_helpers.tpl diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/deployment-agent.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/deployment-agent.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/deployment-agent.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/deployment-agent.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/deployment-agentservices.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/deployment-agentservices.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/deployment-agentservices.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/deployment-agentservices.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/deployment-apiserver.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/deployment-apiserver.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/deployment-apiserver.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/deployment-apiserver.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/deployment-elastic.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/deployment-elastic.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/deployment-elastic.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/deployment-elastic.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/deployment-fileserver.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/deployment-fileserver.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/deployment-fileserver.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/deployment-fileserver.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/deployment-webserver.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/deployment-webserver.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/deployment-webserver.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/deployment-webserver.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/ingress.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/ingress.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/ingress.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/ingress.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/pvc-agentservices.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/pvc-agentservices.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/pvc-agentservices.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/pvc-agentservices.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/pvc-apiserver.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/pvc-apiserver.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/pvc-apiserver.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/pvc-apiserver.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/pvc-fileserver.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/pvc-fileserver.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/pvc-fileserver.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/pvc-fileserver.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/secret-agent.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/secret-agent.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/secret-agent.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/secret-agent.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/secrets.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/secrets.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/secrets.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/secrets.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/service-apiserver.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/service-apiserver.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/service-apiserver.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/service-apiserver.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/service-fileserver.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/service-fileserver.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/service-fileserver.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/service-fileserver.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/service-webserver.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/service-webserver.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/templates/service-webserver.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/templates/service-webserver.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/values.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/values.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/chart/values.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/chart/values.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/ingress.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/ingress.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/ingress.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/ingress.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/main.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/main.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/main.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/clearml/variables.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/clearml/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/__init__.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/__init__.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/__init__.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/__init__.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/__init__.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/__init__.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/__init__.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/__init__.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/config/conda_store_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/config/conda_store_config.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/output.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/output.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/output.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/output.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/server.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/server.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/server.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/server.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/storage.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/storage.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/storage.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/storage.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/variables.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/worker.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/worker.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/worker.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/worker.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/__init__.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/__init__.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/__init__.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/__init__.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/controler.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/controler.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/controler.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/controler.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/crds.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/crds.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/crds.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/crds.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/config/__init__.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/__init__.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/config/__init__.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/__init__.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/controller_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/controller_config.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/controller_config.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/controller_config.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/gateway_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/gateway_config.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/gateway.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/gateway.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/gateway.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/gateway.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/main.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/main.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/main.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/middleware.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/middleware.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/middleware.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/middleware.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/outputs.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/outputs.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/outputs.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/outputs.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/variables.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub-ssh/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub-ssh/main.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub-ssh/main.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub-ssh/main.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub-ssh/sftp.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub-ssh/sftp.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub-ssh/sftp.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub-ssh/sftp.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub-ssh/ssh.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub-ssh/ssh.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub-ssh/ssh.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub-ssh/ssh.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub-ssh/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub-ssh/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub-ssh/variables.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub-ssh/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/__init__.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/__init__.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/__init__.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/__init__.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/configmaps.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/configmaps.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/__init__.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/__init__.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/__init__.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/__init__.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/__init__.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/ipython/__init__.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/__init__.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/ipython/__init__.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/ipython/ipython_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/ipython/ipython_config.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/ipython/ipython_config.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/ipython/ipython_config.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/01-theme.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/01-theme.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/01-theme.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/01-theme.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/__init__.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/__init__.py similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/__init__.py rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/__init__.py diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterlab/overrides.json diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/skel/.bash_logout b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/skel/.bash_logout similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/skel/.bash_logout rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/skel/.bash_logout diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/skel/.bashrc b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/skel/.bashrc similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/skel/.bashrc rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/skel/.bashrc diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/skel/.profile b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/skel/.profile similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/skel/.profile rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/skel/.profile diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/main.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/outputs.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/outputs.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/outputs.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/outputs.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/values.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/values.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/values.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/values.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/variables.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/kbatch/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/kbatch/main.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/kbatch/main.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/kbatch/main.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/kbatch/values.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/kbatch/values.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/kbatch/values.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/kbatch/values.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/kbatch/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/kbatch/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/kbatch/variables.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/kbatch/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/kbatch/versions.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/kbatch/versions.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/kbatch/versions.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/kbatch/versions.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/keycloak-client/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/keycloak-client/main.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/keycloak-client/outputs.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/keycloak-client/outputs.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/keycloak-client/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/keycloak-client/variables.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/keycloak-client/versions.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/versions.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/keycloak-client/versions.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/versions.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/minio/ingress.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/minio/ingress.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/minio/ingress.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/minio/ingress.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/minio/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/minio/main.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/minio/main.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/minio/main.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/minio/outputs.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/minio/outputs.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/minio/outputs.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/minio/outputs.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/minio/values.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/minio/values.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/minio/values.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/minio/values.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/minio/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/minio/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/minio/variables.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/minio/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/dashboards/Main/cluster_information.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/cluster_information.json similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/dashboards/Main/cluster_information.json rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/cluster_information.json diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/dashboards/Main/conda_store.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/conda_store.json similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/dashboards/Main/conda_store.json rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/conda_store.json diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/dashboards/Main/jupyterhub_dashboard.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/jupyterhub_dashboard.json similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/dashboards/Main/jupyterhub_dashboard.json rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/jupyterhub_dashboard.json diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/dashboards/Main/keycloak.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/keycloak.json similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/dashboards/Main/keycloak.json rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/keycloak.json diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/dashboards/Main/traefik.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/traefik.json similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/dashboards/Main/traefik.json rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/traefik.json diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/dashboards/Main/usage_report.json b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/usage_report.json similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/dashboards/Main/usage_report.json rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/dashboards/Main/usage_report.json diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/main.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/main.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/main.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/values.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/values.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/values.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/values.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/variables.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/versions.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/versions.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/monitoring/versions.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/versions.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/postgresql/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/postgresql/main.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/postgresql/main.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/postgresql/main.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/postgresql/outputs.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/postgresql/outputs.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/postgresql/outputs.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/postgresql/outputs.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/postgresql/values.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/postgresql/values.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/postgresql/values.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/postgresql/values.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/postgresql/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/postgresql/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/postgresql/variables.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/postgresql/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/chart/.helmignore b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/chart/.helmignore similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/chart/.helmignore rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/chart/.helmignore diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/chart/Chart.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/chart/Chart.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/chart/Chart.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/chart/Chart.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/chart/templates/_helpers.tpl b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/chart/templates/_helpers.tpl similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/chart/templates/_helpers.tpl rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/chart/templates/_helpers.tpl diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/chart/templates/prefect.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/chart/templates/prefect.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/chart/templates/prefect.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/chart/templates/prefect.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/chart/templates/secret.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/chart/templates/secret.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/chart/templates/secret.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/chart/templates/secret.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/chart/values.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/chart/values.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/chart/values.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/chart/values.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/main.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/main.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/main.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/values.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/values.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/values.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/values.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/prefect/variables.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/prefect/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/redis/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/redis/main.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/redis/main.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/redis/main.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/redis/outputs.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/redis/outputs.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/redis/outputs.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/redis/outputs.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/redis/values.yaml b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/redis/values.yaml similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/redis/values.yaml rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/redis/values.yaml diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/redis/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/redis/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/redis/variables.tf rename to src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/redis/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/monitoring.tf b/src/_nebari/stages/kubernetes_services/template/monitoring.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/monitoring.tf rename to src/_nebari/stages/kubernetes_services/template/monitoring.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/outputs.tf b/src/_nebari/stages/kubernetes_services/template/outputs.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/outputs.tf rename to src/_nebari/stages/kubernetes_services/template/outputs.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/prefect.tf b/src/_nebari/stages/kubernetes_services/template/prefect.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/prefect.tf rename to src/_nebari/stages/kubernetes_services/template/prefect.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/providers.tf b/src/_nebari/stages/kubernetes_services/template/providers.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/providers.tf rename to src/_nebari/stages/kubernetes_services/template/providers.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/variables.tf b/src/_nebari/stages/kubernetes_services/template/variables.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/variables.tf rename to src/_nebari/stages/kubernetes_services/template/variables.tf diff --git a/src/_nebari/template/stages/07-kubernetes-services/versions.tf b/src/_nebari/stages/kubernetes_services/template/versions.tf similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/versions.tf rename to src/_nebari/stages/kubernetes_services/template/versions.tf diff --git a/src/_nebari/stages/nebari_tf_extensions/__init__.py b/src/_nebari/stages/nebari_tf_extensions/__init__.py new file mode 100644 index 000000000..5a80a17c2 --- /dev/null +++ b/src/_nebari/stages/nebari_tf_extensions/__init__.py @@ -0,0 +1,53 @@ +from typing import List, Dict +import pathlib +import sys + +from nebari.hookspecs import NebariStage, hookimpl +from _nebari.stages.base import NebariTerraformStage +from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider + +from _nebari import schema + + +class NebariTFExtensionsStage(NebariTerraformStage): + @property + def name(self): + return "08-nebari-tf-extensions" + + @property + def priority(self): + return 80 + + def tf_objects(self) -> List[Dict]: + return [ + NebariTerraformState(self.name, self.config), + NebariKubernetesProvider(self.config), + NebariHelmProvider(self.config), + ] + + def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): + return { + "environment": self.config.namespace, + "endpoint": self.config.domain, + "realm_id": stage_outputs["stages/06-kubernetes-keycloak-configuration"][ + "realm_id" + ]["value"], + "tf_extensions": self.config.tf_extensions, + "nebari_config_yaml": self.config, + "keycloak_nebari_bot_password": stage_outputs["stages/05-kubernetes-keycloak"][ + "keycloak_nebari_bot_password" + ]["value"], + "helm_extensions": self.config.helm_extensions, + } + + +@hookimpl +def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> List[NebariStage]: + return [ + NebariTFExtensionsStage( + install_directory, + config, + template_directory=(pathlib.Path(__file__).parent / "template"), + stage_prefix=pathlib.Path("stages/08-nebari-tf-extensions"), + ) + ] diff --git a/src/_nebari/template/stages/08-nebari-tf-extensions/helm-extension.tf b/src/_nebari/stages/nebari_tf_extensions/template/helm-extension.tf similarity index 100% rename from src/_nebari/template/stages/08-nebari-tf-extensions/helm-extension.tf rename to src/_nebari/stages/nebari_tf_extensions/template/helm-extension.tf diff --git a/src/_nebari/template/stages/08-nebari-tf-extensions/modules/helm-extensions/main.tf b/src/_nebari/stages/nebari_tf_extensions/template/modules/helm-extensions/main.tf similarity index 100% rename from src/_nebari/template/stages/08-nebari-tf-extensions/modules/helm-extensions/main.tf rename to src/_nebari/stages/nebari_tf_extensions/template/modules/helm-extensions/main.tf diff --git a/src/_nebari/template/stages/08-nebari-tf-extensions/modules/helm-extensions/variables.tf b/src/_nebari/stages/nebari_tf_extensions/template/modules/helm-extensions/variables.tf similarity index 100% rename from src/_nebari/template/stages/08-nebari-tf-extensions/modules/helm-extensions/variables.tf rename to src/_nebari/stages/nebari_tf_extensions/template/modules/helm-extensions/variables.tf diff --git a/src/_nebari/template/stages/08-nebari-tf-extensions/modules/nebariextension/ingress.tf b/src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/ingress.tf similarity index 100% rename from src/_nebari/template/stages/08-nebari-tf-extensions/modules/nebariextension/ingress.tf rename to src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/ingress.tf diff --git a/src/_nebari/template/stages/08-nebari-tf-extensions/modules/nebariextension/keycloak-config.tf b/src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/keycloak-config.tf similarity index 100% rename from src/_nebari/template/stages/08-nebari-tf-extensions/modules/nebariextension/keycloak-config.tf rename to src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/keycloak-config.tf diff --git a/src/_nebari/template/stages/08-nebari-tf-extensions/modules/nebariextension/locals.tf b/src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/locals.tf similarity index 100% rename from src/_nebari/template/stages/08-nebari-tf-extensions/modules/nebariextension/locals.tf rename to src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/locals.tf diff --git a/src/_nebari/template/stages/08-nebari-tf-extensions/modules/nebariextension/main.tf b/src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/main.tf similarity index 100% rename from src/_nebari/template/stages/08-nebari-tf-extensions/modules/nebariextension/main.tf rename to src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/main.tf diff --git a/src/_nebari/template/stages/08-nebari-tf-extensions/modules/nebariextension/variables.tf b/src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/variables.tf similarity index 100% rename from src/_nebari/template/stages/08-nebari-tf-extensions/modules/nebariextension/variables.tf rename to src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/variables.tf diff --git a/src/_nebari/template/stages/08-nebari-tf-extensions/nebari-config.tf b/src/_nebari/stages/nebari_tf_extensions/template/nebari-config.tf similarity index 100% rename from src/_nebari/template/stages/08-nebari-tf-extensions/nebari-config.tf rename to src/_nebari/stages/nebari_tf_extensions/template/nebari-config.tf diff --git a/src/_nebari/template/stages/08-nebari-tf-extensions/providers.tf b/src/_nebari/stages/nebari_tf_extensions/template/providers.tf similarity index 100% rename from src/_nebari/template/stages/08-nebari-tf-extensions/providers.tf rename to src/_nebari/stages/nebari_tf_extensions/template/providers.tf diff --git a/src/_nebari/template/stages/08-nebari-tf-extensions/tf-extensions.tf b/src/_nebari/stages/nebari_tf_extensions/template/tf-extensions.tf similarity index 100% rename from src/_nebari/template/stages/08-nebari-tf-extensions/tf-extensions.tf rename to src/_nebari/stages/nebari_tf_extensions/template/tf-extensions.tf diff --git a/src/_nebari/template/stages/08-nebari-tf-extensions/variables.tf b/src/_nebari/stages/nebari_tf_extensions/template/variables.tf similarity index 100% rename from src/_nebari/template/stages/08-nebari-tf-extensions/variables.tf rename to src/_nebari/stages/nebari_tf_extensions/template/variables.tf diff --git a/src/_nebari/template/stages/08-nebari-tf-extensions/versions.tf b/src/_nebari/stages/nebari_tf_extensions/template/versions.tf similarity index 100% rename from src/_nebari/template/stages/08-nebari-tf-extensions/versions.tf rename to src/_nebari/stages/nebari_tf_extensions/template/versions.tf diff --git a/src/_nebari/stages/state_imports.py b/src/_nebari/stages/state_imports.py deleted file mode 100644 index 77d3ce367..000000000 --- a/src/_nebari/stages/state_imports.py +++ /dev/null @@ -1,50 +0,0 @@ -import os - - -def stage_01_terraform_state(stage_outputs, config): - if config["provider"] == "do": - return [ - ( - "module.terraform-state.module.spaces.digitalocean_spaces_bucket.main", - f"{config['digital_ocean']['region']},{config['project_name']}-{config['namespace']}-terraform-state", - ) - ] - elif config["provider"] == "gcp": - return [ - ( - "module.terraform-state.module.gcs.google_storage_bucket.static-site", - f"{config['project_name']}-{config['namespace']}-terraform-state", - ) - ] - elif config["provider"] == "azure": - subscription_id = os.environ["ARM_SUBSCRIPTION_ID"] - resource_name_prefix = f"{config['project_name']}-{config['namespace']}" - state_resource_group_name = f"{resource_name_prefix}-state" - state_resource_name_prefix_safe = resource_name_prefix.replace("-", "") - resource_group_url = f"/subscriptions/{subscription_id}/resourceGroups/{state_resource_group_name}" - - return [ - ( - "module.terraform-state.azurerm_resource_group.terraform-state-resource-group", - resource_group_url, - ), - ( - "module.terraform-state.azurerm_storage_account.terraform-state-storage-account", - f"{resource_group_url}/providers/Microsoft.Storage/storageAccounts/{state_resource_name_prefix_safe}{config['azure']['storage_account_postfix']}", - ), - ( - "module.terraform-state.azurerm_storage_container.storage_container", - f"https://{state_resource_name_prefix_safe}{config['azure']['storage_account_postfix']}.blob.core.windows.net/{resource_name_prefix}-state", - ), - ] - elif config["provider"] == "aws": - return [ - ( - "module.terraform-state.aws_s3_bucket.terraform-state", - f"{config['project_name']}-{config['namespace']}-terraform-state", - ), - ( - "module.terraform-state.aws_dynamodb_table.terraform-state-lock", - f"{config['project_name']}-{config['namespace']}-terraform-state-lock", - ), - ] diff --git a/src/_nebari/stages/terraform_state/__init__.py b/src/_nebari/stages/terraform_state/__init__.py new file mode 100644 index 000000000..c81c08344 --- /dev/null +++ b/src/_nebari/stages/terraform_state/__init__.py @@ -0,0 +1,122 @@ +import os +from typing import List, Tuple +import pathlib +import sys + +from nebari.hookspecs import hookimpl, NebariStage +from _nebari.stages.base import NebariTerraformStage +from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider + +from _nebari import schema + + +class KubernetesInitializeStage(NebariTerraformStage): + @property + def name(self): + return "01-terraform-state" + + @property + def priority(self): + return 10 + + def state_imports(self) -> List[Tuple[str, str]]: + if self.config.provider == schema.ProviderEnum.do: + return [ + ( + "module.terraform-state.module.spaces.digitalocean_spaces_bucket.main", + f"{self.config.digital_ocean.region},{self.config.project_name}-{self.config.namespace}-terraform-state", + ) + ] + elif self.config.provider == schema.ProviderEnum.gcp: + return [ + ( + "module.terraform-state.module.gcs.google_storage_bucket.static-site", + f"{self.config.project_name}-{self.config.namespace}-terraform-state", + ) + ] + elif self.config.provider == schema.ProviderEnum.azure: + subscription_id = os.environ["ARM_SUBSCRIPTION_ID"] + resource_name_prefix = f"{self.config.project_name}-{self.config.namespace}" + state_resource_group_name = f"{resource_name_prefix}-state" + state_resource_name_prefix_safe = resource_name_prefix.replace("-", "") + resource_group_url = f"/subscriptions/{subscription_id}/resourceGroups/{state_resource_group_name}" + + return [ + ( + "module.terraform-state.azurerm_resource_group.terraform-state-resource-group", + resource_group_url, + ), + ( + "module.terraform-state.azurerm_storage_account.terraform-state-storage-account", + f"{resource_group_url}/providers/Microsoft.Storage/storageAccounts/{state_resource_name_prefix_safe}{self.config.azure.storage_account_postfix}", + ), + ( + "module.terraform-state.azurerm_storage_container.storage_container", + f"https://{state_resource_name_prefix_safe}{self.config.azure.storage_account_postfix}.blob.core.windows.net/{resource_name_prefix}-state", + ), + ] + elif self.config.provider == schema.ProviderEnum.aws: + return [ + ( + "module.terraform-state.aws_s3_bucket.terraform-state", + f"{self.config.project_name}-{self.config.namespace}-terraform-state", + ), + ( + "module.terraform-state.aws_dynamodb_table.terraform-state-lock", + f"{self.config.project_name}-{self.config.namespace}-terraform-state-lock", + ), + ] + + def tf_objects(self) -> List[Dict]: + return [ + NebariTerraformState(self.name, self.config), + NebariKubernetesProvider(self.config), + NebariHelmProvider(self.config), + ] + + def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): + if self.config.provider == schema.ProviderEnum.do: + return { + "name": self.config.project_name, + "namespace": self.config.namespace, + "region": self.config.digital_ocean.region, + } + elif self.config.provider == schema.ProviderEnum.gcp: + return { + "name": self.config.project_name, + "namespace": self.config.namespace, + "region": self.config.google_cloud_platform.region, + } + elif self.config.provider == schema.ProviderEnum.aws: + return { + "name": self.config.project_name, + "namespace": self.config.namespace, + } + elif self.config.provider == schema.ProviderEnum.azure: + return { + "name": self.config.project_name, + "namespace": self.config.namespace, + "region": self.config.azure.region, + "storage_account_postfix": self.config.azure.storage_account_postfix, + "state_resource_group_name": f'{self.config.project_name}-{self.config.namespace}-state', + } + else: + return {} + + +@hookimpl +def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> List[NebariStage]: + if config.provider in [schema.ProviderEnum.local, schema.ProviderEnum.existing]: + return [] + + template_directory = pathlib.Path(__file__).parent / "template" / config.provider.value + stage_prefix = pathlib.Path("stages/01-terraform-state") / config.provider.value + + return [ + TerraformStateStage( + install_directory, + config, + template_directory=template_directory, + stage_prefix=stage_prefix, + ) + ] diff --git a/src/_nebari/template/stages/01-terraform-state/aws/main.tf b/src/_nebari/stages/terraform_state/template/aws/main.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/aws/main.tf rename to src/_nebari/stages/terraform_state/template/aws/main.tf diff --git a/src/_nebari/template/stages/01-terraform-state/aws/modules/terraform-state/main.tf b/src/_nebari/stages/terraform_state/template/aws/modules/terraform-state/main.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/aws/modules/terraform-state/main.tf rename to src/_nebari/stages/terraform_state/template/aws/modules/terraform-state/main.tf diff --git a/src/_nebari/template/stages/01-terraform-state/aws/modules/terraform-state/output.tf b/src/_nebari/stages/terraform_state/template/aws/modules/terraform-state/output.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/aws/modules/terraform-state/output.tf rename to src/_nebari/stages/terraform_state/template/aws/modules/terraform-state/output.tf diff --git a/src/_nebari/template/stages/01-terraform-state/aws/modules/terraform-state/variables.tf b/src/_nebari/stages/terraform_state/template/aws/modules/terraform-state/variables.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/aws/modules/terraform-state/variables.tf rename to src/_nebari/stages/terraform_state/template/aws/modules/terraform-state/variables.tf diff --git a/src/_nebari/template/stages/01-terraform-state/azure/main.tf b/src/_nebari/stages/terraform_state/template/azure/main.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/azure/main.tf rename to src/_nebari/stages/terraform_state/template/azure/main.tf diff --git a/src/_nebari/template/stages/01-terraform-state/azure/modules/terraform-state/main.tf b/src/_nebari/stages/terraform_state/template/azure/modules/terraform-state/main.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/azure/modules/terraform-state/main.tf rename to src/_nebari/stages/terraform_state/template/azure/modules/terraform-state/main.tf diff --git a/src/_nebari/template/stages/01-terraform-state/azure/modules/terraform-state/variables.tf b/src/_nebari/stages/terraform_state/template/azure/modules/terraform-state/variables.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/azure/modules/terraform-state/variables.tf rename to src/_nebari/stages/terraform_state/template/azure/modules/terraform-state/variables.tf diff --git a/src/_nebari/template/stages/01-terraform-state/do/main.tf b/src/_nebari/stages/terraform_state/template/do/main.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/do/main.tf rename to src/_nebari/stages/terraform_state/template/do/main.tf diff --git a/src/_nebari/template/stages/01-terraform-state/do/modules/spaces/main.tf b/src/_nebari/stages/terraform_state/template/do/modules/spaces/main.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/do/modules/spaces/main.tf rename to src/_nebari/stages/terraform_state/template/do/modules/spaces/main.tf diff --git a/src/_nebari/template/stages/01-terraform-state/do/modules/spaces/variables.tf b/src/_nebari/stages/terraform_state/template/do/modules/spaces/variables.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/do/modules/spaces/variables.tf rename to src/_nebari/stages/terraform_state/template/do/modules/spaces/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/do/modules/registry/versions.tf b/src/_nebari/stages/terraform_state/template/do/modules/spaces/versions.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/do/modules/registry/versions.tf rename to src/_nebari/stages/terraform_state/template/do/modules/spaces/versions.tf diff --git a/src/_nebari/template/stages/01-terraform-state/do/modules/terraform-state/main.tf b/src/_nebari/stages/terraform_state/template/do/modules/terraform-state/main.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/do/modules/terraform-state/main.tf rename to src/_nebari/stages/terraform_state/template/do/modules/terraform-state/main.tf diff --git a/src/_nebari/template/stages/01-terraform-state/do/modules/terraform-state/variables.tf b/src/_nebari/stages/terraform_state/template/do/modules/terraform-state/variables.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/do/modules/terraform-state/variables.tf rename to src/_nebari/stages/terraform_state/template/do/modules/terraform-state/variables.tf diff --git a/src/_nebari/template/stages/02-infrastructure/do/versions.tf b/src/_nebari/stages/terraform_state/template/do/modules/terraform-state/versions.tf similarity index 100% rename from src/_nebari/template/stages/02-infrastructure/do/versions.tf rename to src/_nebari/stages/terraform_state/template/do/modules/terraform-state/versions.tf diff --git a/src/_nebari/template/stages/01-terraform-state/gcp/main.tf b/src/_nebari/stages/terraform_state/template/gcp/main.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/gcp/main.tf rename to src/_nebari/stages/terraform_state/template/gcp/main.tf diff --git a/src/_nebari/template/stages/01-terraform-state/gcp/modules/gcs/main.tf b/src/_nebari/stages/terraform_state/template/gcp/modules/gcs/main.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/gcp/modules/gcs/main.tf rename to src/_nebari/stages/terraform_state/template/gcp/modules/gcs/main.tf diff --git a/src/_nebari/template/stages/01-terraform-state/gcp/modules/gcs/variables.tf b/src/_nebari/stages/terraform_state/template/gcp/modules/gcs/variables.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/gcp/modules/gcs/variables.tf rename to src/_nebari/stages/terraform_state/template/gcp/modules/gcs/variables.tf diff --git a/src/_nebari/template/stages/01-terraform-state/gcp/modules/terraform-state/main.tf b/src/_nebari/stages/terraform_state/template/gcp/modules/terraform-state/main.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/gcp/modules/terraform-state/main.tf rename to src/_nebari/stages/terraform_state/template/gcp/modules/terraform-state/main.tf diff --git a/src/_nebari/template/stages/01-terraform-state/gcp/modules/terraform-state/variables.tf b/src/_nebari/stages/terraform_state/template/gcp/modules/terraform-state/variables.tf similarity index 100% rename from src/_nebari/template/stages/01-terraform-state/gcp/modules/terraform-state/variables.tf rename to src/_nebari/stages/terraform_state/template/gcp/modules/terraform-state/variables.tf diff --git a/src/_nebari/stages/tf_objects.py b/src/_nebari/stages/tf_objects.py index 3a961f678..8b1aedf4b 100644 --- a/src/_nebari/stages/tf_objects.py +++ b/src/_nebari/stages/tf_objects.py @@ -137,163 +137,3 @@ def NebariTerraformState(directory: str, nebari_config: Dict): ) else: raise NotImplementedError("state not implemented") - - -def stage_01_terraform_state(config): - if config["provider"] == "gcp": - return { - Path("stages") - / "01-terraform-state" - / "gcp" - / "_nebari.tf.json": tf_render_objects( - [ - NebariGCPProvider(config), - ] - ) - } - elif config["provider"] == "aws": - return { - Path("stages") - / "01-terraform-state" - / "aws" - / "_nebari.tf.json": tf_render_objects( - [ - NebariAWSProvider(config), - ] - ) - } - else: - return {} - - -def stage_02_infrastructure(config): - if config["provider"] == "gcp": - return { - Path("stages") - / "02-infrastructure" - / "gcp" - / "_nebari.tf.json": tf_render_objects( - [ - NebariGCPProvider(config), - NebariTerraformState("02-infrastructure", config), - ] - ) - } - elif config["provider"] == "do": - return { - Path("stages") - / "02-infrastructure" - / "do" - / "_nebari.tf.json": tf_render_objects( - [ - NebariTerraformState("02-infrastructure", config), - ] - ) - } - elif config["provider"] == "azure": - return { - Path("stages") - / "02-infrastructure" - / "azure" - / "_nebari.tf.json": tf_render_objects( - [ - NebariTerraformState("02-infrastructure", config), - ] - ), - } - elif config["provider"] == "aws": - return { - Path("stages") - / "02-infrastructure" - / "aws" - / "_nebari.tf.json": tf_render_objects( - [ - NebariAWSProvider(config), - NebariTerraformState("02-infrastructure", config), - ] - ) - } - else: - return {} - - -def stage_03_kubernetes_initialize(config): - return { - Path("stages") - / "03-kubernetes-initialize" - / "_nebari.tf.json": tf_render_objects( - [ - NebariTerraformState("03-kubernetes-initialize", config), - NebariKubernetesProvider(config), - NebariHelmProvider(config), - ] - ), - } - - -def stage_04_kubernetes_ingress(config): - return { - Path("stages") - / "04-kubernetes-ingress" - / "_nebari.tf.json": tf_render_objects( - [ - NebariTerraformState("04-kubernetes-ingress", config), - NebariKubernetesProvider(config), - NebariHelmProvider(config), - ] - ), - } - - -def stage_05_kubernetes_keycloak(config): - return { - Path("stages") - / "05-kubernetes-keycloak" - / "_nebari.tf.json": tf_render_objects( - [ - NebariTerraformState("05-kubernetes-keycloak", config), - NebariKubernetesProvider(config), - NebariHelmProvider(config), - ] - ), - } - - -def stage_06_kubernetes_keycloak_configuration(config): - return { - Path("stages") - / "06-kubernetes-keycloak-configuration" - / "_nebari.tf.json": tf_render_objects( - [ - NebariTerraformState("06-kubernetes-keycloak-configuration", config), - ] - ), - } - - -def stage_07_kubernetes_services(config): - return { - Path("stages") - / "07-kubernetes-services" - / "_nebari.tf.json": tf_render_objects( - [ - NebariTerraformState("07-kubernetes-services", config), - NebariKubernetesProvider(config), - NebariHelmProvider(config), - ] - ), - } - - -def stage_08_nebari_tf_extensions(config): - return { - Path("stages") - / "08-nebari-tf-extensions" - / "_nebari.tf.json": tf_render_objects( - [ - NebariTerraformState("08-nebari-tf-extensions", config), - NebariKubernetesProvider(config), - NebariHelmProvider(config), - ] - ), - } diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/ipython/__init__.py b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/ipython/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/__init__.py b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/_nebari/template/stages/__init__.py b/src/_nebari/template/stages/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 4ca07f82b..46a84e4b7 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -297,44 +297,6 @@ def modified_environ(*remove: List[str], **update: Dict[str, str]): [env.pop(k) for k in remove_after] -@contextlib.contextmanager -def kubernetes_provider_context(kubernetes_credentials: Dict[str, str]): - credential_mapping = { - "config_path": "KUBE_CONFIG_PATH", - "config_context": "KUBE_CTX", - "username": "KUBE_USER", - "password": "KUBE_PASSWORD", - "client_certificate": "KUBE_CLIENT_CERT_DATA", - "client_key": "KUBE_CLIENT_KEY_DATA", - "cluster_ca_certificate": "KUBE_CLUSTER_CA_CERT_DATA", - "host": "KUBE_HOST", - "token": "KUBE_TOKEN", - } - - credentials = { - credential_mapping[k]: v - for k, v in kubernetes_credentials.items() - if v is not None - } - with modified_environ(**credentials): - yield - - -@contextlib.contextmanager -def keycloak_provider_context(keycloak_credentials: Dict[str, str]): - credential_mapping = { - "client_id": "KEYCLOAK_CLIENT_ID", - "url": "KEYCLOAK_URL", - "username": "KEYCLOAK_USER", - "password": "KEYCLOAK_PASSWORD", - "realm": "KEYCLOAK_REALM", - } - - credentials = {credential_mapping[k]: v for k, v in keycloak_credentials.items()} - with modified_environ(**credentials): - yield - - def deep_merge(*args): """Deep merge multiple dictionaries. diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py new file mode 100644 index 000000000..339717056 --- /dev/null +++ b/src/nebari/hookspecs.py @@ -0,0 +1,50 @@ +from typing import Optional +from collections.abc import Iterable +import contextlib +import importlib +import pathlib +import logging + +from pluggy import HookimplMarker +from pluggy import HookspecMarker + +hookspec = HookspecMarker("nebari") +hookimpl = HookimplMarker("nebari") + +from _nebari import schema + + +class NebariStage: + def __init__(self, output_directory: pathlib.Path, config: schema.Main): + self.output_directory = output_directory + self.config = config + + @property + def name(self) -> str: + raise NotImplementedError() + + def validate(self): + pass + + @propery + def priority(self) -> int: + raise NotImplementedError() + + def render(self, output_directory: pathlib.Path): + raise NotImplementedError() + + @contextlib.contextmanager + def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): + raise NotImplementedError() + + def check(self, stage_outputs: Dict[str, Dict[str, Any]]) -> bool: + pass + + @contextlib.contextmanager + def destroy(self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool]): + raise NotImplementedError() + + +@hookspec +def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> Iterable[NebariStage]: + """Registers stages in nebari""" diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py new file mode 100644 index 000000000..a9022f868 --- /dev/null +++ b/src/nebari/plugins.py @@ -0,0 +1,21 @@ +import sys + +import pluggy + +from nebari import hookspecs + +DEFAULT_PLUGINS = [ + "_nebari.stage.kubernetes_initialize_20", +] + +pm = pluggy.PluginManager("nebari") +pm.add_hookspecs(hookspecs) + +if not hasattr(sys, "_called_from_test"): + # Only load plugins if not running tests + pm.load_setuptools_entrypoints("datasette") + +# Load default plugins +for plugin in DEFAULT_PLUGINS: + mod = importlib.import_module(plugin) + pm.register(mod, plugin) From 4dfe5f20c2fb0db5bc2b9b502ace1a3e386b4eee Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 13 Jun 2023 21:56:01 +0000 Subject: [PATCH 002/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/stages/base.py | 4 ++-- src/_nebari/stages/infrastructure/__init__.py | 17 +++++++------ .../stages/kubernetes_ingress/__init__.py | 16 +++++++------ .../stages/kubernetes_initialize/__init__.py | 13 ++++++---- .../stages/kubernetes_keycloak/__init__.py | 17 +++++++------ .../__init__.py | 7 +++--- .../stages/kubernetes_services/__init__.py | 13 ++++++---- .../stages/nebari_tf_extensions/__init__.py | 24 +++++++++++-------- .../stages/terraform_state/__init__.py | 24 ++++++++++++------- src/_nebari/stages/tf_objects.py | 7 +----- src/nebari/hookspecs.py | 16 ++++++------- 11 files changed, 88 insertions(+), 70 deletions(-) diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index 1f3058e7e..3df44cb3b 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -1,10 +1,10 @@ -from typing import List, Dict, Tuple import pathlib +from typing import Dict, List, Tuple -from nebari.hookspecs import NebariStage from _nebari import schema from _nebari.provider import terraform from _nebari.stages.tf_objects import NebariTerraformState +from nebari.hookspecs import NebariStage class NebariTerraformStage(NebariStage): diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 1c30c7695..1ab936214 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -1,14 +1,17 @@ -from typing import List, Dict +import contextlib import pathlib import sys -import contextlib - -from nebari.hookspecs import hookimpl, NebariStage -from _nebari.utils import modified_environ -from _nebari.stages.base import NebariTerraformStage -from _nebari.stages.tf_objects import NebariTerraformState, NebariGCPProvider, NebariAWSProvider +from typing import Dict, List from _nebari import schema +from _nebari.stages.base import NebariTerraformStage +from _nebari.stages.tf_objects import ( + NebariAWSProvider, + NebariGCPProvider, + NebariTerraformState, +) +from _nebari.utils import modified_environ +from nebari.hookspecs import NebariStage, hookimpl @contextlib.contextmanager diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index d61a5d587..6e44ebf04 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -1,14 +1,16 @@ -from typing import List, Dict import pathlib -import sys import socket +import sys +from typing import Dict, List -from nebari.hookspecs import NebariStage, hookimpl +from _nebari import constants, schema from _nebari.stages.base import NebariTerraformStage -from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider - -from _nebari import schema, constants - +from _nebari.stages.tf_objects import ( + NebariHelmProvider, + NebariKubernetesProvider, + NebariTerraformState, +) +from nebari.hookspecs import NebariStage, hookimpl # check and retry settings NUM_ATTEMPTS = 10 diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py index feed5cfaf..219b0789b 100644 --- a/src/_nebari/stages/kubernetes_initialize/__init__.py +++ b/src/_nebari/stages/kubernetes_initialize/__init__.py @@ -1,12 +1,15 @@ -from typing import List, Dict import pathlib import sys - -from nebari.hookspecs import NebariStage, hookimpl -from _nebari.stages.base import NebariTerraformStage -from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider +from typing import Dict, List from _nebari import schema +from _nebari.stages.base import NebariTerraformStage +from _nebari.stages.tf_objects import ( + NebariHelmProvider, + NebariKubernetesProvider, + NebariTerraformState, +) +from nebari.hookspecs import NebariStage, hookimpl class KubernetesInitializeStage(NebariTerraformStage): diff --git a/src/_nebari/stages/kubernetes_keycloak/__init__.py b/src/_nebari/stages/kubernetes_keycloak/__init__.py index e24ba97d3..a3f72f3e9 100644 --- a/src/_nebari/stages/kubernetes_keycloak/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak/__init__.py @@ -1,14 +1,17 @@ -from typing import List, Dict +import json import pathlib import sys -import json - -from nebari.hookspecs import NebariStage, hookimpl -from _nebari.utils import modified_environ -from _nebari.stages.base import NebariTerraformStage -from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider +from typing import Dict, List from _nebari import schema +from _nebari.stages.base import NebariTerraformStage +from _nebari.stages.tf_objects import ( + NebariHelmProvider, + NebariKubernetesProvider, + NebariTerraformState, +) +from _nebari.utils import modified_environ +from nebari.hookspecs import NebariStage, hookimpl @contextlib.contextmanager diff --git a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py index c309986eb..96e582cb3 100644 --- a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py @@ -1,12 +1,11 @@ -from typing import List, Dict import pathlib import sys +from typing import Dict, List -from nebari.hookspecs import NebariStage, hookimpl +from _nebari import schema from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import NebariTerraformState - -from _nebari import schema +from nebari.hookspecs import NebariStage, hookimpl class KubernetesKeycloakConfigurationStage(NebariTerraformStage): diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index 86c8c08bc..258b73075 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -1,13 +1,16 @@ -from typing import List, Dict import pathlib import sys +from typing import Dict, List from urllib.parse import urlencode -from nebari.hookspecs import NebariStage, hookimpl -from _nebari.stages.base import NebariTerraformStage -from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider - from _nebari import schema +from _nebari.stages.base import NebariTerraformStage +from _nebari.stages.tf_objects import ( + NebariHelmProvider, + NebariKubernetesProvider, + NebariTerraformState, +) +from nebari.hookspecs import NebariStage, hookimpl # check and retry settings NUM_ATTEMPTS = 10 diff --git a/src/_nebari/stages/nebari_tf_extensions/__init__.py b/src/_nebari/stages/nebari_tf_extensions/__init__.py index 5a80a17c2..7c3bef394 100644 --- a/src/_nebari/stages/nebari_tf_extensions/__init__.py +++ b/src/_nebari/stages/nebari_tf_extensions/__init__.py @@ -1,12 +1,14 @@ -from typing import List, Dict import pathlib -import sys - -from nebari.hookspecs import NebariStage, hookimpl -from _nebari.stages.base import NebariTerraformStage -from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider +from typing import Dict, List from _nebari import schema +from _nebari.stages.base import NebariTerraformStage +from _nebari.stages.tf_objects import ( + NebariHelmProvider, + NebariKubernetesProvider, + NebariTerraformState, +) +from nebari.hookspecs import NebariStage, hookimpl class NebariTFExtensionsStage(NebariTerraformStage): @@ -34,15 +36,17 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): ]["value"], "tf_extensions": self.config.tf_extensions, "nebari_config_yaml": self.config, - "keycloak_nebari_bot_password": stage_outputs["stages/05-kubernetes-keycloak"][ - "keycloak_nebari_bot_password" - ]["value"], + "keycloak_nebari_bot_password": stage_outputs[ + "stages/05-kubernetes-keycloak" + ]["keycloak_nebari_bot_password"]["value"], "helm_extensions": self.config.helm_extensions, } @hookimpl -def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> List[NebariStage]: +def nebari_stage( + install_directory: pathlib.Path, config: schema.Main +) -> List[NebariStage]: return [ NebariTFExtensionsStage( install_directory, diff --git a/src/_nebari/stages/terraform_state/__init__.py b/src/_nebari/stages/terraform_state/__init__.py index c81c08344..6c05b6494 100644 --- a/src/_nebari/stages/terraform_state/__init__.py +++ b/src/_nebari/stages/terraform_state/__init__.py @@ -1,13 +1,15 @@ import os -from typing import List, Tuple import pathlib -import sys - -from nebari.hookspecs import hookimpl, NebariStage -from _nebari.stages.base import NebariTerraformStage -from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider +from typing import List, Tuple from _nebari import schema +from _nebari.stages.base import NebariTerraformStage +from _nebari.stages.tf_objects import ( + NebariHelmProvider, + NebariKubernetesProvider, + NebariTerraformState, +) +from nebari.hookspecs import NebariStage, hookimpl class KubernetesInitializeStage(NebariTerraformStage): @@ -98,18 +100,22 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "namespace": self.config.namespace, "region": self.config.azure.region, "storage_account_postfix": self.config.azure.storage_account_postfix, - "state_resource_group_name": f'{self.config.project_name}-{self.config.namespace}-state', + "state_resource_group_name": f"{self.config.project_name}-{self.config.namespace}-state", } else: return {} @hookimpl -def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> List[NebariStage]: +def nebari_stage( + install_directory: pathlib.Path, config: schema.Main +) -> List[NebariStage]: if config.provider in [schema.ProviderEnum.local, schema.ProviderEnum.existing]: return [] - template_directory = pathlib.Path(__file__).parent / "template" / config.provider.value + template_directory = ( + pathlib.Path(__file__).parent / "template" / config.provider.value + ) stage_prefix = pathlib.Path("stages/01-terraform-state") / config.provider.value return [ diff --git a/src/_nebari/stages/tf_objects.py b/src/_nebari/stages/tf_objects.py index 8b1aedf4b..2ecb8a56c 100644 --- a/src/_nebari/stages/tf_objects.py +++ b/src/_nebari/stages/tf_objects.py @@ -1,12 +1,7 @@ from pathlib import Path from typing import Dict -from _nebari.provider.terraform import ( - Data, - Provider, - TerraformBackend, - tf_render_objects, -) +from _nebari.provider.terraform import Data, Provider, TerraformBackend from _nebari.utils import deep_merge diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index 339717056..38c8444ff 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -1,12 +1,8 @@ -from typing import Optional -from collections.abc import Iterable import contextlib -import importlib import pathlib -import logging +from collections.abc import Iterable -from pluggy import HookimplMarker -from pluggy import HookspecMarker +from pluggy import HookimplMarker, HookspecMarker hookspec = HookspecMarker("nebari") hookimpl = HookimplMarker("nebari") @@ -41,10 +37,14 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]) -> bool: pass @contextlib.contextmanager - def destroy(self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool]): + def destroy( + self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool] + ): raise NotImplementedError() @hookspec -def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> Iterable[NebariStage]: +def nebari_stage( + install_directory: pathlib.Path, config: schema.Main +) -> Iterable[NebariStage]: """Registers stages in nebari""" From e35af55850c04d6d804b19af69c6ea10e8898755 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 14 Jun 2023 12:48:47 -0400 Subject: [PATCH 003/147] Work to make runnable along with working on schema --- src/_nebari/keycloak.py | 4 +- src/_nebari/stages/base.py | 11 +++-- src/_nebari/stages/infrastructure/__init__.py | 15 +++--- .../stages/kubernetes_ingress/__init__.py | 17 ++++--- .../stages/kubernetes_initialize/__init__.py | 14 ++---- .../stages/kubernetes_keycloak/__init__.py | 29 +++++------- .../__init__.py | 18 +++---- .../stages/kubernetes_services/__init__.py | 26 ++++------ .../stages/nebari_tf_extensions/__init__.py | 14 ++---- .../stages/terraform_state/__init__.py | 14 ++---- src/_nebari/upgrade.py | 2 +- src/nebari/hookspecs.py | 14 ++---- src/nebari/plugins.py | 10 +++- src/{_nebari => nebari}/schema.py | 47 +++++++++---------- 14 files changed, 102 insertions(+), 133 deletions(-) rename src/{_nebari => nebari}/schema.py (94%) diff --git a/src/_nebari/keycloak.py b/src/_nebari/keycloak.py index 4579298c7..ae612be86 100644 --- a/src/_nebari/keycloak.py +++ b/src/_nebari/keycloak.py @@ -7,8 +7,8 @@ import requests import rich -from .schema import verify -from .utils import load_yaml +from nebari.schema import verify +from _nebari.utils import load_yaml logger = logging.getLogger(__name__) diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index 3df44cb3b..e8a514409 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -1,5 +1,6 @@ +from typing import List, Dict, Tuple, Any import pathlib -from typing import Dict, List, Tuple +import contextlib from _nebari import schema from _nebari.provider import terraform @@ -15,7 +16,7 @@ def __init__( template_directory: pathlib.Path, stage_prefix: pathlib.Path, ): - super().__init__(self, output_directory, config) + super().__init__(output_directory, config) self.template_directory = pathlib.Path(template_directory) self.stage_prefix = pathlib.Path(stage_prefix) @@ -43,7 +44,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): @contextlib.contextmanager def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): deploy_config = dict( - directory=str(output_directory / stage_prefix), + directory=str(self.output_directory / self.stage_prefix), input_vars=self.input_vars(stage_outputs) ) state_imports = self.state_imports() @@ -59,8 +60,8 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]): @contextlib.contextmanager def destroy(self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool]): - stage_outputs["stages/" + self.name] = terraform.deploy(( - directory=str(output_directory / stage_prefix), + stage_outputs["stages/" + self.name] = terraform.deploy( + directory=str(self.output_directory / self.stage_prefix), input_vars=self.input_vars(stage_outputs), terraform_init=True, terraform_import=True, diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 1ab936214..1c465a68e 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -1,7 +1,7 @@ import contextlib +from typing import List, Dict, Any import pathlib import sys -from typing import Dict, List from _nebari import schema from _nebari.stages.base import NebariTerraformStage @@ -13,6 +13,8 @@ from _nebari.utils import modified_environ from nebari.hookspecs import NebariStage, hookimpl +from nebari import schema + @contextlib.contextmanager def kubernetes_provider_context(kubernetes_credentials: Dict[str, str]): @@ -38,13 +40,8 @@ def kubernetes_provider_context(kubernetes_credentials: Dict[str, str]): class KubernetesInfrastructureStage(NebariTerraformStage): - @property - def name(self): - return "02-infrastructure" - - @property - def priority(self): - return 20 + name = "02-infrastructure" + priority = 20 def tf_objects(self) -> List[Dict]: if self.config.provider == schema.ProviderEnum.gcp: @@ -156,7 +153,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): else: return {} - def check(self, stage_outputs: stage_outputs: Dict[str, Dict[str, Any]]): + def check(self, stage_outputs: Dict[str, Dict[str, Any]]): from kubernetes import client, config from kubernetes.client.rest import ApiException diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index 6e44ebf04..f6def3c07 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -1,7 +1,7 @@ +from typing import List, Dict, Any import pathlib import socket import sys -from typing import Dict, List from _nebari import constants, schema from _nebari.stages.base import NebariTerraformStage @@ -11,6 +11,10 @@ NebariTerraformState, ) from nebari.hookspecs import NebariStage, hookimpl +from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider + +from nebari import schema +from _nebari import constants # check and retry settings NUM_ATTEMPTS = 10 @@ -94,13 +98,8 @@ def _attempt_dns_lookup( class KubernetesIngressStage(NebariTerraformStage): - @property - def name(self): - return "04-kubernetes-ingress" - - @property - def priority(self): - return 40 + name = "04-kubernetes-ingress" + priority = 40 def tf_objects(self) -> List[Dict]: return [ @@ -132,7 +131,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): **cert_details, } - def check(self, stage_outputs: stage_outputs: Dict[str, Dict[str, Any]]): + def check(self, stage_outputs: Dict[str, Dict[str, Any]]): def _attempt_tcp_connect(host, port, num_attempts=NUM_ATTEMPTS, timeout=TIMEOUT): for i in range(num_attempts): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py index 219b0789b..8319e1297 100644 --- a/src/_nebari/stages/kubernetes_initialize/__init__.py +++ b/src/_nebari/stages/kubernetes_initialize/__init__.py @@ -1,6 +1,6 @@ +from typing import List, Dict, Any import pathlib import sys -from typing import Dict, List from _nebari import schema from _nebari.stages.base import NebariTerraformStage @@ -10,16 +10,12 @@ NebariTerraformState, ) from nebari.hookspecs import NebariStage, hookimpl +from nebari import schema class KubernetesInitializeStage(NebariTerraformStage): - @property - def name(self): - return "03-kubernetes-initialize" - - @property - def priority(self): - return 30 + name = "03-kubernetes-initialize" + priority = 30 def tf_objects(self) -> List[Dict]: return [ @@ -58,7 +54,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "gpu_node_group_names": gpu_node_group_names, } - def check(self, stage_outputs: stage_outputs: Dict[str, Dict[str, Any]]): + def check(self, stage_outputs: Dict[str, Dict[str, Any]]): from kubernetes import client, config from kubernetes.client.rest import ApiException diff --git a/src/_nebari/stages/kubernetes_keycloak/__init__.py b/src/_nebari/stages/kubernetes_keycloak/__init__.py index a3f72f3e9..5c6c4357f 100644 --- a/src/_nebari/stages/kubernetes_keycloak/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak/__init__.py @@ -1,17 +1,15 @@ -import json +from typing import List, Dict, Any import pathlib import sys -from typing import Dict, List +import json +import contextlib -from _nebari import schema -from _nebari.stages.base import NebariTerraformStage -from _nebari.stages.tf_objects import ( - NebariHelmProvider, - NebariKubernetesProvider, - NebariTerraformState, -) -from _nebari.utils import modified_environ from nebari.hookspecs import NebariStage, hookimpl +from _nebari.utils import modified_environ +from _nebari.stages.base import NebariTerraformStage +from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider + +from nebari import schema @contextlib.contextmanager @@ -57,13 +55,8 @@ def _calculate_node_groups(config: schema.Main): class KubernetesKeycloakStage(NebariTerraformStage): - @property - def name(self): - return "05-kubernetes-keycloak" - - @property - def priority(self): - return 50 + name = "05-kubernetes-keycloak" + priority = 50 def tf_objects(self) -> List[Dict]: return [ @@ -85,7 +78,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): } - def check(self, stage_outputs: stage_outputs: Dict[str, Dict[str, Any]]): + def check(self, stage_outputs: Dict[str, Dict[str, Any]]): from keycloak import KeycloakAdmin from keycloak.exceptions import KeycloakError diff --git a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py index 96e582cb3..3704f1a9b 100644 --- a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py @@ -1,21 +1,17 @@ +from typing import List, Dict, Any import pathlib import sys -from typing import Dict, List -from _nebari import schema from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import NebariTerraformState from nebari.hookspecs import NebariStage, hookimpl +from nebari import schema -class KubernetesKeycloakConfigurationStage(NebariTerraformStage): - @property - def name(self): - return "06-kubernetes-keycloak-configuration" - @property - def priority(self): - return 60 +class KubernetesKeycloakConfigurationStage(NebariTerraformStage): + name = "06-kubernetes-keycloak-configuration" + priority = 60 def tf_objects(self) -> List[Dict]: return [ @@ -38,7 +34,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "default_groups": ["analyst"] + users_group, } - def check(self, stage_outputs: stage_outputs: Dict[str, Dict[str, Any]]): + def check(self, stage_outputs: Dict[str, Dict[str, Any]]): directory = "stages/05-kubernetes-keycloak" from keycloak import KeycloakAdmin @@ -110,6 +106,6 @@ def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> List[N install_directory, config, template_directory=(pathlib.Path(__file__).parent / "template"), - stage_directory=pathlib.Path("stages/06-kubernetes-keycloak-configuration"), + stage_prefix=pathlib.Path("stages/06-kubernetes-keycloak-configuration"), ) ] diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index 258b73075..20edef929 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -1,16 +1,13 @@ +from typing import List, Dict, Any import pathlib import sys -from typing import Dict, List from urllib.parse import urlencode -from _nebari import schema -from _nebari.stages.base import NebariTerraformStage -from _nebari.stages.tf_objects import ( - NebariHelmProvider, - NebariKubernetesProvider, - NebariTerraformState, -) from nebari.hookspecs import NebariStage, hookimpl +from _nebari.stages.base import NebariTerraformStage +from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider + +from nebari import schema # check and retry settings NUM_ATTEMPTS = 10 @@ -50,13 +47,8 @@ def _calculate_node_groups(config: schema.Main): class KubernetesServicesStage(NebariTerraformStage): - @property - def name(self): - return "07-kubernetes-services" - - @property - def priority(self): - return 70 + name = "07-kubernetes-services" + priority = 70 def tf_objects(self) -> List[Dict]: return [ @@ -159,7 +151,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "prefect-image": self.config.prefect.image, "prefect-overrides": self.config.prefect.overrides, # clearml - "clearml-enabled": self.config.clearml.enabled + "clearml-enabled": self.config.clearml.enabled, "clearml-enable-forwardauth": self.config.clearml.enable_forward_auth, "clearml-overrides": [ json.dumps(self.config.clearml.overrides) @@ -168,7 +160,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): } - def check(self, stage_outputs: stage_outputs: Dict[str, Dict[str, Any]]): + def check(self, stage_outputs: Dict[str, Dict[str, Any]]): directory = "stages/07-kubernetes-services" import requests diff --git a/src/_nebari/stages/nebari_tf_extensions/__init__.py b/src/_nebari/stages/nebari_tf_extensions/__init__.py index 7c3bef394..8694ca73d 100644 --- a/src/_nebari/stages/nebari_tf_extensions/__init__.py +++ b/src/_nebari/stages/nebari_tf_extensions/__init__.py @@ -1,7 +1,6 @@ +from typing import List, Dict, Any import pathlib -from typing import Dict, List -from _nebari import schema from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( NebariHelmProvider, @@ -10,15 +9,12 @@ ) from nebari.hookspecs import NebariStage, hookimpl +from nebari import schema -class NebariTFExtensionsStage(NebariTerraformStage): - @property - def name(self): - return "08-nebari-tf-extensions" - @property - def priority(self): - return 80 +class NebariTFExtensionsStage(NebariTerraformStage): + name = "08-nebari-tf-extensions" + priority = 80 def tf_objects(self) -> List[Dict]: return [ diff --git a/src/_nebari/stages/terraform_state/__init__.py b/src/_nebari/stages/terraform_state/__init__.py index 6c05b6494..6e8143097 100644 --- a/src/_nebari/stages/terraform_state/__init__.py +++ b/src/_nebari/stages/terraform_state/__init__.py @@ -1,8 +1,7 @@ import os +from typing import List, Tuple, Dict, Any import pathlib -from typing import List, Tuple -from _nebari import schema from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( NebariHelmProvider, @@ -11,15 +10,12 @@ ) from nebari.hookspecs import NebariStage, hookimpl +from nebari import schema -class KubernetesInitializeStage(NebariTerraformStage): - @property - def name(self): - return "01-terraform-state" - @property - def priority(self): - return 10 +class KubernetesInitializeStage(NebariTerraformStage): + name = "01-terraform-state" + priority = 10 def state_imports(self) -> List[Tuple[str, str]]: if self.config.provider == schema.ProviderEnum.do: diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 95eb02bb5..551904179 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -10,7 +10,7 @@ from pydantic.error_wrappers import ValidationError from rich.prompt import Prompt -from .schema import is_version_accepted, verify +from nebari.schema import is_version_accepted, verify from .utils import backup_config_file, load_yaml, yaml from .version import __version__, rounded_ver_parse diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index 38c8444ff..b1035b1df 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -1,6 +1,7 @@ +from typing import Optional, Dict, Any +from collections.abc import Iterable import contextlib import pathlib -from collections.abc import Iterable from pluggy import HookimplMarker, HookspecMarker @@ -11,21 +12,16 @@ class NebariStage: + name = None + priority = None + def __init__(self, output_directory: pathlib.Path, config: schema.Main): self.output_directory = output_directory self.config = config - @property - def name(self) -> str: - raise NotImplementedError() - def validate(self): pass - @propery - def priority(self) -> int: - raise NotImplementedError() - def render(self, output_directory: pathlib.Path): raise NotImplementedError() diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index a9022f868..a705d3c46 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -1,11 +1,19 @@ import sys +import importlib import pluggy from nebari import hookspecs DEFAULT_PLUGINS = [ - "_nebari.stage.kubernetes_initialize_20", + "_nebari.stages.terraform_state", + "_nebari.stages.infrastructure", + "_nebari.stages.kubernetes_initialize", + "_nebari.stages.kubernetes_ingress", + "_nebari.stages.kubernetes_keycloak", + "_nebari.stages.kubernetes_keycloak_configuration", + "_nebari.stages.kubernetes_services", + "_nebari.stages.nebari_tf_extensions", ] pm = pluggy.PluginManager("nebari") diff --git a/src/_nebari/schema.py b/src/nebari/schema.py similarity index 94% rename from src/_nebari/schema.py rename to src/nebari/schema.py index 83a627e33..b594a9824 100644 --- a/src/_nebari/schema.py +++ b/src/nebari/schema.py @@ -6,8 +6,7 @@ from pydantic import root_validator, validator from _nebari.utils import namestr_regex - -from .version import __version__, rounded_ver_parse +from _nebari.version import __version__, rounded_ver_parse class CertificateEnum(str, enum.Enum): @@ -67,11 +66,11 @@ class Config: class CICD(Base): - type: CiEnum - branch: str - commit_render: typing.Optional[bool] = True - before_script: typing.Optional[typing.List[typing.Union[str, typing.Dict]]] - after_script: typing.Optional[typing.List[typing.Union[str, typing.Dict]]] + type: CiEnum = CiEnum.none + branch: str = "main" + commit_render: bool = True + before_script: typing.List[typing.Union[str, typing.Dict]] = [] + after_script: typing.List[typing.Union[str, typing.Dict]] = [] # ======== Generic Helm Extensions ======== @@ -115,18 +114,18 @@ class Monitoring(Base): class ClearML(Base): - enabled: bool + enabled: bool = False enable_forward_auth: typing.Optional[bool] - overrides: typing.Optional[typing.Dict] + overrides: typing.Dict = {} # ============== Prefect ============= class Prefect(Base): - enabled: bool + enabled: bool = False image: typing.Optional[str] - overrides: typing.Optional[typing.Dict] + overrides: typing.Dict = {} # =========== Conda-Store ============== @@ -152,7 +151,7 @@ class TerraformState(Base): class Certificate(Base): - type: CertificateEnum + type: CertificateEnum = CertificateEnum.selfsigned # existing secret_name: typing.Optional[str] # lets-encrypt @@ -245,7 +244,7 @@ class Keycloak(Base): class Security(Base): authentication: Authentication - shared_users_group: typing.Optional[bool] + shared_users_group: bool = True keycloak: typing.Optional[Keycloak] @@ -451,9 +450,9 @@ class CondaEnvironment(Base): class CDSDashboards(Base): - enabled: bool - cds_hide_user_named_servers: typing.Optional[bool] - cds_hide_user_dashboard_servers: typing.Optional[bool] + enabled: bool = True + cds_hide_user_named_servers: bool = True + cds_hide_user_dashboard_servers: bool = False # =============== Extensions = = ============== @@ -580,16 +579,16 @@ class InitInputs(Base): class Main(Base): provider: ProviderEnum project_name: str - namespace: typing.Optional[letter_dash_underscore_pydantic] = "dev" - nebari_version: str = "" - ci_cd: typing.Optional[CICD] + namespace: letter_dash_underscore_pydantic = "dev" + nebari_version: str = __version__ + ci_cd: CICD = CICD() domain: str terraform_state: typing.Optional[TerraformState] - certificate: Certificate - helm_extensions: typing.Optional[typing.List[HelmExtension]] - prefect: typing.Optional[Prefect] - cdsdashboards: CDSDashboards - security: Security + certificate: Certificate = Certificate() + helm_extensions: typing.List[HelmExtension] = [] + prefect: Prefect = Prefect() + cdsdashboards: CDSDashboards = CDSDashboards() + security: Security = Security() external_container_reg: typing.Optional[ExtContainerReg] default_images: DefaultImages storage: typing.Dict[str, str] From 4540a1de45881bd8ddbbebb046f4671d566540d1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 14 Jun 2023 19:22:22 +0000 Subject: [PATCH 004/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/keycloak.py | 2 +- src/_nebari/stages/base.py | 27 ++++---- src/_nebari/stages/infrastructure/__init__.py | 29 +++++---- .../stages/kubernetes_ingress/__init__.py | 24 ++++--- .../stages/kubernetes_initialize/__init__.py | 19 +++--- .../stages/kubernetes_keycloak/__init__.py | 62 +++++++++++-------- .../__init__.py | 17 ++--- .../stages/kubernetes_services/__init__.py | 29 ++++----- .../stages/nebari_tf_extensions/__init__.py | 5 +- .../stages/terraform_state/__init__.py | 5 +- src/_nebari/upgrade.py | 1 + src/nebari/hookspecs.py | 4 +- src/nebari/plugins.py | 2 +- 13 files changed, 124 insertions(+), 102 deletions(-) diff --git a/src/_nebari/keycloak.py b/src/_nebari/keycloak.py index ae612be86..9f49278ff 100644 --- a/src/_nebari/keycloak.py +++ b/src/_nebari/keycloak.py @@ -7,8 +7,8 @@ import requests import rich -from nebari.schema import verify from _nebari.utils import load_yaml +from nebari.schema import verify logger = logging.getLogger(__name__) diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index e8a514409..e332ca627 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -1,6 +1,6 @@ -from typing import List, Dict, Tuple, Any -import pathlib import contextlib +import pathlib +from typing import Any, Dict, List, Tuple from _nebari import schema from _nebari.provider import terraform @@ -24,20 +24,21 @@ def state_imports(self) -> List[Tuple[str, str]]: return [] def tf_objects(self) -> List[Dict]: - return [ - NebariTerraformState(self.name, config) - ] + return [NebariTerraformState(self.name, config)] def render(self): contents = { - str(self.output_directory / stage_prefix / "_nebari.tf.json"): terraform.tf_render_objects(self.tf_objects()) + str( + self.output_directory / stage_prefix / "_nebari.tf.json" + ): terraform.tf_render_objects(self.tf_objects()) } for root, dirs, files in os.walk(self.template_directory): for filename in filenames: - contents[os.path.join(root, filename)] = open(os.path.join(root, filename)).read() + contents[os.path.join(root, filename)] = open( + os.path.join(root, filename) + ).read() return contents - def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): return {} @@ -45,12 +46,12 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): deploy_config = dict( directory=str(self.output_directory / self.stage_prefix), - input_vars=self.input_vars(stage_outputs) + input_vars=self.input_vars(stage_outputs), ) state_imports = self.state_imports() if state_imports: - deploy_config['terraform_import'] = True - deploy_config['state_imports'] = state_imports + deploy_config["terraform_import"] = True + deploy_config["state_imports"] = state_imports stage_outputs["stages/" + self.name] = terraform.deploy(**deploy_config) yield @@ -59,7 +60,9 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]): pass @contextlib.contextmanager - def destroy(self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool]): + def destroy( + self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool] + ): stage_outputs["stages/" + self.name] = terraform.deploy( directory=str(self.output_directory / self.stage_prefix), input_vars=self.input_vars(stage_outputs), diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 1c465a68e..cc103fd55 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -1,7 +1,7 @@ import contextlib -from typing import List, Dict, Any import pathlib import sys +from typing import Any, Dict, List from _nebari import schema from _nebari.stages.base import NebariTerraformStage @@ -11,9 +11,8 @@ NebariTerraformState, ) from _nebari.utils import modified_environ -from nebari.hookspecs import NebariStage, hookimpl - from nebari import schema +from nebari.hookspecs import NebariStage, hookimpl @contextlib.contextmanager @@ -123,8 +122,8 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "kubeconfig_filename": os.path.join( tempfile.gettempdir(), "NEBARI_KUBECONFIG" ), - "resource_group_name": f'{self.config.project_name}-{self.config.namespace}', - "node_resource_group_name": f'{self.config.project_name}-{self.config.namespace}-node-resource-group', + "resource_group_name": f"{self.config.project_name}-{self.config.namespace}", + "node_resource_group_name": f"{self.config.project_name}-{self.config.namespace}-node-resource-group", **self.config.azure.terraform_overrides, } elif self.config.provider == schema.ProviderEnum.aws: @@ -157,11 +156,10 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]): from kubernetes import client, config from kubernetes.client.rest import ApiException - directory = "stages/02-infrastructure" config.load_kube_config( - config_file=stage_outputs["stages/02-infrastructure"]["kubeconfig_filename"][ - "value" - ] + config_file=stage_outputs["stages/02-infrastructure"][ + "kubeconfig_filename" + ]["value"] ) try: @@ -192,16 +190,23 @@ def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): yield @contextlib.contextmanager - def destroy(self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool]): + def destroy( + self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool] + ): with super().destroy(stage_outputs, status): with kubernetes_provider_context( stage_outputs["stages/" + self.name]["kubernetes_credentials"]["value"] ): yield + @hookimpl -def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> List[NebariStage]: - template_directory = pathlib.Path(__file__).parent / "template" / config.provider.value +def nebari_stage( + install_directory: pathlib.Path, config: schema.Main +) -> List[NebariStage]: + template_directory = ( + pathlib.Path(__file__).parent / "template" / config.provider.value + ) stage_prefix = pathlib.Path("stages/02-infrastructure") / config.provider.value return [ diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index f6def3c07..4db63e60d 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -1,7 +1,7 @@ -from typing import List, Dict, Any import pathlib import socket import sys +from typing import Any, Dict, List from _nebari import constants, schema from _nebari.stages.base import NebariTerraformStage @@ -10,16 +10,14 @@ NebariKubernetesProvider, NebariTerraformState, ) -from nebari.hookspecs import NebariStage, hookimpl -from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider - from nebari import schema -from _nebari import constants +from nebari.hookspecs import NebariStage, hookimpl # check and retry settings NUM_ATTEMPTS = 10 TIMEOUT = 10 # seconds + def _calculate_node_groups(config: schema.Main): if config.provider == schema.ProviderEnum.aws: return { @@ -115,7 +113,9 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): cert_details["acme-email"] = self.config.certificate.acme_email cert_details["acme-server"] = self.config.certificate.acme_server elif cert_type == "existing": - cert_details["certificate-secret-name"] = self.config.certificate.secret_name + cert_details[ + "certificate-secret-name" + ] = self.config.certificate.secret_name return { **{ @@ -132,7 +132,9 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): } def check(self, stage_outputs: Dict[str, Dict[str, Any]]): - def _attempt_tcp_connect(host, port, num_attempts=NUM_ATTEMPTS, timeout=TIMEOUT): + def _attempt_tcp_connect( + host, port, num_attempts=NUM_ATTEMPTS, timeout=TIMEOUT + ): for i in range(num_attempts): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: @@ -141,7 +143,9 @@ def _attempt_tcp_connect(host, port, num_attempts=NUM_ATTEMPTS, timeout=TIMEOUT) s.settimeout(5) result = s.connect_ex((ip, port)) if result == 0: - print(f"Attempt {i+1} succeeded to connect to tcp://{ip}:{port}") + print( + f"Attempt {i+1} succeeded to connect to tcp://{ip}:{port}" + ) return True print(f"Attempt {i+1} failed to connect to tcp tcp://{ip}:{port}") except socket.gaierror: @@ -161,7 +165,9 @@ def _attempt_tcp_connect(host, port, num_attempts=NUM_ATTEMPTS, timeout=TIMEOUT) 9080, # minio 8786, # dask-scheduler } - ip_or_name = stage_outputs["stages/" + self.name]["load_balancer_address"]["value"] + ip_or_name = stage_outputs["stages/" + self.name]["load_balancer_address"][ + "value" + ] host = ip_or_name["hostname"] or ip_or_name["ip"] host = host.strip("\n") diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py index 8319e1297..42f6a2895 100644 --- a/src/_nebari/stages/kubernetes_initialize/__init__.py +++ b/src/_nebari/stages/kubernetes_initialize/__init__.py @@ -1,6 +1,6 @@ -from typing import List, Dict, Any import pathlib import sys +from typing import Any, Dict, List from _nebari import schema from _nebari.stages.base import NebariTerraformStage @@ -9,8 +9,8 @@ NebariKubernetesProvider, NebariTerraformState, ) -from nebari.hookspecs import NebariStage, hookimpl from nebari import schema +from nebari.hookspecs import NebariStage, hookimpl class KubernetesInitializeStage(NebariTerraformStage): @@ -59,9 +59,9 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]): from kubernetes.client.rest import ApiException config.load_kube_config( - config_file=stage_outputs["stages/02-infrastructure"]["kubeconfig_filename"][ - "value" - ] + config_file=stage_outputs["stages/02-infrastructure"][ + "kubeconfig_filename" + ]["value"] ) try: @@ -80,14 +80,13 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]): ) sys.exit(1) - self.log.info( - f"After stage={self.name} kubernetes initialized successfully" - ) - + self.log.info(f"After stage={self.name} kubernetes initialized successfully") @hookimpl -def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> List[NebariStage]: +def nebari_stage( + install_directory: pathlib.Path, config: schema.Main +) -> List[NebariStage]: return [ KubernetesInitializeStage( install_directory, diff --git a/src/_nebari/stages/kubernetes_keycloak/__init__.py b/src/_nebari/stages/kubernetes_keycloak/__init__.py index 5c6c4357f..b060e771d 100644 --- a/src/_nebari/stages/kubernetes_keycloak/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak/__init__.py @@ -1,15 +1,18 @@ -from typing import List, Dict, Any +import contextlib +import json import pathlib import sys -import json -import contextlib +from typing import Any, Dict, List -from nebari.hookspecs import NebariStage, hookimpl -from _nebari.utils import modified_environ from _nebari.stages.base import NebariTerraformStage -from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider - +from _nebari.stages.tf_objects import ( + NebariHelmProvider, + NebariKubernetesProvider, + NebariTerraformState, +) +from _nebari.utils import modified_environ from nebari import schema +from nebari.hookspecs import NebariStage, hookimpl @contextlib.contextmanager @@ -71,20 +74,15 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "environment": self.config.namespace, "endpoint": self.config.domain, "initial-root-password": self.config.security.keycloak.initial_root_password, - "overrides": [ - json.dumps(self.config.security.keycloak.overrides) - ], + "overrides": [json.dumps(self.config.security.keycloak.overrides)], "node-group": _calculate_node_groups(self.config)["general"], } - def check(self, stage_outputs: Dict[str, Dict[str, Any]]): from keycloak import KeycloakAdmin from keycloak.exceptions import KeycloakError - keycloak_url = ( - f"{stage_outputs['stages/' + self.name]['keycloak_credentials']['value']['url']}/auth/" - ) + keycloak_url = f"{stage_outputs['stages/' + self.name]['keycloak_credentials']['value']['url']}/auth/" def _attempt_keycloak_connection( keycloak_url, @@ -106,19 +104,31 @@ def _attempt_keycloak_connection( client_id=client_id, verify=verify, ) - self.log.info(f"Attempt {i+1} succeeded connecting to keycloak master realm") + self.log.info( + f"Attempt {i+1} succeeded connecting to keycloak master realm" + ) return True except KeycloakError: - self.log.info(f"Attempt {i+1} failed connecting to keycloak master realm") + self.log.info( + f"Attempt {i+1} failed connecting to keycloak master realm" + ) time.sleep(timeout) return False if not _attempt_keycloak_connection( keycloak_url, - stage_outputs['stages/' + self.name]["keycloak_credentials"]["value"]["username"], - stage_outputs['stages/' + self.name]["keycloak_credentials"]["value"]["password"], - stage_outputs['stages/' + self.name]["keycloak_credentials"]["value"]["realm"], - stage_outputs['stages/' + self.name]["keycloak_credentials"]["value"]["client_id"], + stage_outputs["stages/" + self.name]["keycloak_credentials"]["value"][ + "username" + ], + stage_outputs["stages/" + self.name]["keycloak_credentials"]["value"][ + "password" + ], + stage_outputs["stages/" + self.name]["keycloak_credentials"]["value"][ + "realm" + ], + stage_outputs["stages/" + self.name]["keycloak_credentials"]["value"][ + "client_id" + ], verify=False, ): self.log.error( @@ -132,19 +142,17 @@ def _attempt_keycloak_connection( def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): with super().deploy(stage_outputs): with keycloak_provider_context( - stage_outputs["stages/" + self.name]["keycloak_credentials"][ - "value" - ] + stage_outputs["stages/" + self.name]["keycloak_credentials"]["value"] ): yield @contextlib.contextmanager - def destroy(self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool]): + def destroy( + self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool] + ): with super().destroy(stage_outputs, status): with keycloak_provider_context( - stage_outputs["stages/" + self.name]["keycloak_credentials"][ - "value" - ] + stage_outputs["stages/" + self.name]["keycloak_credentials"]["value"] ): yield diff --git a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py index 3704f1a9b..5436e7d60 100644 --- a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py @@ -1,12 +1,11 @@ -from typing import List, Dict, Any import pathlib import sys +from typing import Any, Dict, List from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import NebariTerraformState -from nebari.hookspecs import NebariStage, hookimpl - from nebari import schema +from nebari.hookspecs import NebariStage, hookimpl class KubernetesKeycloakConfigurationStage(NebariTerraformStage): @@ -21,9 +20,7 @@ def tf_objects(self) -> List[Dict]: def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): realm_id = "nebari" - users_group = ( - ["users"] if self.config.security.shared_users_group else [] - ) + users_group = ["users"] if self.config.security.shared_users_group else [] return { "realm": realm_id, @@ -76,7 +73,9 @@ def _attempt_keycloak_connection( f"Attempt {i+1} succeeded connecting to keycloak but nebari realm did not exist" ) except KeycloakError: - self.log.info(f"Attempt {i+1} failed connecting to keycloak master realm") + self.log.info( + f"Attempt {i+1} failed connecting to keycloak master realm" + ) time.sleep(timeout) return False @@ -100,7 +99,9 @@ def _attempt_keycloak_connection( @hookimpl -def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> List[NebariStage]: +def nebari_stage( + install_directory: pathlib.Path, config: schema.Main +) -> List[NebariStage]: return [ KubernetesKeycloakConfigurationStage( install_directory, diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index 20edef929..647bb67a2 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -1,13 +1,16 @@ -from typing import List, Dict, Any import pathlib import sys +from typing import Any, Dict, List from urllib.parse import urlencode -from nebari.hookspecs import NebariStage, hookimpl from _nebari.stages.base import NebariTerraformStage -from _nebari.stages.tf_objects import NebariTerraformState, NebariKubernetesProvider, NebariHelmProvider - +from _nebari.stages.tf_objects import ( + NebariHelmProvider, + NebariKubernetesProvider, + NebariTerraformState, +) from nebari import schema +from nebari.hookspecs import NebariStage, hookimpl # check and retry settings NUM_ATTEMPTS = 10 @@ -118,9 +121,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "jupyterlab-image": _split_docker_image_name( self.config.default_images.jupyterlab ), - "jupyterhub-overrides": [ - json.dumps(self.config.jupyterhub.overrides) - ], + "jupyterhub-overrides": [json.dumps(self.config.jupyterhub.overrides)], "jupyterhub-hub-extraEnv": json.dumps( self.config.jupyterhub.overrides.hub.extraEnv ), @@ -153,13 +154,10 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): # clearml "clearml-enabled": self.config.clearml.enabled, "clearml-enable-forwardauth": self.config.clearml.enable_forward_auth, - "clearml-overrides": [ - json.dumps(self.config.clearml.overrides) - ], + "clearml-overrides": [json.dumps(self.config.clearml.overrides)], "jupyterhub-logout-redirect-url": final_logout_uri, } - def check(self, stage_outputs: Dict[str, Dict[str, Any]]): directory = "stages/07-kubernetes-services" import requests @@ -186,13 +184,16 @@ def _attempt_connect_url( for service_name, service in services.items(): service_url = service["health_url"] if service_url and not _attempt_connect_url(service_url): - self.log.error(f"ERROR: Service {service_name} DOWN when checking url={service_url}") + self.log.error( + f"ERROR: Service {service_name} DOWN when checking url={service_url}" + ) sys.exit(1) - @hookimpl -def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> List[NebariStage]: +def nebari_stage( + install_directory: pathlib.Path, config: schema.Main +) -> List[NebariStage]: return [ KubernetesServicesStage( install_directory, diff --git a/src/_nebari/stages/nebari_tf_extensions/__init__.py b/src/_nebari/stages/nebari_tf_extensions/__init__.py index 8694ca73d..fb1a74e6f 100644 --- a/src/_nebari/stages/nebari_tf_extensions/__init__.py +++ b/src/_nebari/stages/nebari_tf_extensions/__init__.py @@ -1,5 +1,5 @@ -from typing import List, Dict, Any import pathlib +from typing import Any, Dict, List from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( @@ -7,9 +7,8 @@ NebariKubernetesProvider, NebariTerraformState, ) -from nebari.hookspecs import NebariStage, hookimpl - from nebari import schema +from nebari.hookspecs import NebariStage, hookimpl class NebariTFExtensionsStage(NebariTerraformStage): diff --git a/src/_nebari/stages/terraform_state/__init__.py b/src/_nebari/stages/terraform_state/__init__.py index 6e8143097..22351bc55 100644 --- a/src/_nebari/stages/terraform_state/__init__.py +++ b/src/_nebari/stages/terraform_state/__init__.py @@ -1,6 +1,6 @@ import os -from typing import List, Tuple, Dict, Any import pathlib +from typing import Any, Dict, List, Tuple from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( @@ -8,9 +8,8 @@ NebariKubernetesProvider, NebariTerraformState, ) -from nebari.hookspecs import NebariStage, hookimpl - from nebari import schema +from nebari.hookspecs import NebariStage, hookimpl class KubernetesInitializeStage(NebariTerraformStage): diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 551904179..2f53492ae 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -11,6 +11,7 @@ from rich.prompt import Prompt from nebari.schema import is_version_accepted, verify + from .utils import backup_config_file, load_yaml, yaml from .version import __version__, rounded_ver_parse diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index b1035b1df..07116f4ce 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -1,7 +1,7 @@ -from typing import Optional, Dict, Any -from collections.abc import Iterable import contextlib import pathlib +from collections.abc import Iterable +from typing import Any, Dict from pluggy import HookimplMarker, HookspecMarker diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index a705d3c46..cb2f198a3 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -1,5 +1,5 @@ -import sys import importlib +import sys import pluggy From 39f6419259c6b01d8d00dc45cd9a74e85c95c7b8 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 14 Jun 2023 16:15:10 -0400 Subject: [PATCH 005/147] Setting defaults for schema --- src/_nebari/stages/base.py | 3 +- src/_nebari/stages/infrastructure/__init__.py | 2 +- .../stages/kubernetes_ingress/__init__.py | 2 +- .../stages/kubernetes_initialize/__init__.py | 2 +- src/nebari/hookspecs.py | 2 +- src/nebari/schema.py | 160 +++++++++++++++--- 6 files changed, 147 insertions(+), 24 deletions(-) diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index e332ca627..405ab1b2a 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -2,11 +2,12 @@ import pathlib from typing import Any, Dict, List, Tuple -from _nebari import schema from _nebari.provider import terraform from _nebari.stages.tf_objects import NebariTerraformState from nebari.hookspecs import NebariStage +from nebari import schema + class NebariTerraformStage(NebariStage): def __init__( diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index cc103fd55..e6657be89 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -3,7 +3,6 @@ import sys from typing import Any, Dict, List -from _nebari import schema from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( NebariAWSProvider, @@ -11,6 +10,7 @@ NebariTerraformState, ) from _nebari.utils import modified_environ + from nebari import schema from nebari.hookspecs import NebariStage, hookimpl diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index 4db63e60d..663d29ba6 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -3,7 +3,7 @@ import sys from typing import Any, Dict, List -from _nebari import constants, schema +from _nebari import constants from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( NebariHelmProvider, diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py index 42f6a2895..15f295517 100644 --- a/src/_nebari/stages/kubernetes_initialize/__init__.py +++ b/src/_nebari/stages/kubernetes_initialize/__init__.py @@ -2,13 +2,13 @@ import sys from typing import Any, Dict, List -from _nebari import schema from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( NebariHelmProvider, NebariKubernetesProvider, NebariTerraformState, ) + from nebari import schema from nebari.hookspecs import NebariStage, hookimpl diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index 07116f4ce..90938233a 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -8,7 +8,7 @@ hookspec = HookspecMarker("nebari") hookimpl = HookimplMarker("nebari") -from _nebari import schema +from nebari import schema class NebariStage: diff --git a/src/nebari/schema.py b/src/nebari/schema.py index b594a9824..4024878ff 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -1,11 +1,13 @@ import enum import typing from abc import ABC +import secrets +import string import pydantic -from pydantic import root_validator, validator +from pydantic import root_validator, validator, Field -from _nebari.utils import namestr_regex +from _nebari.utils import namestr_regex, set_docker_image_tag, set_nebari_dask_version from _nebari.version import __version__, rounded_ver_parse @@ -163,9 +165,16 @@ class Certificate(Base): class DefaultImages(Base): - jupyterhub: str - jupyterlab: str - dask_worker: str + jupyterhub: str = f"quay.io/nebari/nebari-jupyterhub:{set_docker_image_tag()}" + jupyterlab: str = f"quay.io/nebari/nebari-jupyterlab:{set_docker_image_tag()}" + dask_worker: str = f"quay.io/nebari/nebari-dask-worker:{set_docker_image_tag()}" + + +# =========== Storage ============= + +class Storage(Base): + conda_store: str = "200Gi" + shared_filesystem: str = "200Gi" # =========== Authentication ============== @@ -231,21 +240,25 @@ class GitHubAuthentication(Authentication): # ================= Keycloak ================== +def random_password(length: int = 32): + return "".join( + secrets.choice(string.ascii_letters + string.digits) for i in range(16) + ) class Keycloak(Base): - initial_root_password: typing.Optional[str] - overrides: typing.Optional[typing.Dict] - realm_display_name: typing.Optional[str] + initial_root_password: str = Field(default_factory=random_password) + overrides: typing.Dict = {} + realm_display_name: str = "Nebari" # ============== Security ================ class Security(Base): - authentication: Authentication + authentication: Authentication = PasswordAuthentication(type=AuthenticationEnum.password) shared_users_group: bool = True - keycloak: typing.Optional[Keycloak] + keycloak: Keycloak = Keycloak() # ================ Providers =============== @@ -348,7 +361,13 @@ class ExistingProvider(Base): class Theme(Base): - jupyterhub: typing.Dict[str, typing.Union[str, list]] + jupyterhub: typing.Dict[str, typing.Union[str, list]] = dict( + hub_title = "Nebari", + hub_subtitle = "Your open source data science platform", + welcome = """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""", + logo = "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg", + display_version=True, + ) # ================= Theme ================== @@ -423,8 +442,45 @@ class Config: class Profiles(Base): - jupyterlab: typing.List[JupyterLabProfile] - dask_worker: typing.Dict[str, DaskWorkerProfile] + jupyterlab: typing.List[JupyterLabProfile] = [ + JupyterLabProfile( + display_name = "Small Instance", + description = "Stable environment with 2 cpu / 8 GB ram", + default = True, + kubespawner_override = KubeSpawner( + cpu_limit = 2, + cpu_guarantee = 1.5, + mem_limit = "8G", + mem_guarantee = "5G", + ) + ), + JupyterLabProfile( + display_name = "Medium Instance", + description = "Stable environment with 4 cpu / 16 GB ram", + kubespawner_override = KubeSpawner( + cpu_limit = 4, + cpu_guarantee = 3, + mem_limit = "16G", + mem_guarantee = "10G", + ) + ) + ] + dask_worker: typing.Dict[str, DaskWorkerProfile] = { + "Small Worker": DaskWorkerProfile( + worker_cores_limit = 2, + worker_cores = 1.5, + worker_memory_limit = "8G", + worker_memory = "5G", + worker_threads = 2, + ), + "Medium Worker": DaskWorkerProfile( + worker_cores_limit = 4, + worker_cores = 3, + worker_memory_limit = "16G", + worker_memory = "10G", + worker_threads = 4, + ) + } @validator("jupyterlab") def check_default(cls, v, values): @@ -577,7 +633,7 @@ class InitInputs(Base): class Main(Base): - provider: ProviderEnum + provider: ProviderEnum = ProviderEnum.local project_name: str namespace: letter_dash_underscore_pydantic = "dev" nebari_version: str = __version__ @@ -590,17 +646,83 @@ class Main(Base): cdsdashboards: CDSDashboards = CDSDashboards() security: Security = Security() external_container_reg: typing.Optional[ExtContainerReg] - default_images: DefaultImages - storage: typing.Dict[str, str] + default_images: DefaultImages = DefaultImages() + storage: Storage = Storage() local: typing.Optional[LocalProvider] existing: typing.Optional[ExistingProvider] google_cloud_platform: typing.Optional[GoogleCloudPlatformProvider] amazon_web_services: typing.Optional[AmazonWebServicesProvider] azure: typing.Optional[AzureProvider] digital_ocean: typing.Optional[DigitalOceanProvider] - theme: Theme - profiles: Profiles - environments: typing.Dict[str, CondaEnvironment] + theme: Theme = Theme() + profiles: Profiles = Profiles() + environments: typing.Dict[str, CondaEnvironment] = { + "environment-dask.yaml": CondaEnvironment( + name = "dask", + channels = ["conda-forge"], + dependencies = [ + "python=3.10.8", + "ipykernel=6.21.0", + "ipywidgets==7.7.1", + f"nebari-dask =={set_nebari_dask_version()}", + "python-graphviz=0.20.1", + "pyarrow=10.0.1", + "s3fs=2023.1.0", + "gcsfs=2023.1.0", + "numpy=1.23.5", + "numba=0.56.4", + "pandas=1.5.3", + { + "pip": [ + "kbatch==0.4.1", + ], + }, + ], + ), + "environment-dashboard.yaml": CondaEnvironment( + name = "dashboard", + channels = ["conda-forge"], + dependencies = [ + "python=3.10", + "cdsdashboards-singleuser=0.6.3", + "cufflinks-py=0.17.3", + "dash=2.8.1", + "geopandas=0.12.2", + "geopy=2.3.0", + "geoviews=1.9.6", + "gunicorn=20.1.0", + "holoviews=1.15.4", + "ipykernel=6.21.2", + "ipywidgets=8.0.4", + "jupyter=1.0.0", + "jupyterlab=3.6.1", + "jupyter_bokeh=3.0.5", + "matplotlib=3.7.0", + f"nebari-dask=={set_nebari_dask_version()}", + "nodejs=18.12.1", + "numpy", + "openpyxl=3.1.1", + "pandas=1.5.3", + "panel=0.14.3", + "param=1.12.3", + "plotly=5.13.0", + "python-graphviz=0.20.1", + "rich=13.3.1", + "streamlit=1.9.0", + "sympy=1.11.1", + "voila=0.4.0", + "pip=23.0", + { + "pip": [ + "streamlit-image-comparison==0.0.3", + "noaa-coops==0.2.1", + "dash_core_components==2.0.0", + "dash_html_components==2.0.0", + ], + }, + ], + ), + } conda_store: typing.Optional[CondaStore] argo_workflows: typing.Optional[ArgoWorkflows] kbatch: typing.Optional[KBatch] From c533d3df87c1d46209ac79672a19ab942c6a38a4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 14 Jun 2023 20:15:29 +0000 Subject: [PATCH 006/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/stages/base.py | 3 +- src/_nebari/stages/infrastructure/__init__.py | 1 - .../stages/kubernetes_initialize/__init__.py | 1 - src/nebari/schema.py | 89 ++++++++++--------- 4 files changed, 47 insertions(+), 47 deletions(-) diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index 405ab1b2a..f57484a24 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -4,9 +4,8 @@ from _nebari.provider import terraform from _nebari.stages.tf_objects import NebariTerraformState -from nebari.hookspecs import NebariStage - from nebari import schema +from nebari.hookspecs import NebariStage class NebariTerraformStage(NebariStage): diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index e6657be89..7c9d8c575 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -10,7 +10,6 @@ NebariTerraformState, ) from _nebari.utils import modified_environ - from nebari import schema from nebari.hookspecs import NebariStage, hookimpl diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py index 15f295517..b5805e6d8 100644 --- a/src/_nebari/stages/kubernetes_initialize/__init__.py +++ b/src/_nebari/stages/kubernetes_initialize/__init__.py @@ -8,7 +8,6 @@ NebariKubernetesProvider, NebariTerraformState, ) - from nebari import schema from nebari.hookspecs import NebariStage, hookimpl diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 4024878ff..88aff7eb5 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -1,11 +1,11 @@ import enum -import typing -from abc import ABC import secrets import string +import typing +from abc import ABC import pydantic -from pydantic import root_validator, validator, Field +from pydantic import Field, root_validator, validator from _nebari.utils import namestr_regex, set_docker_image_tag, set_nebari_dask_version from _nebari.version import __version__, rounded_ver_parse @@ -172,6 +172,7 @@ class DefaultImages(Base): # =========== Storage ============= + class Storage(Base): conda_store: str = "200Gi" shared_filesystem: str = "200Gi" @@ -256,7 +257,9 @@ class Keycloak(Base): class Security(Base): - authentication: Authentication = PasswordAuthentication(type=AuthenticationEnum.password) + authentication: Authentication = PasswordAuthentication( + type=AuthenticationEnum.password + ) shared_users_group: bool = True keycloak: Keycloak = Keycloak() @@ -362,10 +365,10 @@ class ExistingProvider(Base): class Theme(Base): jupyterhub: typing.Dict[str, typing.Union[str, list]] = dict( - hub_title = "Nebari", - hub_subtitle = "Your open source data science platform", - welcome = """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""", - logo = "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg", + hub_title="Nebari", + hub_subtitle="Your open source data science platform", + welcome="""Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""", + logo="https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg", display_version=True, ) @@ -444,42 +447,42 @@ class Config: class Profiles(Base): jupyterlab: typing.List[JupyterLabProfile] = [ JupyterLabProfile( - display_name = "Small Instance", - description = "Stable environment with 2 cpu / 8 GB ram", - default = True, - kubespawner_override = KubeSpawner( - cpu_limit = 2, - cpu_guarantee = 1.5, - mem_limit = "8G", - mem_guarantee = "5G", - ) + display_name="Small Instance", + description="Stable environment with 2 cpu / 8 GB ram", + default=True, + kubespawner_override=KubeSpawner( + cpu_limit=2, + cpu_guarantee=1.5, + mem_limit="8G", + mem_guarantee="5G", + ), ), JupyterLabProfile( - display_name = "Medium Instance", - description = "Stable environment with 4 cpu / 16 GB ram", - kubespawner_override = KubeSpawner( - cpu_limit = 4, - cpu_guarantee = 3, - mem_limit = "16G", - mem_guarantee = "10G", - ) - ) + display_name="Medium Instance", + description="Stable environment with 4 cpu / 16 GB ram", + kubespawner_override=KubeSpawner( + cpu_limit=4, + cpu_guarantee=3, + mem_limit="16G", + mem_guarantee="10G", + ), + ), ] dask_worker: typing.Dict[str, DaskWorkerProfile] = { "Small Worker": DaskWorkerProfile( - worker_cores_limit = 2, - worker_cores = 1.5, - worker_memory_limit = "8G", - worker_memory = "5G", - worker_threads = 2, + worker_cores_limit=2, + worker_cores=1.5, + worker_memory_limit="8G", + worker_memory="5G", + worker_threads=2, ), "Medium Worker": DaskWorkerProfile( - worker_cores_limit = 4, - worker_cores = 3, - worker_memory_limit = "16G", - worker_memory = "10G", - worker_threads = 4, - ) + worker_cores_limit=4, + worker_cores=3, + worker_memory_limit="16G", + worker_memory="10G", + worker_threads=4, + ), } @validator("jupyterlab") @@ -658,9 +661,9 @@ class Main(Base): profiles: Profiles = Profiles() environments: typing.Dict[str, CondaEnvironment] = { "environment-dask.yaml": CondaEnvironment( - name = "dask", - channels = ["conda-forge"], - dependencies = [ + name="dask", + channels=["conda-forge"], + dependencies=[ "python=3.10.8", "ipykernel=6.21.0", "ipywidgets==7.7.1", @@ -680,9 +683,9 @@ class Main(Base): ], ), "environment-dashboard.yaml": CondaEnvironment( - name = "dashboard", - channels = ["conda-forge"], - dependencies = [ + name="dashboard", + channels=["conda-forge"], + dependencies=[ "python=3.10", "cdsdashboards-singleuser=0.6.3", "cufflinks-py=0.17.3", From 828d3d3b14d4bdba1304e4f88638275e2b67f88f Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 14 Jun 2023 21:37:52 -0400 Subject: [PATCH 007/147] Working subcommands using extension system --- pyproject.toml | 2 +- src/_nebari/__main__.py | 10 ++ src/_nebari/cli/dev.py | 43 ------- src/_nebari/render.py | 26 ++--- src/_nebari/stages/base.py | 38 +++++-- src/_nebari/stages/infrastructure/__init__.py | 25 ++-- .../stages/kubernetes_ingress/__init__.py | 9 +- .../stages/kubernetes_initialize/__init__.py | 11 +- .../stages/kubernetes_keycloak/__init__.py | 9 +- .../__init__.py | 11 +- .../stages/kubernetes_services/__init__.py | 11 +- .../stages/nebari_tf_extensions/__init__.py | 11 +- .../stages/terraform_state/__init__.py | 38 +++---- .../terraform_state/template/existing/main.tf | 0 .../terraform_state/template/local/main.tf | 0 src/_nebari/subcommands/__init__.py | 42 +++++++ src/_nebari/subcommands/deploy.py | 83 ++++++++++++++ src/_nebari/subcommands/destroy.py | 61 ++++++++++ src/_nebari/subcommands/dev.py | 53 +++++++++ src/_nebari/subcommands/init.py | 107 ++++++++++++++++++ src/_nebari/subcommands/keycloak.py | 86 ++++++++++++++ src/_nebari/subcommands/render.py | 46 ++++++++ src/_nebari/subcommands/support.py | 93 +++++++++++++++ src/_nebari/subcommands/upgrade.py | 40 +++++++ src/_nebari/subcommands/validate.py | 40 +++++++ src/nebari/hookspecs.py | 10 +- src/nebari/plugins.py | 11 ++ 27 files changed, 757 insertions(+), 159 deletions(-) create mode 100644 src/_nebari/__main__.py delete mode 100644 src/_nebari/cli/dev.py create mode 100644 src/_nebari/stages/terraform_state/template/existing/main.tf create mode 100644 src/_nebari/stages/terraform_state/template/local/main.tf create mode 100644 src/_nebari/subcommands/__init__.py create mode 100644 src/_nebari/subcommands/deploy.py create mode 100644 src/_nebari/subcommands/destroy.py create mode 100644 src/_nebari/subcommands/dev.py create mode 100644 src/_nebari/subcommands/init.py create mode 100644 src/_nebari/subcommands/keycloak.py create mode 100644 src/_nebari/subcommands/render.py create mode 100644 src/_nebari/subcommands/support.py create mode 100644 src/_nebari/subcommands/upgrade.py create mode 100644 src/_nebari/subcommands/validate.py diff --git a/pyproject.toml b/pyproject.toml index b8666a49f..85ac6a1e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ Documentation = "https://www.nebari.dev/docs" Source = "https://github.com/nebari-dev/nebari" [project.scripts] -nebari = "_nebari.cli.main:app" +nebari = "_nebari.__main__:main" [tool.ruff] select = [ diff --git a/src/_nebari/__main__.py b/src/_nebari/__main__.py new file mode 100644 index 000000000..ca69c5354 --- /dev/null +++ b/src/_nebari/__main__.py @@ -0,0 +1,10 @@ +from _nebari.subcommands import create_cli + + +def main(): + cli = create_cli() + cli() + + +if __name__ == "__main__": + main() diff --git a/src/_nebari/cli/dev.py b/src/_nebari/cli/dev.py deleted file mode 100644 index 824125532..000000000 --- a/src/_nebari/cli/dev.py +++ /dev/null @@ -1,43 +0,0 @@ -import json -from pathlib import Path - -import typer - -from _nebari.keycloak import keycloak_rest_api_call - -app_dev = typer.Typer( - add_completion=False, - no_args_is_help=True, - rich_markup_mode="rich", - context_settings={"help_option_names": ["-h", "--help"]}, -) - - -@app_dev.command(name="keycloak-api") -def keycloak_api( - config_filename: str = typer.Option( - ..., - "-c", - "--config", - help="nebari configuration file path", - ), - request: str = typer.Option( - ..., - "-r", - "--request", - help="Send a REST API request, valid requests follow patterns found here: [green]keycloak.org/docs-api/15.0/rest-api[/green]", - ), -): - """ - Interact with the Keycloak REST API directly. - - This is an advanced tool which can have potentially destructive consequences. - Please use this at your own risk. - - """ - if isinstance(config_filename, str): - config_filename = Path(config_filename) - - r = keycloak_rest_api_call(config_filename, request=request) - - print(json.dumps(r, indent=4)) diff --git a/src/_nebari/render.py b/src/_nebari/render.py index c25aa9c2e..568b784da 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -16,8 +16,9 @@ from _nebari.deprecate import DEPRECATED_FILE_PATHS from _nebari.provider.cicd.github import gen_nebari_linter, gen_nebari_ops from _nebari.provider.cicd.gitlab import gen_gitlab_ci -from _nebari.stages import tf_objects from _nebari.utils import is_relative_to +from _nebari.stages.base import get_available_stages +from nebari.hookspecs import NebariStage def render_template(output_directory, config_filename, dry_run=False): @@ -51,10 +52,12 @@ def render_template(output_directory, config_filename, dry_run=False): # corresponding env var. set_env_vars_in_config(config) - config["repo_directory"] = output_directory.name - config["nebari_config_yaml_path"] = str(config_filename.absolute()) + stages = [_( + output_directory=output_directory, + config=config, + ) for _ in get_available_stages()] - contents = render_contents(config) + contents = render_contents(stages, config) directories = [ f"stages/02-infrastructure/{config['provider']}", @@ -143,18 +146,11 @@ def render_template(output_directory, config_filename, dry_run=False): shutil.rmtree(abs_path) -def render_contents(config: Dict): +def render_contents(stages: List[NebariStage], config: Dict): """Dynamically generated contents from _nebari configuration.""" - contents = { - **tf_objects.stage_01_terraform_state(config), - **tf_objects.stage_02_infrastructure(config), - **tf_objects.stage_03_kubernetes_initialize(config), - **tf_objects.stage_04_kubernetes_ingress(config), - **tf_objects.stage_05_kubernetes_keycloak(config), - **tf_objects.stage_06_kubernetes_keycloak_configuration(config), - **tf_objects.stage_07_kubernetes_services(config), - **tf_objects.stage_08_nebari_tf_extensions(config), - } + contents = {} + for stage in stages: + contents.update(stage.render()) if config.get("ci_cd"): for fn, workflow in gen_cicd(config).items(): diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index f57484a24..db37f6321 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -1,5 +1,7 @@ import contextlib import pathlib +import inspect +import itertools from typing import Any, Dict, List, Tuple from _nebari.provider import terraform @@ -9,16 +11,13 @@ class NebariTerraformStage(NebariStage): - def __init__( - self, - output_directory: pathlib.Path, - config: schema.Main, - template_directory: pathlib.Path, - stage_prefix: pathlib.Path, - ): - super().__init__(output_directory, config) - self.template_directory = pathlib.Path(template_directory) - self.stage_prefix = pathlib.Path(stage_prefix) + @property + def template_directory(self): + return pathlib.Path(inspect.getfile(self.__class__).parent) / "template" + + @property + def stage_prefix(self): + return pathlib.Path('stages') / self.name def state_imports(self) -> List[Tuple[str, str]]: return [] @@ -77,3 +76,22 @@ def destroy( input_vars=self.input_vars(stage_outputs), ignore_errors=True, ) + + +def get_available_stages(): + from nebari.plugins import pm + stages = itertools.chain.from_iterable(pm.hook.nebari_stage()) + + # order stages by priority + sorted_stages = sorted(stages, key=lambda s: s.priority) + + # filter out duplicate stages with same name (keep highest priority) + visited_stage_names = set() + filtered_stages = [] + for stage in reversed(sorted_stages): + if stage.name in visited_stage_names: + continue + filtered_stages.insert(0, stage) + visited_stage_names.add(stage.name) + + return filtered_stages diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 7c9d8c575..93c8752e0 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -1,6 +1,7 @@ import contextlib import pathlib import sys +import inspect from typing import Any, Dict, List from _nebari.stages.base import NebariTerraformStage @@ -41,6 +42,14 @@ class KubernetesInfrastructureStage(NebariTerraformStage): name = "02-infrastructure" priority = 20 + @property + def template_directory(self): + return pathlib.Path(inspect.getfile(self.__class__).parent) / "template" / self.config.provider.value + + @property + def stage_prefix(self): + return pathlib.Path('stages') / self.name / self.config.provider.value + def tf_objects(self) -> List[Dict]: if self.config.provider == schema.ProviderEnum.gcp: return [ @@ -200,19 +209,7 @@ def destroy( @hookimpl -def nebari_stage( - install_directory: pathlib.Path, config: schema.Main -) -> List[NebariStage]: - template_directory = ( - pathlib.Path(__file__).parent / "template" / config.provider.value - ) - stage_prefix = pathlib.Path("stages/02-infrastructure") / config.provider.value - +def nebari_stage() -> List[NebariStage]: return [ - KubernetesInfrastructureStage( - install_directory, - config, - template_directory=template_directory, - stage_prefix=stage_prefix, - ) + KubernetesInfrastructureStage ] diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index 663d29ba6..2ea119b75 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -186,12 +186,7 @@ def _attempt_tcp_connect( @hookimpl -def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> [NebariStage]: +def nebari_stage() -> List[NebariStage]: return [ - KubernetesIngressStage( - install_directory, - config, - template_directory=(pathlib.Path(__file__).parent / "template"), - stage_prefix=pathlib.Path("stages/04-kubernetes-ingress"), - ) + KubernetesIngressStage ] diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py index b5805e6d8..e09effca6 100644 --- a/src/_nebari/stages/kubernetes_initialize/__init__.py +++ b/src/_nebari/stages/kubernetes_initialize/__init__.py @@ -83,14 +83,7 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]): @hookimpl -def nebari_stage( - install_directory: pathlib.Path, config: schema.Main -) -> List[NebariStage]: +def nebari_stage() -> List[NebariStage]: return [ - KubernetesInitializeStage( - install_directory, - config, - template_directory=(pathlib.Path(__file__).parent / "template"), - stage_prefix=pathlib.Path("stages/03-kubernetes-initialize"), - ) + KubernetesInitializeStage ] diff --git a/src/_nebari/stages/kubernetes_keycloak/__init__.py b/src/_nebari/stages/kubernetes_keycloak/__init__.py index b060e771d..182c6dafe 100644 --- a/src/_nebari/stages/kubernetes_keycloak/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak/__init__.py @@ -158,12 +158,7 @@ def destroy( @hookimpl -def nebari_stage(install_directory: pathlib.Path, config: schema.Main) -> [NebariStage]: +def nebari_stage() -> List[NebariStage]: return [ - KubernetesKeycloakStage( - install_directory, - config, - template_directory=(pathlib.Path(__file__).parent / "template"), - stage_prefix=pathlib.Path("stages/05-kubernetes-keycloak"), - ) + KubernetesKeycloakStage ] diff --git a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py index 5436e7d60..410bf96d6 100644 --- a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py @@ -99,14 +99,7 @@ def _attempt_keycloak_connection( @hookimpl -def nebari_stage( - install_directory: pathlib.Path, config: schema.Main -) -> List[NebariStage]: +def nebari_stage() -> List[NebariStage]: return [ - KubernetesKeycloakConfigurationStage( - install_directory, - config, - template_directory=(pathlib.Path(__file__).parent / "template"), - stage_prefix=pathlib.Path("stages/06-kubernetes-keycloak-configuration"), - ) + KubernetesKeycloakConfigurationStage ] diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index 647bb67a2..657c5f5d1 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -191,14 +191,7 @@ def _attempt_connect_url( @hookimpl -def nebari_stage( - install_directory: pathlib.Path, config: schema.Main -) -> List[NebariStage]: +def nebari_stage() -> List[NebariStage]: return [ - KubernetesServicesStage( - install_directory, - config, - template_directory=(pathlib.Path(__file__).parent / "template"), - stage_prefix=pathlib.Path("stages/07-kubernetes-services"), - ) + KubernetesServicesStage ] diff --git a/src/_nebari/stages/nebari_tf_extensions/__init__.py b/src/_nebari/stages/nebari_tf_extensions/__init__.py index fb1a74e6f..054cc43d2 100644 --- a/src/_nebari/stages/nebari_tf_extensions/__init__.py +++ b/src/_nebari/stages/nebari_tf_extensions/__init__.py @@ -39,14 +39,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): @hookimpl -def nebari_stage( - install_directory: pathlib.Path, config: schema.Main -) -> List[NebariStage]: +def nebari_stage() -> List[NebariStage]: return [ - NebariTFExtensionsStage( - install_directory, - config, - template_directory=(pathlib.Path(__file__).parent / "template"), - stage_prefix=pathlib.Path("stages/08-nebari-tf-extensions"), - ) + NebariTFExtensionsStage ] diff --git a/src/_nebari/stages/terraform_state/__init__.py b/src/_nebari/stages/terraform_state/__init__.py index 22351bc55..f4f71736c 100644 --- a/src/_nebari/stages/terraform_state/__init__.py +++ b/src/_nebari/stages/terraform_state/__init__.py @@ -1,5 +1,6 @@ import os import pathlib +import inspect from typing import Any, Dict, List, Tuple from _nebari.stages.base import NebariTerraformStage @@ -12,10 +13,18 @@ from nebari.hookspecs import NebariStage, hookimpl -class KubernetesInitializeStage(NebariTerraformStage): +class TerraformStateStage(NebariTerraformStage): name = "01-terraform-state" priority = 10 + @property + def template_directory(self): + return pathlib.Path(inspect.getfile(self.__class__).parent) / "template" / self.config.provider.value + + @property + def stage_prefix(self): + return pathlib.Path('stages') / self.name / self.config.provider.value + def state_imports(self) -> List[Tuple[str, str]]: if self.config.provider == schema.ProviderEnum.do: return [ @@ -63,13 +72,11 @@ def state_imports(self) -> List[Tuple[str, str]]: f"{self.config.project_name}-{self.config.namespace}-terraform-state-lock", ), ] + else: + return [] def tf_objects(self) -> List[Dict]: - return [ - NebariTerraformState(self.name, self.config), - NebariKubernetesProvider(self.config), - NebariHelmProvider(self.config), - ] + return [] def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): if self.config.provider == schema.ProviderEnum.do: @@ -102,22 +109,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): @hookimpl -def nebari_stage( - install_directory: pathlib.Path, config: schema.Main -) -> List[NebariStage]: - if config.provider in [schema.ProviderEnum.local, schema.ProviderEnum.existing]: - return [] - - template_directory = ( - pathlib.Path(__file__).parent / "template" / config.provider.value - ) - stage_prefix = pathlib.Path("stages/01-terraform-state") / config.provider.value - +def nebari_stage() -> List[NebariStage]: return [ - TerraformStateStage( - install_directory, - config, - template_directory=template_directory, - stage_prefix=stage_prefix, - ) + TerraformStateStage ] diff --git a/src/_nebari/stages/terraform_state/template/existing/main.tf b/src/_nebari/stages/terraform_state/template/existing/main.tf new file mode 100644 index 000000000..e69de29bb diff --git a/src/_nebari/stages/terraform_state/template/local/main.tf b/src/_nebari/stages/terraform_state/template/local/main.tf new file mode 100644 index 000000000..e69de29bb diff --git a/src/_nebari/subcommands/__init__.py b/src/_nebari/subcommands/__init__.py new file mode 100644 index 000000000..273cfd1ba --- /dev/null +++ b/src/_nebari/subcommands/__init__.py @@ -0,0 +1,42 @@ +from typing import Optional + +import typer +from click import Context +from typer.core import TyperGroup + +from _nebari.version import __version__ +from nebari.plugins import pm + + +class OrderCommands(TyperGroup): + def list_commands(self, ctx: typer.Context): + """Return list of commands in the order appear.""" + return list(self.commands) + + +def create_cli(): + app = typer.Typer( + cls=OrderCommands, + help="Nebari CLI 🪴", + add_completion=False, + no_args_is_help=True, + rich_markup_mode="rich", + context_settings={"help_option_names": ["-h", "--help"]}, + ) + pm.hook.nebari_subcommand(cli=app) + + @app.callback(invoke_without_command=True) + def version( + version: Optional[bool] = typer.Option( + None, + "-V", + "--version", + help="Nebari version number", + is_eager=True, + ), + ): + if version: + print(__version__) + raise typer.Exit() + + return app diff --git a/src/_nebari/subcommands/deploy.py b/src/_nebari/subcommands/deploy.py new file mode 100644 index 000000000..6c2d6cd85 --- /dev/null +++ b/src/_nebari/subcommands/deploy.py @@ -0,0 +1,83 @@ +import pathlib + +import typer + +from nebari.hookspecs import hookimpl +from nebari import schema +from _nebari.utils import load_yaml +from _nebari.render import render_template +from _nebari.deploy import deploy_configuration + + +@hookimpl +def nebari_subcommand(cli: typer.Typer): + @cli.command() + def deploy( + config: str = typer.Option( + ..., + "--config", + "-c", + help="nebari configuration yaml file path", + ), + output: str = typer.Option( + "./", + "-o", + "--output", + help="output directory", + ), + dns_provider: str = typer.Option( + False, + "--dns-provider", + help="dns provider to use for registering domain name mapping", + ), + dns_auto_provision: bool = typer.Option( + False, + "--dns-auto-provision", + help="Attempt to automatically provision DNS, currently only available for `cloudflare`", + ), + disable_prompt: bool = typer.Option( + False, + "--disable-prompt", + help="Disable human intervention", + ), + disable_render: bool = typer.Option( + False, + "--disable-render", + help="Disable auto-rendering in deploy stage", + ), + disable_checks: bool = typer.Option( + False, + "--disable-checks", + help="Disable the checks performed after each stage", + ), + skip_remote_state_provision: bool = typer.Option( + False, + "--skip-remote-state-provision", + help="Skip terraform state deployment which is often required in CI once the terraform remote state bootstrapping phase is complete", + ), + ): + """ + Deploy the Nebari cluster from your [purple]nebari-config.yaml[/purple] file. + """ + config_filename = pathlib.Path(config) + + if not config_filename.is_file(): + raise ValueError( + f"passed in configuration filename={config_filename} must exist" + ) + + config_yaml = load_yaml(config_filename) + + schema.verify(config_yaml) + + if not disable_render: + render_template(output, config) + + deploy_configuration( + config_yaml, + dns_provider=dns_provider, + dns_auto_provision=dns_auto_provision, + disable_prompt=disable_prompt, + disable_checks=disable_checks, + skip_remote_state_provision=skip_remote_state_provision, + ) diff --git a/src/_nebari/subcommands/destroy.py b/src/_nebari/subcommands/destroy.py new file mode 100644 index 000000000..19fa7f519 --- /dev/null +++ b/src/_nebari/subcommands/destroy.py @@ -0,0 +1,61 @@ +import pathlib + +import typer + +from nebari.hookspecs import hookimpl +from nebari import schema +from _nebari.utils import load_yaml +from _nebari.render import render_template +from _nebari.destroy import destroy_configuration + + +@hookimpl +def nebari_subcommand(cli: typer.Typer): + @app.command() + def destroy( + config: str = typer.Option( + ..., "-c", "--config", help="nebari configuration file path" + ), + output: str = typer.Option( + "./", + "-o", + "--output", + help="output directory", + ), + disable_render: bool = typer.Option( + False, + "--disable-render", + help="Disable auto-rendering before destroy", + ), + disable_prompt: bool = typer.Option( + False, + "--disable-prompt", + help="Destroy entire Nebari cluster without confirmation request. Suggested for CI use.", + ), + ): + """ + Destroy the Nebari cluster from your [purple]nebari-config.yaml[/purple] file. + """ + + def _run_destroy(config=config, disable_render=disable_render): + config_filename = pathlib.Path(config) + if not config_filename.is_file(): + raise ValueError( + f"passed in configuration filename={config_filename} must exist" + ) + + config_yaml = load_yaml(config_filename) + + schema.verify(config_yaml) + + if not disable_render: + render_template(output, config) + + destroy_configuration(config_yaml) + + if disable_prompt: + _run_destroy() + elif typer.confirm("Are you sure you want to destroy your Nebari cluster?"): + _run_destroy() + else: + raise typer.Abort() diff --git a/src/_nebari/subcommands/dev.py b/src/_nebari/subcommands/dev.py new file mode 100644 index 000000000..c66529f83 --- /dev/null +++ b/src/_nebari/subcommands/dev.py @@ -0,0 +1,53 @@ +import json +from pathlib import Path + +import typer + +from nebari.hookspecs import hookimpl +from _nebari.keycloak import keycloak_rest_api_call + + +@hookimpl +def nebari_subcommand(cli: typer.Typer): + app_dev = typer.Typer( + add_completion=False, + no_args_is_help=True, + rich_markup_mode="rich", + context_settings={"help_option_names": ["-h", "--help"]}, + ) + + cli.add_typer( + app_dev, + name="dev", + help="Development tools and advanced features.", + rich_help_panel="Additional Commands", + ) + + @app_dev.command(name="keycloak-api") + def keycloak_api( + config_filename: str = typer.Option( + ..., + "-c", + "--config", + help="nebari configuration file path", + ), + request: str = typer.Option( + ..., + "-r", + "--request", + help="Send a REST API request, valid requests follow patterns found here: [green]keycloak.org/docs-api/15.0/rest-api[/green]", + ), + ): + """ + Interact with the Keycloak REST API directly. + + This is an advanced tool which can have potentially destructive consequences. + Please use this at your own risk. + + """ + if isinstance(config_filename, str): + config_filename = Path(config_filename) + + r = keycloak_rest_api_call(config_filename, request=request) + + print(json.dumps(r, indent=4)) diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py new file mode 100644 index 000000000..d3f18e174 --- /dev/null +++ b/src/_nebari/subcommands/init.py @@ -0,0 +1,107 @@ +from _nebari.cli.init import ( + check_auth_provider_creds, + check_cloud_provider_creds, + check_project_name, + check_ssl_cert_email, + enum_to_list, + guided_init_wizard, + handle_init, +) + +@app.command() +def init( + cloud_provider: str = typer.Argument( + "local", + help=f"options: {enum_to_list(ProviderEnum)}", + callback=check_cloud_provider_creds, + is_eager=True, + ), + # Although this unused below, the functionality is contained in the callback. Thus, + # this attribute cannot be removed. + guided_init: bool = typer.Option( + False, + help=GUIDED_INIT_MSG, + callback=guided_init_wizard, + is_eager=True, + ), + project_name: str = typer.Option( + ..., + "--project-name", + "--project", + "-p", + callback=check_project_name, + ), + domain_name: str = typer.Option( + ..., + "--domain-name", + "--domain", + "-d", + ), + namespace: str = typer.Option( + "dev", + ), + auth_provider: str = typer.Option( + "password", + help=f"options: {enum_to_list(AuthenticationEnum)}", + callback=check_auth_provider_creds, + ), + auth_auto_provision: bool = typer.Option( + False, + ), + repository: str = typer.Option( + None, + help=f"options: {enum_to_list(GitRepoEnum)}", + ), + repository_auto_provision: bool = typer.Option( + False, + ), + ci_provider: str = typer.Option( + None, + help=f"options: {enum_to_list(CiEnum)}", + ), + terraform_state: str = typer.Option( + "remote", help=f"options: {enum_to_list(TerraformStateEnum)}" + ), + kubernetes_version: str = typer.Option( + "latest", + ), + ssl_cert_email: str = typer.Option( + None, + callback=check_ssl_cert_email, + ), + disable_prompt: bool = typer.Option( + False, + is_eager=True, + ), +): + """ + Create and initialize your [purple]nebari-config.yaml[/purple] file. + + This command will create and initialize your [purple]nebari-config.yaml[/purple] :sparkles: + + This file contains all your Nebari cluster configuration details and, + is used as input to later commands such as [green]nebari render[/green], [green]nebari deploy[/green], etc. + + If you're new to Nebari, we recommend you use the Guided Init wizard. + To get started simply run: + + [green]nebari init --guided-init[/green] + + """ + inputs = InitInputs() + + inputs.cloud_provider = cloud_provider + inputs.project_name = project_name + inputs.domain_name = domain_name + inputs.namespace = namespace + inputs.auth_provider = auth_provider + inputs.auth_auto_provision = auth_auto_provision + inputs.repository = repository + inputs.repository_auto_provision = repository_auto_provision + inputs.ci_provider = ci_provider + inputs.terraform_state = terraform_state + inputs.kubernetes_version = kubernetes_version + inputs.ssl_cert_email = ssl_cert_email + inputs.disable_prompt = disable_prompt + + handle_init(inputs) diff --git a/src/_nebari/subcommands/keycloak.py b/src/_nebari/subcommands/keycloak.py new file mode 100644 index 000000000..f1357191e --- /dev/null +++ b/src/_nebari/subcommands/keycloak.py @@ -0,0 +1,86 @@ +import json +from pathlib import Path +from typing import Tuple + +import typer + +from _nebari.keycloak import do_keycloak, export_keycloak_users +from nebari.hookspecs import hookimpl + + +@hookimpl +def nebari_subcommand(cli: typer.Typer): + app_keycloak = typer.Typer( + add_completion=False, + no_args_is_help=True, + rich_markup_mode="rich", + context_settings={"help_option_names": ["-h", "--help"]}, + ) + + cli.add_typer( + app_keycloak, + name="keycloak", + help="Interact with the Nebari Keycloak identity and access management tool.", + rich_help_panel="Additional Commands", + ) + + @app_keycloak.command(name="adduser") + def add_user( + add_users: Tuple[str, str] = typer.Option( + ..., "--user", help="Provide both: " + ), + config_filename: str = typer.Option( + ..., + "-c", + "--config", + help="nebari configuration file path", + ), + ): + """Add a user to Keycloak. User will be automatically added to the [italic]analyst[/italic] group.""" + if isinstance(config_filename, str): + config_filename = Path(config_filename) + + args = ["adduser", add_users[0], add_users[1]] + + do_keycloak(config_filename, *args) + + + @app_keycloak.command(name="listusers") + def list_users( + config_filename: str = typer.Option( + ..., + "-c", + "--config", + help="nebari configuration file path", + ) + ): + """List the users in Keycloak.""" + if isinstance(config_filename, str): + config_filename = Path(config_filename) + + args = ["listusers"] + + do_keycloak(config_filename, *args) + + + @app_keycloak.command(name="export-users") + def export_users( + config_filename: str = typer.Option( + ..., + "-c", + "--config", + help="nebari configuration file path", + ), + realm: str = typer.Option( + "nebari", + "--realm", + help="realm from which users are to be exported", + ), + ): + """Export the users in Keycloak.""" + if isinstance(config_filename, str): + config_filename = Path(config_filename) + + r = export_keycloak_users(config_filename, realm=realm) + + print(json.dumps(r, indent=4)) diff --git a/src/_nebari/subcommands/render.py b/src/_nebari/subcommands/render.py new file mode 100644 index 000000000..16c613ddf --- /dev/null +++ b/src/_nebari/subcommands/render.py @@ -0,0 +1,46 @@ +import typer +import pathlib + +from nebari.hookspecs import hookimpl +from nebari import schema +from _nebari.utils import load_yaml +from _nebari.render import render_template + + +@hookimpl +def nebari_subcommand(cli: typer.Typer): + @cli.command(rich_help_panel="Additional Commands") + def render( + output: str = typer.Option( + "./", + "-o", + "--output", + help="output directory", + ), + config: str = typer.Option( + ..., + "-c", + "--config", + help="nebari configuration yaml file path", + ), + dry_run: bool = typer.Option( + False, + "--dry-run", + help="simulate rendering files without actually writing or updating any files", + ), + ): + """ + Dynamically render the Terraform scripts and other files from your [purple]nebari-config.yaml[/purple] file. + """ + config_filename = Path(config) + + if not config_filename.is_file(): + raise ValueError( + f"passed in configuration filename={config_filename} must exist" + ) + + config_yaml = load_yaml(config_filename) + + schema.verify(config_yaml) + + render_template(output, config, dry_run=dry_run) diff --git a/src/_nebari/subcommands/support.py b/src/_nebari/subcommands/support.py new file mode 100644 index 000000000..ae90dd9c7 --- /dev/null +++ b/src/_nebari/subcommands/support.py @@ -0,0 +1,93 @@ +import pathlib + +import typer + +from nebari.hookspecs import hookimpl +from nebari import schema +from _nebari.utils import load_yaml +from _nebari.upgrade import do_upgrade + + +def get_config_namespace(config): + config_filename = Path(config) + if not config_filename.is_file(): + raise ValueError( + f"passed in configuration filename={config_filename} must exist" + ) + + with config_filename.open() as f: + config = yaml.safe_load(f.read()) + + return config["namespace"] + + +@hookimpl +def nebari_subcommand(cli: typer.Typer): + @cli.command(rich_help_panel="Additional Commands") + def support( + config_filename: str = typer.Option( + ..., + "-c", + "--config", + help="nebari configuration file path", + ), + output: str = typer.Option( + "./nebari-support-logs.zip", + "-o", + "--output", + help="output filename", + ), + ): + """ + Support tool to write all Kubernetes logs locally and compress them into a zip file. + + The Nebari team recommends k9s to manage and inspect the state of the cluster. + However, this command occasionally helpful for debugging purposes should the logs need to be shared. + """ + kube_config.load_kube_config() + + v1 = client.CoreV1Api() + + namespace = get_config_namespace(config=config_filename) + + pods = v1.list_namespaced_pod(namespace=namespace) + + for pod in pods.items: + Path(f"./log/{namespace}").mkdir(parents=True, exist_ok=True) + path = Path(f"./log/{namespace}/{pod.metadata.name}.txt") + with path.open(mode="wt") as file: + try: + file.write( + "%s\t%s\t%s\n" + % ( + pod.status.pod_ip, + namespace, + pod.metadata.name, + ) + ) + + # some pods are running multiple containers + containers = [ + _.name if len(pod.spec.containers) > 1 else None + for _ in pod.spec.containers + ] + + for container in containers: + if container is not None: + file.write(f"Container: {container}\n") + file.write( + v1.read_namespaced_pod_log( + name=pod.metadata.name, + namespace=namespace, + container=container, + ) + ) + + except client.exceptions.ApiException as e: + file.write("%s not available" % pod.metadata.name) + raise e + + with ZipFile(output, "w") as zip: + for file in list(Path(f"./log/{namespace}").glob("*.txt")): + print(file) + zip.write(file) diff --git a/src/_nebari/subcommands/upgrade.py b/src/_nebari/subcommands/upgrade.py new file mode 100644 index 000000000..73fe1e26c --- /dev/null +++ b/src/_nebari/subcommands/upgrade.py @@ -0,0 +1,40 @@ +import pathlib + +import typer + +from nebari.hookspecs import hookimpl +from nebari import schema +from _nebari.utils import load_yaml +from _nebari.upgrade import do_upgrade + + +@hookimpl +def nebari_subcommand(cli: typer.Typer): + @cli.command(rich_help_panel="Additional Commands") + def upgrade( + config: str = typer.Option( + ..., + "-c", + "--config", + help="nebari configuration file path", + ), + attempt_fixes: bool = typer.Option( + False, + "--attempt-fixes", + help="Attempt to fix the config for any incompatibilities between your old and new Nebari versions.", + ), + ): + """ + Upgrade your [purple]nebari-config.yaml[/purple]. + + Upgrade your [purple]nebari-config.yaml[/purple] after an nebari upgrade. If necessary, prompts users to perform manual upgrade steps required for the deploy process. + + See the project [green]RELEASE.md[/green] for details. + """ + config_filename = pathlib.Path(config) + if not config_filename.is_file(): + raise ValueError( + f"passed in configuration filename={config_filename} must exist" + ) + + do_upgrade(config_filename, attempt_fixes=attempt_fixes) diff --git a/src/_nebari/subcommands/validate.py b/src/_nebari/subcommands/validate.py new file mode 100644 index 000000000..28fecb79f --- /dev/null +++ b/src/_nebari/subcommands/validate.py @@ -0,0 +1,40 @@ +import typer +import pathlib + +from nebari.hookspecs import hookimpl +from nebari import schema +from _nebari.utils import load_yaml + + +@hookimpl +def nebari_subcommand(cli: typer.Typer): + @cli.command(rich_help_panel="Additional Commands") + def validate( + config: str = typer.Option( + ..., + "--config", + "-c", + help="nebari configuration yaml file path, please pass in as -c/--config flag", + ), + enable_commenting: bool = typer.Option( + False, "--enable-commenting", help="Toggle PR commenting on GitHub Actions" + ), + ): + """ + Validate the values in the [purple]nebari-config.yaml[/purple] file are acceptable. + """ + config_filename = pathlib.Path(config) + if not config_filename.is_file(): + raise ValueError( + f"Passed in configuration filename={config_filename} must exist." + ) + + config = load_yaml(config_filename) + + if enable_commenting: + # for PR's only + # comment_on_pr(config) + pass + else: + schema.verify(config) + print("[bold purple]Successfully validated configuration.[/bold purple]") diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index 90938233a..197b7e7b5 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -4,6 +4,7 @@ from typing import Any, Dict from pluggy import HookimplMarker, HookspecMarker +import typer hookspec = HookspecMarker("nebari") hookimpl = HookimplMarker("nebari") @@ -40,7 +41,10 @@ def destroy( @hookspec -def nebari_stage( - install_directory: pathlib.Path, config: schema.Main -) -> Iterable[NebariStage]: +def nebari_stage() -> Iterable[NebariStage]: """Registers stages in nebari""" + + +@hookspec +def nebari_subcommand(cli: typer.Typer): + """Register Typer subcommand in nebari""" diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index cb2f198a3..41fa2a779 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -6,6 +6,7 @@ from nebari import hookspecs DEFAULT_PLUGINS = [ + # stages "_nebari.stages.terraform_state", "_nebari.stages.infrastructure", "_nebari.stages.kubernetes_initialize", @@ -14,6 +15,16 @@ "_nebari.stages.kubernetes_keycloak_configuration", "_nebari.stages.kubernetes_services", "_nebari.stages.nebari_tf_extensions", + + # subcommands + "_nebari.subcommands.dev", + # "_nebari.subcommands.deploy", + # "_nebari.subcommands.destroy", + "_nebari.subcommands.keycloak", + "_nebari.subcommands.render", + "_nebari.subcommands.support", + # "_nebari.subcommands.upgrade", + "_nebari.subcommands.validate", ] pm = pluggy.PluginManager("nebari") From 154ed538fab10d57378910ede8d8885e2496962c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Jun 2023 01:38:14 +0000 Subject: [PATCH 008/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/render.py | 11 +++++++---- src/_nebari/stages/base.py | 6 +++--- src/_nebari/stages/infrastructure/__init__.py | 14 ++++++++------ .../stages/kubernetes_ingress/__init__.py | 5 +---- .../stages/kubernetes_initialize/__init__.py | 5 +---- .../stages/kubernetes_keycloak/__init__.py | 5 +---- .../__init__.py | 6 +----- .../stages/kubernetes_services/__init__.py | 5 +---- .../stages/nebari_tf_extensions/__init__.py | 6 +----- .../stages/terraform_state/__init__.py | 19 ++++++++----------- src/_nebari/subcommands/__init__.py | 1 - src/_nebari/subcommands/deploy.py | 8 ++++---- src/_nebari/subcommands/destroy.py | 8 ++++---- src/_nebari/subcommands/dev.py | 2 +- src/_nebari/subcommands/init.py | 1 + src/_nebari/subcommands/keycloak.py | 2 -- src/_nebari/subcommands/render.py | 7 +++---- src/_nebari/subcommands/support.py | 4 ---- src/_nebari/subcommands/upgrade.py | 4 +--- src/_nebari/subcommands/validate.py | 7 ++++--- src/nebari/hookspecs.py | 2 +- src/nebari/plugins.py | 1 - 22 files changed, 51 insertions(+), 78 deletions(-) diff --git a/src/_nebari/render.py b/src/_nebari/render.py index 568b784da..1a11db601 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -52,10 +52,13 @@ def render_template(output_directory, config_filename, dry_run=False): # corresponding env var. set_env_vars_in_config(config) - stages = [_( - output_directory=output_directory, - config=config, - ) for _ in get_available_stages()] + stages = [ + _( + output_directory=output_directory, + config=config, + ) + for _ in get_available_stages() + ] contents = render_contents(stages, config) diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index db37f6321..d95b9f4c8 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -1,12 +1,11 @@ import contextlib -import pathlib import inspect import itertools +import pathlib from typing import Any, Dict, List, Tuple from _nebari.provider import terraform from _nebari.stages.tf_objects import NebariTerraformState -from nebari import schema from nebari.hookspecs import NebariStage @@ -17,7 +16,7 @@ def template_directory(self): @property def stage_prefix(self): - return pathlib.Path('stages') / self.name + return pathlib.Path("stages") / self.name def state_imports(self) -> List[Tuple[str, str]]: return [] @@ -80,6 +79,7 @@ def destroy( def get_available_stages(): from nebari.plugins import pm + stages = itertools.chain.from_iterable(pm.hook.nebari_stage()) # order stages by priority diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 93c8752e0..883a91805 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -1,7 +1,7 @@ import contextlib +import inspect import pathlib import sys -import inspect from typing import Any, Dict, List from _nebari.stages.base import NebariTerraformStage @@ -44,11 +44,15 @@ class KubernetesInfrastructureStage(NebariTerraformStage): @property def template_directory(self): - return pathlib.Path(inspect.getfile(self.__class__).parent) / "template" / self.config.provider.value + return ( + pathlib.Path(inspect.getfile(self.__class__).parent) + / "template" + / self.config.provider.value + ) @property def stage_prefix(self): - return pathlib.Path('stages') / self.name / self.config.provider.value + return pathlib.Path("stages") / self.name / self.config.provider.value def tf_objects(self) -> List[Dict]: if self.config.provider == schema.ProviderEnum.gcp: @@ -210,6 +214,4 @@ def destroy( @hookimpl def nebari_stage() -> List[NebariStage]: - return [ - KubernetesInfrastructureStage - ] + return [KubernetesInfrastructureStage] diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index 2ea119b75..64d9fa4c7 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -1,4 +1,3 @@ -import pathlib import socket import sys from typing import Any, Dict, List @@ -187,6 +186,4 @@ def _attempt_tcp_connect( @hookimpl def nebari_stage() -> List[NebariStage]: - return [ - KubernetesIngressStage - ] + return [KubernetesIngressStage] diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py index e09effca6..41eff6d76 100644 --- a/src/_nebari/stages/kubernetes_initialize/__init__.py +++ b/src/_nebari/stages/kubernetes_initialize/__init__.py @@ -1,4 +1,3 @@ -import pathlib import sys from typing import Any, Dict, List @@ -84,6 +83,4 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]): @hookimpl def nebari_stage() -> List[NebariStage]: - return [ - KubernetesInitializeStage - ] + return [KubernetesInitializeStage] diff --git a/src/_nebari/stages/kubernetes_keycloak/__init__.py b/src/_nebari/stages/kubernetes_keycloak/__init__.py index 182c6dafe..11cb6f50f 100644 --- a/src/_nebari/stages/kubernetes_keycloak/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak/__init__.py @@ -1,6 +1,5 @@ import contextlib import json -import pathlib import sys from typing import Any, Dict, List @@ -159,6 +158,4 @@ def destroy( @hookimpl def nebari_stage() -> List[NebariStage]: - return [ - KubernetesKeycloakStage - ] + return [KubernetesKeycloakStage] diff --git a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py index 410bf96d6..68651568e 100644 --- a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py @@ -1,10 +1,8 @@ -import pathlib import sys from typing import Any, Dict, List from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import NebariTerraformState -from nebari import schema from nebari.hookspecs import NebariStage, hookimpl @@ -100,6 +98,4 @@ def _attempt_keycloak_connection( @hookimpl def nebari_stage() -> List[NebariStage]: - return [ - KubernetesKeycloakConfigurationStage - ] + return [KubernetesKeycloakConfigurationStage] diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index 657c5f5d1..03ef04121 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -1,4 +1,3 @@ -import pathlib import sys from typing import Any, Dict, List from urllib.parse import urlencode @@ -192,6 +191,4 @@ def _attempt_connect_url( @hookimpl def nebari_stage() -> List[NebariStage]: - return [ - KubernetesServicesStage - ] + return [KubernetesServicesStage] diff --git a/src/_nebari/stages/nebari_tf_extensions/__init__.py b/src/_nebari/stages/nebari_tf_extensions/__init__.py index 054cc43d2..47d37a71e 100644 --- a/src/_nebari/stages/nebari_tf_extensions/__init__.py +++ b/src/_nebari/stages/nebari_tf_extensions/__init__.py @@ -1,4 +1,3 @@ -import pathlib from typing import Any, Dict, List from _nebari.stages.base import NebariTerraformStage @@ -7,7 +6,6 @@ NebariKubernetesProvider, NebariTerraformState, ) -from nebari import schema from nebari.hookspecs import NebariStage, hookimpl @@ -40,6 +38,4 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): @hookimpl def nebari_stage() -> List[NebariStage]: - return [ - NebariTFExtensionsStage - ] + return [NebariTFExtensionsStage] diff --git a/src/_nebari/stages/terraform_state/__init__.py b/src/_nebari/stages/terraform_state/__init__.py index f4f71736c..57ef2f176 100644 --- a/src/_nebari/stages/terraform_state/__init__.py +++ b/src/_nebari/stages/terraform_state/__init__.py @@ -1,14 +1,9 @@ +import inspect import os import pathlib -import inspect from typing import Any, Dict, List, Tuple from _nebari.stages.base import NebariTerraformStage -from _nebari.stages.tf_objects import ( - NebariHelmProvider, - NebariKubernetesProvider, - NebariTerraformState, -) from nebari import schema from nebari.hookspecs import NebariStage, hookimpl @@ -19,11 +14,15 @@ class TerraformStateStage(NebariTerraformStage): @property def template_directory(self): - return pathlib.Path(inspect.getfile(self.__class__).parent) / "template" / self.config.provider.value + return ( + pathlib.Path(inspect.getfile(self.__class__).parent) + / "template" + / self.config.provider.value + ) @property def stage_prefix(self): - return pathlib.Path('stages') / self.name / self.config.provider.value + return pathlib.Path("stages") / self.name / self.config.provider.value def state_imports(self) -> List[Tuple[str, str]]: if self.config.provider == schema.ProviderEnum.do: @@ -110,6 +109,4 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): @hookimpl def nebari_stage() -> List[NebariStage]: - return [ - TerraformStateStage - ] + return [TerraformStateStage] diff --git a/src/_nebari/subcommands/__init__.py b/src/_nebari/subcommands/__init__.py index 273cfd1ba..15df491e0 100644 --- a/src/_nebari/subcommands/__init__.py +++ b/src/_nebari/subcommands/__init__.py @@ -1,7 +1,6 @@ from typing import Optional import typer -from click import Context from typer.core import TyperGroup from _nebari.version import __version__ diff --git a/src/_nebari/subcommands/deploy.py b/src/_nebari/subcommands/deploy.py index 6c2d6cd85..881e0839b 100644 --- a/src/_nebari/subcommands/deploy.py +++ b/src/_nebari/subcommands/deploy.py @@ -2,11 +2,11 @@ import typer -from nebari.hookspecs import hookimpl -from nebari import schema -from _nebari.utils import load_yaml -from _nebari.render import render_template from _nebari.deploy import deploy_configuration +from _nebari.render import render_template +from _nebari.utils import load_yaml +from nebari import schema +from nebari.hookspecs import hookimpl @hookimpl diff --git a/src/_nebari/subcommands/destroy.py b/src/_nebari/subcommands/destroy.py index 19fa7f519..ee27d1044 100644 --- a/src/_nebari/subcommands/destroy.py +++ b/src/_nebari/subcommands/destroy.py @@ -2,11 +2,11 @@ import typer -from nebari.hookspecs import hookimpl -from nebari import schema -from _nebari.utils import load_yaml -from _nebari.render import render_template from _nebari.destroy import destroy_configuration +from _nebari.render import render_template +from _nebari.utils import load_yaml +from nebari import schema +from nebari.hookspecs import hookimpl @hookimpl diff --git a/src/_nebari/subcommands/dev.py b/src/_nebari/subcommands/dev.py index c66529f83..215dcaa4e 100644 --- a/src/_nebari/subcommands/dev.py +++ b/src/_nebari/subcommands/dev.py @@ -3,8 +3,8 @@ import typer -from nebari.hookspecs import hookimpl from _nebari.keycloak import keycloak_rest_api_call +from nebari.hookspecs import hookimpl @hookimpl diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index d3f18e174..fb464a9a6 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -8,6 +8,7 @@ handle_init, ) + @app.command() def init( cloud_provider: str = typer.Argument( diff --git a/src/_nebari/subcommands/keycloak.py b/src/_nebari/subcommands/keycloak.py index f1357191e..6f33c6788 100644 --- a/src/_nebari/subcommands/keycloak.py +++ b/src/_nebari/subcommands/keycloak.py @@ -44,7 +44,6 @@ def add_user( do_keycloak(config_filename, *args) - @app_keycloak.command(name="listusers") def list_users( config_filename: str = typer.Option( @@ -62,7 +61,6 @@ def list_users( do_keycloak(config_filename, *args) - @app_keycloak.command(name="export-users") def export_users( config_filename: str = typer.Option( diff --git a/src/_nebari/subcommands/render.py b/src/_nebari/subcommands/render.py index 16c613ddf..dede798b9 100644 --- a/src/_nebari/subcommands/render.py +++ b/src/_nebari/subcommands/render.py @@ -1,10 +1,9 @@ import typer -import pathlib -from nebari.hookspecs import hookimpl -from nebari import schema -from _nebari.utils import load_yaml from _nebari.render import render_template +from _nebari.utils import load_yaml +from nebari import schema +from nebari.hookspecs import hookimpl @hookimpl diff --git a/src/_nebari/subcommands/support.py b/src/_nebari/subcommands/support.py index ae90dd9c7..89cddccba 100644 --- a/src/_nebari/subcommands/support.py +++ b/src/_nebari/subcommands/support.py @@ -1,11 +1,7 @@ -import pathlib import typer from nebari.hookspecs import hookimpl -from nebari import schema -from _nebari.utils import load_yaml -from _nebari.upgrade import do_upgrade def get_config_namespace(config): diff --git a/src/_nebari/subcommands/upgrade.py b/src/_nebari/subcommands/upgrade.py index 73fe1e26c..b1c173af4 100644 --- a/src/_nebari/subcommands/upgrade.py +++ b/src/_nebari/subcommands/upgrade.py @@ -2,10 +2,8 @@ import typer -from nebari.hookspecs import hookimpl -from nebari import schema -from _nebari.utils import load_yaml from _nebari.upgrade import do_upgrade +from nebari.hookspecs import hookimpl @hookimpl diff --git a/src/_nebari/subcommands/validate.py b/src/_nebari/subcommands/validate.py index 28fecb79f..d3883b54f 100644 --- a/src/_nebari/subcommands/validate.py +++ b/src/_nebari/subcommands/validate.py @@ -1,9 +1,10 @@ -import typer import pathlib -from nebari.hookspecs import hookimpl -from nebari import schema +import typer + from _nebari.utils import load_yaml +from nebari import schema +from nebari.hookspecs import hookimpl @hookimpl diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index 197b7e7b5..897a7da2e 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -3,8 +3,8 @@ from collections.abc import Iterable from typing import Any, Dict -from pluggy import HookimplMarker, HookspecMarker import typer +from pluggy import HookimplMarker, HookspecMarker hookspec = HookspecMarker("nebari") hookimpl = HookimplMarker("nebari") diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index 41fa2a779..a2b9c24ac 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -15,7 +15,6 @@ "_nebari.stages.kubernetes_keycloak_configuration", "_nebari.stages.kubernetes_services", "_nebari.stages.nebari_tf_extensions", - # subcommands "_nebari.subcommands.dev", # "_nebari.subcommands.deploy", From 813f2316a05c012d6ae982c2419cae7f269ab38b Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 14 Jun 2023 23:30:42 -0400 Subject: [PATCH 009/147] Working render! --- src/_nebari/provider/cicd/github.py | 28 ++- src/_nebari/provider/cicd/gitlab.py | 16 +- src/_nebari/render.py | 181 ++++-------------- src/_nebari/stages/base.py | 18 +- src/_nebari/stages/bootstrap/__init__.py | 81 ++++++++ src/_nebari/stages/infrastructure/__init__.py | 2 +- .../stages/terraform_state/__init__.py | 2 +- src/_nebari/stages/tf_objects.py | 87 +++++---- src/_nebari/subcommands/render.py | 3 +- src/_nebari/subcommands/validate.py | 1 + src/_nebari/utils.py | 4 +- src/nebari/hookspecs.py | 8 +- src/nebari/plugins.py | 1 + 13 files changed, 196 insertions(+), 236 deletions(-) create mode 100644 src/_nebari/stages/bootstrap/__init__.py diff --git a/src/_nebari/provider/cicd/github.py b/src/_nebari/provider/cicd/github.py index 84ab389b5..b096930ff 100644 --- a/src/_nebari/provider/cicd/github.py +++ b/src/_nebari/provider/cicd/github.py @@ -105,29 +105,29 @@ def gha_env_vars(config): env_vars["NEBARI_GH_BRANCH"] = "${{ secrets.NEBARI_GH_BRANCH }}" # This assumes that the user is using the omitting sensitive values configuration for the token. - if config.get("prefect", {}).get("enabled", False): + if config.prefect.enabled: env_vars[ "NEBARI_SECRET_prefect_token" ] = "${{ secrets.NEBARI_SECRET_PREFECT_TOKEN }}" - if config["provider"] == "aws": + if config.provider == schema.ProviderEnum.aws: env_vars["AWS_ACCESS_KEY_ID"] = "${{ secrets.AWS_ACCESS_KEY_ID }}" env_vars["AWS_SECRET_ACCESS_KEY"] = "${{ secrets.AWS_SECRET_ACCESS_KEY }}" env_vars["AWS_DEFAULT_REGION"] = "${{ secrets.AWS_DEFAULT_REGION }}" - elif config["provider"] == "azure": + elif config.provider == schema.ProviderEnum.azure: env_vars["ARM_CLIENT_ID"] = "${{ secrets.ARM_CLIENT_ID }}" env_vars["ARM_CLIENT_SECRET"] = "${{ secrets.ARM_CLIENT_SECRET }}" env_vars["ARM_SUBSCRIPTION_ID"] = "${{ secrets.ARM_SUBSCRIPTION_ID }}" env_vars["ARM_TENANT_ID"] = "${{ secrets.ARM_TENANT_ID }}" - elif config["provider"] == "do": + elif config.provider == schema.ProviderEnum.do: env_vars["AWS_ACCESS_KEY_ID"] = "${{ secrets.AWS_ACCESS_KEY_ID }}" env_vars["AWS_SECRET_ACCESS_KEY"] = "${{ secrets.AWS_SECRET_ACCESS_KEY }}" env_vars["SPACES_ACCESS_KEY_ID"] = "${{ secrets.SPACES_ACCESS_KEY_ID }}" env_vars["SPACES_SECRET_ACCESS_KEY"] = "${{ secrets.SPACES_SECRET_ACCESS_KEY }}" env_vars["DIGITALOCEAN_TOKEN"] = "${{ secrets.DIGITALOCEAN_TOKEN }}" - elif config["provider"] == "gcp": + elif config.provider == schema.ProviderEnum.gcp: env_vars["GOOGLE_CREDENTIALS"] = "${{ secrets.GOOGLE_CREDENTIALS }}" - elif config["provider"] in ["local", "existing"]: + elif config.provider in [schema.ProviderEnum.local, schema.ProviderEnum.existing]: # create mechanism to allow for extra env vars? pass else: @@ -231,16 +231,13 @@ def install_nebari_step(nebari_version): def gen_nebari_ops(config): env_vars = gha_env_vars(config) - branch = config["ci_cd"]["branch"] - commit_render = config["ci_cd"].get("commit_render", True) - nebari_version = config["nebari_version"] - push = GHA_on_extras(branches=[branch], paths=["nebari-config.yaml"]) + push = GHA_on_extras(branches=[config.ci_cd.branch], paths=["nebari-config.yaml"]) on = GHA_on(__root__={"push": push}) step1 = checkout_image_step() step2 = setup_python_step() - step3 = install_nebari_step(nebari_version) + step3 = install_nebari_step(config.nebari_version) gha_steps = [step1, step2, step3] for step in config["ci_cd"].get("before_script", []): @@ -267,7 +264,7 @@ def gen_nebari_ops(config): ) }, ) - if commit_render: + if config.ci_cd.commit_render: gha_steps.append(step5) for step in config["ci_cd"].get("after_script", []): @@ -300,15 +297,12 @@ def gen_nebari_linter(config): else: env_vars = None - branch = config["ci_cd"]["branch"] - nebari_version = config["nebari_version"] - - pull_request = GHA_on_extras(branches=[branch], paths=["nebari-config.yaml"]) + pull_request = GHA_on_extras(branches=[config.ci_cd.branch], paths=["nebari-config.yaml"]) on = GHA_on(__root__={"pull_request": pull_request}) step1 = checkout_image_step() step2 = setup_python_step() - step3 = install_nebari_step(nebari_version) + step3 = install_nebari_step(config.nebari_version) step4_envs = { "PR_NUMBER": GHA_job_steps_extras(__root__="${{ github.event.number }}"), diff --git a/src/_nebari/provider/cicd/gitlab.py b/src/_nebari/provider/cicd/gitlab.py index d5b6726ba..f7f3b1201 100644 --- a/src/_nebari/provider/cicd/gitlab.py +++ b/src/_nebari/provider/cicd/gitlab.py @@ -38,19 +38,13 @@ class GLCI(BaseModel): def gen_gitlab_ci(config): - branch = config["ci_cd"]["branch"] - commit_render = config["ci_cd"].get("commit_render", True) - before_script = config["ci_cd"].get("before_script") - after_script = config["ci_cd"].get("after_script") - pip_install = pip_install_nebari(config["nebari_version"]) - render_vars = { "COMMIT_MSG": "nebari-config.yaml automated commit: {{ '$CI_COMMIT_SHA' }}", } script = [ - f"git checkout {branch}", - f"{pip_install}", + f"git checkout {config.ci_cd.branch}", + pip_install_nebari(config.nebari_version), "nebari deploy --config nebari-config.yaml --disable-prompt --skip-remote-state-provision", ] @@ -62,7 +56,7 @@ def gen_gitlab_ci(config): f"git push origin {branch})", ] - if commit_render: + if config.ci_cd.commit_render: script += commit_render_script rules = [ @@ -75,8 +69,8 @@ def gen_gitlab_ci(config): render_nebari = GLCI_job( image=f"python:{LATEST_SUPPORTED_PYTHON_VERSION}", variables=render_vars, - before_script=before_script, - after_script=after_script, + before_script=config.ci_cd.before_script, + after_script=config.ci_cd.after_script, script=script, rules=rules, ) diff --git a/src/_nebari/render.py b/src/_nebari/render.py index 1a11db601..fb5880685 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -14,23 +14,14 @@ import _nebari from _nebari.deprecate import DEPRECATED_FILE_PATHS -from _nebari.provider.cicd.github import gen_nebari_linter, gen_nebari_ops -from _nebari.provider.cicd.gitlab import gen_gitlab_ci -from _nebari.utils import is_relative_to from _nebari.stages.base import get_available_stages -from nebari.hookspecs import NebariStage +from nebari import schema def render_template(output_directory, config_filename, dry_run=False): - # get directory for nebari templates - template_directory = Path(_nebari.__file__).parent / "template" + output_directory = pathlib.Path(output_directory).resolve() - # would be nice to remove assumption that input directory - # is in local filesystem and a directory - if not template_directory.is_dir(): - raise ValueError(f"template directory={template_directory} is not a directory") - - if output_directory == Path.home(): + if output_directory == str(pathlib.Path.home()): print("ERROR: Deploying Nebari in home directory is not advised!") sys.exit(1) @@ -52,38 +43,15 @@ def render_template(output_directory, config_filename, dry_run=False): # corresponding env var. set_env_vars_in_config(config) - stages = [ - _( - output_directory=output_directory, - config=config, - ) - for _ in get_available_stages() - ] - - contents = render_contents(stages, config) - - directories = [ - f"stages/02-infrastructure/{config['provider']}", - "stages/03-kubernetes-initialize", - "stages/04-kubernetes-ingress", - "stages/05-kubernetes-keycloak", - "stages/06-kubernetes-keycloak-configuration", - "stages/07-kubernetes-services", - "stages/08-nebari-tf-extensions", - ] - if ( - config["provider"] not in {"existing", "local"} - and config["terraform_state"]["type"] == "remote" - ): - directories.append(f"stages/01-terraform-state/{config['provider']}") + contents = {} + config = schema.Main.parse_obj(config) + for stage in get_available_stages(): + contents.update(stage(output_directory=output_directory, config=config).render()) + + print(contents.keys()) - source_dirs = [template_directory / Path(directory) for directory in directories] - output_dirs = [output_directory / Path(directory) for directory in directories] new, untracked, updated, deleted = inspect_files( - source_dirs, - output_dirs, - source_base_dir=template_directory, - output_base_dir=output_directory, + output_base_dir=str(output_directory), ignore_filenames=[ "terraform.tfstate", ".terraform.lock.hcl", @@ -125,15 +93,15 @@ def render_template(output_directory, config_filename, dry_run=False): print("dry-run enabled no files will be created, updated, or deleted") else: for filename in new | updated: - input_filename = template_directory / filename - output_filename = output_directory / filename - output_filename.parent.mkdir(parents=True, exist_ok=True) + output_filename = os.path.join(str(output_directory), filename) + os.makedirs(os.path.dirname(output_filename), exist_ok=True) - if input_filename.exists(): - shutil.copy(input_filename, output_filename) - else: + if isinstance(contents[filename], str): with open(output_filename, "w") as f: f.write(contents[filename]) + else: + with open(output_filename, "wb") as f: + f.write(contents[filename]) for path in deleted: abs_path = (output_directory / path).resolve() @@ -149,83 +117,9 @@ def render_template(output_directory, config_filename, dry_run=False): shutil.rmtree(abs_path) -def render_contents(stages: List[NebariStage], config: Dict): - """Dynamically generated contents from _nebari configuration.""" - contents = {} - for stage in stages: - contents.update(stage.render()) - - if config.get("ci_cd"): - for fn, workflow in gen_cicd(config).items(): - workflow_json = workflow.json( - indent=2, - by_alias=True, - exclude_unset=True, - exclude_defaults=True, - ) - workflow_yaml = yaml.dump( - json.loads(workflow_json), sort_keys=False, indent=2 - ) - contents.update({fn: workflow_yaml}) - - contents.update(gen_gitignore()) - - return contents - - -def gen_gitignore(): - """ - Generate `.gitignore` file. - Add files as needed. - """ - from inspect import cleandoc - - files_to_ignore = """ - # ignore terraform state - .terraform - terraform.tfstate - terraform.tfstate.backup - .terraform.tfstate.lock.info - - # python - __pycache__ - """ - return {Path(".gitignore"): cleandoc(files_to_ignore)} - - -def gen_cicd(config): - """ - Use cicd schema to generate workflow files based on the - `ci_cd` key in the `config`. - - For more detail on schema: - GiHub-Actions - nebari/providers/cicd/github.py - GitLab-CI - nebari/providers/cicd/gitlab.py - """ - cicd_files = {} - cicd_provider = config["ci_cd"]["type"] - - if cicd_provider == "github-actions": - gha_dir = Path(".github") / "workflows" - cicd_files[gha_dir / "nebari-ops.yaml"] = gen_nebari_ops(config) - cicd_files[gha_dir / "nebari-linter.yaml"] = gen_nebari_linter(config) - - elif cicd_provider == "gitlab-ci": - cicd_files[Path(".gitlab-ci.yml")] = gen_gitlab_ci(config) - - else: - raise ValueError( - f"The ci_cd provider, {cicd_provider}, is not supported. Supported providers include: `github-actions`, `gitlab-ci`." - ) - - return cicd_files - - def inspect_files( - source_dirs: Path, - output_dirs: Path, - source_base_dir: Path, - output_base_dir: Path, + output_base_dir: str, +>>>>>>> c9abe710 (Working render!) ignore_filenames: List[str] = None, ignore_directories: List[str] = None, deleted_paths: List[Path] = None, @@ -234,10 +128,7 @@ def inspect_files( """Return created, updated and untracked files by computing a checksum over the provided directory. Args: - source_dirs (Path): The source dir used as base for comparison - output_dirs (Path): The destination dir which will be matched with - source_base_dir (Path): Relative base path to source directory - output_base_dir (Path): Relative base path to output directory + output_base_dir (str): Relative base path to output directory ignore_filenames (list[str]): Filenames to ignore while comparing for changes ignore_directories (list[str]): Directories to ignore while comparing for changes deleted_paths (list[Path]): Paths that if exist in output directory should be deleted @@ -257,20 +148,18 @@ def list_files( if not path.is_file(): continue - if path.name in ignore_filenames: - continue - - if any( - d in ignore_directories for d in path.relative_to(directory).parts[:-1] - ): - continue - - yield path + for filename in contents: + if isinstance(contents[filename], str): + source_files[filename] = hashlib.sha256( + contents[filename].encode("utf8") + ).hexdigest() + else: + source_files[filename] = hashlib.sha256( + contents[filename] + ).hexdigest() - for filename, content in contents.items(): - source_files[filename] = hashlib.sha256(content.encode("utf8")).hexdigest() - output_filename = output_base_dir / filename - if output_filename.is_file(): + output_filename = os.path.join(output_base_dir, filename) + if os.path.isfile(output_filename): output_files[filename] = hash_file(filename) deleted_files = set() @@ -279,13 +168,9 @@ def list_files( if absolute_path.exists(): deleted_files.add(path) - for source_dir, output_dir in zip(source_dirs, output_dirs): - for filename in list_files(source_dir, ignore_filenames, ignore_directories): - relative_path = filename.relative_to(source_base_dir) - source_files[relative_path] = hash_file(filename) - - for filename in list_files(output_dir, ignore_filenames, ignore_directories): - relative_path = filename.relative_to(output_base_dir) + for filename in list_files(output_base_dir, ignore_filenames, ignore_directories): + relative_path = os.path.relpath(filename, output_base_dir) + if os.path.isfile(filename): output_files[relative_path] = hash_file(filename) new_files = source_files.keys() - output_files.keys() diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index d95b9f4c8..755a6c57d 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -1,3 +1,4 @@ +import os import contextlib import inspect import itertools @@ -12,7 +13,7 @@ class NebariTerraformStage(NebariStage): @property def template_directory(self): - return pathlib.Path(inspect.getfile(self.__class__).parent) / "template" + return pathlib.Path(inspect.getfile(self.__class__)).parent / "template" @property def stage_prefix(self): @@ -22,18 +23,19 @@ def state_imports(self) -> List[Tuple[str, str]]: return [] def tf_objects(self) -> List[Dict]: - return [NebariTerraformState(self.name, config)] + return [NebariTerraformState(self.name, self.config)] - def render(self): + def render(self) -> Dict[str, str]: contents = { str( - self.output_directory / stage_prefix / "_nebari.tf.json" + self.stage_prefix / "_nebari.tf.json" ): terraform.tf_render_objects(self.tf_objects()) } - for root, dirs, files in os.walk(self.template_directory): + for root, dirs, filenames in os.walk(self.template_directory): for filename in filenames: - contents[os.path.join(root, filename)] = open( - os.path.join(root, filename) + contents[os.path.join(self.stage_prefix, os.path.relpath(os.path.join(root, filename), self.template_directory))] = open( + os.path.join(root, filename), + "rb", ).read() return contents @@ -71,7 +73,7 @@ def destroy( ) yield status["stages/" + self.name] = _terraform_destroy( - directory=str(output_directory / stage_prefix), + directory=str(output_directory / self.stage_prefix), input_vars=self.input_vars(stage_outputs), ignore_errors=True, ) diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py new file mode 100644 index 000000000..0071bbd90 --- /dev/null +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -0,0 +1,81 @@ +from typing import Dict, List +from inspect import cleandoc + + +from _nebari.provider.cicd.github import gen_nebari_linter, gen_nebari_ops +from _nebari.provider.cicd.gitlab import gen_gitlab_ci +from nebari.hookspecs import NebariStage, hookimpl +from nebari import schema + + +def gen_gitignore(): + """ + Generate `.gitignore` file. + Add files as needed. + """ + filestoignore = """ + # ignore terraform state + .terraform + terraform.tfstate + terraform.tfstate.backup + .terraform.tfstate.lock.info + + # python + __pycache__ + """ + return {".gitignore": cleandoc(filestoignore)} + + +def gen_cicd(config): + """ + Use cicd schema to generate workflow files based on the + `ci_cd` key in the `config`. + + For more detail on schema: + GiHub-Actions - nebari/providers/cicd/github.py + GitLab-CI - nebari/providers/cicd/gitlab.py + """ + cicd_files = {} + + if config.ci_cd.type == schema.CiEnum.github_actions: + gha_dir = ".github/workflows/" + cicd_files[gha_dir + "nebari-ops.yaml"] = gen_nebari_ops(config) + cicd_files[gha_dir + "nebari-linter.yaml"] = gen_nebari_linter(config) + + elif config.ci_cd.type == schema.CiEnum.gitlab_ci: + cicd_files[".gitlab-ci.yml"] = gen_gitlab_ci(config) + + else: + raise ValueError( + f"The ci_cd provider, {config.ci_cd.type.value}, is not supported. Supported providers include: `github-actions`, `gitlab-ci`." + ) + + return cicd_files + + +class BootstrapStage(NebariStage): + name = "boostrap" + priority = 0 + + def render(self) -> Dict[str, str]: + contents = {} + if self.config.ci_cd.type != schema.CiEnum.none: + for fn, workflow in gen_cicd(self.config).items(): + workflow_json = workflow.json( + indent=2, + by_alias=True, + exclude_unset=True, + exclude_defaults=True, + ) + workflow_yaml = yaml.dump( + json.loads(workflow_json), sort_keys=False, indent=2 + ) + contents.update({fn: workflow_yaml}) + + contents.update(gen_gitignore()) + return contents + + +@hookimpl +def nebari_stage() -> List[NebariStage]: + return [BootstrapStage] diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 883a91805..974e3bc23 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -45,7 +45,7 @@ class KubernetesInfrastructureStage(NebariTerraformStage): @property def template_directory(self): return ( - pathlib.Path(inspect.getfile(self.__class__).parent) + pathlib.Path(inspect.getfile(self.__class__)).parent / "template" / self.config.provider.value ) diff --git a/src/_nebari/stages/terraform_state/__init__.py b/src/_nebari/stages/terraform_state/__init__.py index 57ef2f176..10a8adae0 100644 --- a/src/_nebari/stages/terraform_state/__init__.py +++ b/src/_nebari/stages/terraform_state/__init__.py @@ -15,7 +15,7 @@ class TerraformStateStage(NebariTerraformStage): @property def template_directory(self): return ( - pathlib.Path(inspect.getfile(self.__class__).parent) + pathlib.Path(inspect.getfile(self.__class__)).parent / "template" / self.config.provider.value ) diff --git a/src/_nebari/stages/tf_objects.py b/src/_nebari/stages/tf_objects.py index 2ecb8a56c..dd69fdc87 100644 --- a/src/_nebari/stages/tf_objects.py +++ b/src/_nebari/stages/tf_objects.py @@ -3,36 +3,37 @@ from _nebari.provider.terraform import Data, Provider, TerraformBackend from _nebari.utils import deep_merge +from nebari import schema -def NebariAWSProvider(nebari_config: Dict): - return Provider("aws", region=nebari_config["amazon_web_services"]["region"]) +def NebariAWSProvider(nebari_config: schema.Main): + return Provider("aws", region=nebari_config.amazon_web_services.region) -def NebariGCPProvider(nebari_config: Dict): +def NebariGCPProvider(nebari_config: schema.Main): return Provider( "google", - project=nebari_config["google_cloud_platform"]["project"], - region=nebari_config["google_cloud_platform"]["region"], + project=nebari_config.google_cloud_platform.project, + region=nebari_config.google_cloud_platform.region, ) -def NebariAzureProvider(nebari_config: Dict): +def NebariAzureProvider(nebari_config: schema.Main): return Provider("azurerm", features={}) -def NebariDigitalOceanProvider(nebari_config: Dict): +def NebariDigitalOceanProvider(nebari_config: schema.Main): return Provider("digitalocean") -def NebariKubernetesProvider(nebari_config: Dict): - if nebari_config["provider"] == "aws": - cluster_name = f"{nebari_config['project_name']}-{nebari_config['namespace']}" +def NebariKubernetesProvider(nebari_config: schema.Main): + if nebari_config.provider == schema.ProviderEnum.aws: + cluster_name = f"{nebari_config.project_name}-{nebari_config.namespace}" # The AWS provider needs to be added, as we are using aws related resources #1254 return deep_merge( Data("aws_eks_cluster", "default", name=cluster_name), Data("aws_eks_cluster_auth", "default", name=cluster_name), - Provider("aws", region=nebari_config["amazon_web_services"]["region"]), + Provider("aws", region=nebari_config.amazon_web_services.region), Provider( "kubernetes", experiments={"manifest_resource": True}, @@ -47,9 +48,9 @@ def NebariKubernetesProvider(nebari_config: Dict): ) -def NebariHelmProvider(nebari_config: Dict): - if nebari_config["provider"] == "aws": - cluster_name = f"{nebari_config['project_name']}-{nebari_config['namespace']}" +def NebariHelmProvider(nebari_config: schema.Main): + if nebari_config.provider == schema.ProviderEnum.aws: + cluster_name = f"{nebari_config.project_name}-{nebari_config.namespace}" return deep_merge( Data("aws_eks_cluster", "default", name=cluster_name), @@ -66,67 +67,65 @@ def NebariHelmProvider(nebari_config: Dict): return Provider("helm") -def NebariTerraformState(directory: str, nebari_config: Dict): - if nebari_config["terraform_state"]["type"] == "local": +def NebariTerraformState(directory: str, nebari_config: schema.Main): + if nebari_config.terraform_state.type == schema.TerraformStateEnum.local: return {} - elif nebari_config["terraform_state"]["type"] == "existing": + elif nebari_config.terraform_state.type == schema.TerraformStateEnum.existing: return TerraformBackend( nebari_config["terraform_state"]["backend"], **nebari_config["terraform_state"]["config"], ) - elif nebari_config["provider"] == "aws": + elif nebari_config.provider == schema.ProviderEnum.aws: return TerraformBackend( "s3", - bucket=f"{nebari_config['project_name']}-{nebari_config['namespace']}-terraform-state", - key=f"terraform/{nebari_config['project_name']}-{nebari_config['namespace']}/{directory}.tfstate", - region=nebari_config["amazon_web_services"]["region"], + bucket=f"{nebari_config.project_name}-{nebari_config.namespace}-terraform-state", + key=f"terraform/{nebari_config.project_name}-{nebari_config.namespace}/{directory}.tfstate", + region=nebari_config.amazon_web_services.region, encrypt=True, - dynamodb_table=f"{nebari_config['project_name']}-{nebari_config['namespace']}-terraform-state-lock", + dynamodb_table=f"{nebari_config.project_name}-{nebari_config.namespace}-terraform-state-lock", ) - elif nebari_config["provider"] == "gcp": + elif nebari_config.provider == schema.ProviderEnum.gcp: return TerraformBackend( "gcs", - bucket=f"{nebari_config['project_name']}-{nebari_config['namespace']}-terraform-state", - prefix=f"terraform/{nebari_config['project_name']}/{directory}", + bucket=f"{nebari_config.project_name}-{nebari_config.namespace}-terraform-state", + prefix=f"terraform/{nebari_config.project_name}/{directory}", ) - elif nebari_config["provider"] == "do": + elif nebari_config.provider == schema.ProviderEnum.do: return TerraformBackend( "s3", - endpoint=f"{nebari_config['digital_ocean']['region']}.digitaloceanspaces.com", + endpoint=f"{nebari_config.digital_ocean.region}.digitaloceanspaces.com", region="us-west-1", # fake aws region required by terraform - bucket=f"{nebari_config['project_name']}-{nebari_config['namespace']}-terraform-state", - key=f"terraform/{nebari_config['project_name']}-{nebari_config['namespace']}/{directory}.tfstate", + bucket=f"{nebari_config.project_name}-{nebari_config.namespace}-terraform-state", + key=f"terraform/{nebari_config.project_name}-{nebari_config.namespace}/{directory}.tfstate", skip_credentials_validation=True, skip_metadata_api_check=True, ) - elif nebari_config["provider"] == "azure": + elif nebari_config.provider == schema.ProviderEnum.azure: return TerraformBackend( "azurerm", - resource_group_name=f"{nebari_config['project_name']}-{nebari_config['namespace']}-state", + resource_group_name=f"{nebari_config.project_name}-{nebari_config.namespace}-state", # storage account must be globally unique - storage_account_name=f"{nebari_config['project_name']}{nebari_config['namespace']}{nebari_config['azure']['storage_account_postfix']}", - container_name=f"{nebari_config['project_name']}-{nebari_config['namespace']}-state", - key=f"terraform/{nebari_config['project_name']}-{nebari_config['namespace']}/{directory}", + storage_account_name=f"{nebari_config.project_name}{nebari_config.namespace}{nebari_config.azure.storage_account_postfix}", + container_name=f"{nebari_config.project_name}-{nebari_config.namespace}-state", + key=f"terraform/{nebari_config.project_name}-{nebari_config.namespace}/{directory}", ) - elif nebari_config["provider"] == "existing": + elif nebari_config.provider == schema.ProviderEnum.existing: optional_kwargs = {} - if "kube_context" in nebari_config["existing"]: - optional_kwargs["confix_context"] = nebari_config["existing"][ - "kube_context" - ] + if "kube_context" in nebari_config.existing: + optional_kwargs["config_context"] = nebari_config.existing.kube_context return TerraformBackend( "kubernetes", - secret_suffix=f"{nebari_config['project_name']}-{nebari_config['namespace']}-{directory}", + secret_suffix=f"{nebari_config.project_name}-{nebari_config.namespace}-{directory}", load_config_file=True, **optional_kwargs, ) - elif nebari_config["provider"] == "local": + elif nebari_config.provider == schema.ProviderEnum.local: optional_kwargs = {} - if "kube_context" in nebari_config["local"]: - optional_kwargs["confix_context"] = nebari_config["local"]["kube_context"] + if "kube_context" in nebari_config.local: + optional_kwargs["config_context"] = nebari_config.local.kube_context return TerraformBackend( "kubernetes", - secret_suffix=f"{nebari_config['project_name']}-{nebari_config['namespace']}-{directory}", + secret_suffix=f"{nebari_config.project_name}-{nebari_config.namespace}-{directory}", load_config_file=True, **optional_kwargs, ) diff --git a/src/_nebari/subcommands/render.py b/src/_nebari/subcommands/render.py index dede798b9..cb27753ee 100644 --- a/src/_nebari/subcommands/render.py +++ b/src/_nebari/subcommands/render.py @@ -1,3 +1,4 @@ +import pathlib import typer from _nebari.render import render_template @@ -31,7 +32,7 @@ def render( """ Dynamically render the Terraform scripts and other files from your [purple]nebari-config.yaml[/purple] file. """ - config_filename = Path(config) + config_filename = pathlib.Path(config) if not config_filename.is_file(): raise ValueError( diff --git a/src/_nebari/subcommands/validate.py b/src/_nebari/subcommands/validate.py index d3883b54f..3b73c5dbf 100644 --- a/src/_nebari/subcommands/validate.py +++ b/src/_nebari/subcommands/validate.py @@ -1,6 +1,7 @@ import pathlib import typer +from rich import print from _nebari.utils import load_yaml from nebari import schema diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 46a84e4b7..04c54216b 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -317,7 +317,9 @@ def deep_merge(*args): >>> print(deep_merge(value_1, value_2)) {'m': 1, 'e': {'f': {'g': {}, 'h': 1}}, 'b': {'d': 2, 'c': 1, 'z': [5, 6, 7]}, 'a': [1, 2, 3, 4]} """ - if len(args) == 1: + if len(args) == 0: + return {} + elif len(args) == 1: return args[0] elif len(args) > 2: return functools.reduce(deep_merge, args, {}) diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index 897a7da2e..097427c23 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -23,12 +23,12 @@ def __init__(self, output_directory: pathlib.Path, config: schema.Main): def validate(self): pass - def render(self, output_directory: pathlib.Path): - raise NotImplementedError() + def render(self) -> Dict[str, str]: + return {} @contextlib.contextmanager def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): - raise NotImplementedError() + return {} def check(self, stage_outputs: Dict[str, Dict[str, Any]]) -> bool: pass @@ -37,7 +37,7 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]) -> bool: def destroy( self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool] ): - raise NotImplementedError() + pass @hookspec diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index a2b9c24ac..475e47f46 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -7,6 +7,7 @@ DEFAULT_PLUGINS = [ # stages + "_nebari.stages.bootstrap", "_nebari.stages.terraform_state", "_nebari.stages.infrastructure", "_nebari.stages.kubernetes_initialize", From 1eedde3352742b59a1d53c961d88992030a0cda0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Jun 2023 03:31:01 +0000 Subject: [PATCH 010/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/provider/cicd/github.py | 4 +++- src/_nebari/render.py | 11 ++++------- src/_nebari/stages/base.py | 17 ++++++++++++----- src/_nebari/stages/bootstrap/__init__.py | 7 +++---- src/_nebari/stages/tf_objects.py | 3 --- src/_nebari/subcommands/render.py | 1 + src/_nebari/subcommands/support.py | 1 - 7 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/_nebari/provider/cicd/github.py b/src/_nebari/provider/cicd/github.py index b096930ff..127df352c 100644 --- a/src/_nebari/provider/cicd/github.py +++ b/src/_nebari/provider/cicd/github.py @@ -297,7 +297,9 @@ def gen_nebari_linter(config): else: env_vars = None - pull_request = GHA_on_extras(branches=[config.ci_cd.branch], paths=["nebari-config.yaml"]) + pull_request = GHA_on_extras( + branches=[config.ci_cd.branch], paths=["nebari-config.yaml"] + ) on = GHA_on(__root__={"pull_request": pull_request}) step1 = checkout_image_step() diff --git a/src/_nebari/render.py b/src/_nebari/render.py index fb5880685..7af263a36 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -1,18 +1,15 @@ import functools import hashlib -import json import os import shutil import sys from pathlib import Path from typing import Dict, List -import yaml from rich import print from rich.table import Table from ruamel.yaml import YAML -import _nebari from _nebari.deprecate import DEPRECATED_FILE_PATHS from _nebari.stages.base import get_available_stages from nebari import schema @@ -46,7 +43,9 @@ def render_template(output_directory, config_filename, dry_run=False): contents = {} config = schema.Main.parse_obj(config) for stage in get_available_stages(): - contents.update(stage(output_directory=output_directory, config=config).render()) + contents.update( + stage(output_directory=output_directory, config=config).render() + ) print(contents.keys()) @@ -154,9 +153,7 @@ def list_files( contents[filename].encode("utf8") ).hexdigest() else: - source_files[filename] = hashlib.sha256( - contents[filename] - ).hexdigest() + source_files[filename] = hashlib.sha256(contents[filename]).hexdigest() output_filename = os.path.join(output_base_dir, filename) if os.path.isfile(output_filename): diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index 755a6c57d..1068e128d 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -1,7 +1,7 @@ -import os import contextlib import inspect import itertools +import os import pathlib from typing import Any, Dict, List, Tuple @@ -27,13 +27,20 @@ def tf_objects(self) -> List[Dict]: def render(self) -> Dict[str, str]: contents = { - str( - self.stage_prefix / "_nebari.tf.json" - ): terraform.tf_render_objects(self.tf_objects()) + str(self.stage_prefix / "_nebari.tf.json"): terraform.tf_render_objects( + self.tf_objects() + ) } for root, dirs, filenames in os.walk(self.template_directory): for filename in filenames: - contents[os.path.join(self.stage_prefix, os.path.relpath(os.path.join(root, filename), self.template_directory))] = open( + contents[ + os.path.join( + self.stage_prefix, + os.path.relpath( + os.path.join(root, filename), self.template_directory + ), + ) + ] = open( os.path.join(root, filename), "rb", ).read() diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index 0071bbd90..c3dfddd80 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -1,11 +1,10 @@ -from typing import Dict, List from inspect import cleandoc - +from typing import Dict, List from _nebari.provider.cicd.github import gen_nebari_linter, gen_nebari_ops from _nebari.provider.cicd.gitlab import gen_gitlab_ci -from nebari.hookspecs import NebariStage, hookimpl from nebari import schema +from nebari.hookspecs import NebariStage, hookimpl def gen_gitignore(): @@ -54,7 +53,7 @@ def gen_cicd(config): class BootstrapStage(NebariStage): - name = "boostrap" + name = "bootstrap" priority = 0 def render(self) -> Dict[str, str]: diff --git a/src/_nebari/stages/tf_objects.py b/src/_nebari/stages/tf_objects.py index dd69fdc87..343fc5116 100644 --- a/src/_nebari/stages/tf_objects.py +++ b/src/_nebari/stages/tf_objects.py @@ -1,6 +1,3 @@ -from pathlib import Path -from typing import Dict - from _nebari.provider.terraform import Data, Provider, TerraformBackend from _nebari.utils import deep_merge from nebari import schema diff --git a/src/_nebari/subcommands/render.py b/src/_nebari/subcommands/render.py index cb27753ee..3760c089f 100644 --- a/src/_nebari/subcommands/render.py +++ b/src/_nebari/subcommands/render.py @@ -1,4 +1,5 @@ import pathlib + import typer from _nebari.render import render_template diff --git a/src/_nebari/subcommands/support.py b/src/_nebari/subcommands/support.py index 89cddccba..c85b4d56d 100644 --- a/src/_nebari/subcommands/support.py +++ b/src/_nebari/subcommands/support.py @@ -1,4 +1,3 @@ - import typer from nebari.hookspecs import hookimpl From c8816ced32f38ba58f865dd61c573ddfc2060a8c Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 15 Jun 2023 00:49:36 -0400 Subject: [PATCH 011/147] Making it further through deployment --- src/_nebari/deploy.py | 228 +----------------- src/_nebari/render.py | 2 +- src/_nebari/stages/infrastructure/__init__.py | 17 ++ .../stages/kubernetes_ingress/__init__.py | 58 ++++- .../stages/kubernetes_initialize/__init__.py | 6 +- src/nebari/hookspecs.py | 4 +- src/nebari/plugins.py | 2 +- src/nebari/schema.py | 5 +- 8 files changed, 96 insertions(+), 226 deletions(-) diff --git a/src/_nebari/deploy.py b/src/_nebari/deploy.py index 0dcd951af..c03f697ac 100644 --- a/src/_nebari/deploy.py +++ b/src/_nebari/deploy.py @@ -1,189 +1,21 @@ import logging +import os +import pathlib import subprocess import textwrap -from pathlib import Path +import contextlib from _nebari.provider import terraform from _nebari.provider.dns.cloudflare import update_record -from _nebari.stages import checks, input_vars, state_imports from _nebari.utils import ( check_cloud_credentials, - keycloak_provider_context, - kubernetes_provider_context, timer, ) +from _nebari.stages.base import get_available_stages +from nebari import schema -logger = logging.getLogger(__name__) - - -def provision_01_terraform_state(stage_outputs, config): - directory = Path("stages/01-terraform-state") - - if config["provider"] in {"existing", "local"}: - stage_outputs[directory] = {} - else: - stage_outputs[directory] = terraform.deploy( - terraform_import=True, - directory=directory / config["provider"], - input_vars=input_vars.stage_01_terraform_state(stage_outputs, config), - state_imports=state_imports.stage_01_terraform_state(stage_outputs, config), - ) - - -def provision_02_infrastructure(stage_outputs, config, disable_checks=False): - """Generalized method to provision infrastructure. - - After successful deployment the following properties are set on - `stage_outputs[directory]`. - - `kubernetes_credentials` which are sufficient credentials to - connect with the kubernetes provider - - `kubeconfig_filename` which is a path to a kubeconfig that can - be used to connect to a kubernetes cluster - - at least one node running such that resources in the - node_group.general can be scheduled - - At a high level this stage is expected to provision a kubernetes - cluster on a given provider. - """ - directory = "stages/02-infrastructure" - - stage_outputs[directory] = terraform.deploy( - Path(directory) / config["provider"], - input_vars=input_vars.stage_02_infrastructure(stage_outputs, config), - ) - - if not disable_checks: - checks.stage_02_infrastructure(stage_outputs, config) - - -def provision_03_kubernetes_initialize(stage_outputs, config, disable_checks=False): - directory = "stages/03-kubernetes-initialize" - - stage_outputs[directory] = terraform.deploy( - directory=directory, - input_vars=input_vars.stage_03_kubernetes_initialize(stage_outputs, config), - ) - - if not disable_checks: - checks.stage_03_kubernetes_initialize(stage_outputs, config) - - -def provision_04_kubernetes_ingress(stage_outputs, config, disable_checks=False): - directory = "stages/04-kubernetes-ingress" - - stage_outputs[directory] = terraform.deploy( - directory=directory, - input_vars=input_vars.stage_04_kubernetes_ingress(stage_outputs, config), - ) - - if not disable_checks: - checks.stage_04_kubernetes_ingress(stage_outputs, config) - - -def add_clearml_dns(zone_name, record_name, record_type, ip_or_hostname): - dns_records = [ - f"app.clearml.{record_name}", - f"api.clearml.{record_name}", - f"files.clearml.{record_name}", - ] - - for dns_record in dns_records: - update_record(zone_name, dns_record, record_type, ip_or_hostname) - - -def provision_ingress_dns( - stage_outputs, - config, - dns_provider: str, - dns_auto_provision: bool, - disable_prompt: bool = True, - disable_checks: bool = False, -): - directory = "stages/04-kubernetes-ingress" - - ip_or_name = stage_outputs[directory]["load_balancer_address"]["value"] - ip_or_hostname = ip_or_name["hostname"] or ip_or_name["ip"] - - if dns_auto_provision and dns_provider == "cloudflare": - record_name, zone_name = ( - config["domain"].split(".")[:-2], - config["domain"].split(".")[-2:], - ) - record_name = ".".join(record_name) - zone_name = ".".join(zone_name) - if config["provider"] in {"do", "gcp", "azure"}: - update_record(zone_name, record_name, "A", ip_or_hostname) - if config.get("clearml", {}).get("enabled"): - add_clearml_dns(zone_name, record_name, "A", ip_or_hostname) - elif config["provider"] == "aws": - update_record(zone_name, record_name, "CNAME", ip_or_hostname) - if config.get("clearml", {}).get("enabled"): - add_clearml_dns(zone_name, record_name, "CNAME", ip_or_hostname) - else: - logger.info( - f"Couldn't update the DNS record for cloud provider: {config['provider']}" - ) - elif not disable_prompt: - input( - f"Take IP Address {ip_or_hostname} and update DNS to point to " - f'"{config["domain"]}" [Press Enter when Complete]' - ) - - if not disable_checks: - checks.check_ingress_dns(stage_outputs, config, disable_prompt) - - -def provision_05_kubernetes_keycloak(stage_outputs, config, disable_checks=False): - directory = "stages/05-kubernetes-keycloak" - - stage_outputs[directory] = terraform.deploy( - directory=directory, - input_vars=input_vars.stage_05_kubernetes_keycloak(stage_outputs, config), - ) - - if not disable_checks: - checks.stage_05_kubernetes_keycloak(stage_outputs, config) - - -def provision_06_kubernetes_keycloak_configuration( - stage_outputs, config, disable_checks=False -): - directory = "stages/06-kubernetes-keycloak-configuration" - - stage_outputs[directory] = terraform.deploy( - directory=directory, - input_vars=input_vars.stage_06_kubernetes_keycloak_configuration( - stage_outputs, config - ), - ) - - if not disable_checks: - checks.stage_06_kubernetes_keycloak_configuration(stage_outputs, config) - - -def provision_07_kubernetes_services(stage_outputs, config, disable_checks=False): - directory = "stages/07-kubernetes-services" - - stage_outputs[directory] = terraform.deploy( - directory=directory, - input_vars=input_vars.stage_07_kubernetes_services(stage_outputs, config), - ) - - if not disable_checks: - checks.stage_07_kubernetes_services(stage_outputs, config) - - -def provision_08_nebari_tf_extensions(stage_outputs, config, disable_checks=False): - directory = "stages/08-nebari-tf-extensions" - - stage_outputs[directory] = terraform.deploy( - directory=directory, - input_vars=input_vars.stage_08_nebari_tf_extensions(stage_outputs, config), - ) - - if not disable_checks: - pass +logger = logging.getLogger(__name__) def guided_install( @@ -198,44 +30,12 @@ def guided_install( check_cloud_credentials(config) stage_outputs = {} - if ( - config["provider"] not in {"existing", "local"} - and config["terraform_state"]["type"] == "remote" - ): - if skip_remote_state_provision: - print("Skipping remote state provision") - else: - provision_01_terraform_state(stage_outputs, config) - - provision_02_infrastructure(stage_outputs, config, disable_checks) - - with kubernetes_provider_context( - stage_outputs["stages/02-infrastructure"]["kubernetes_credentials"]["value"] - ): - provision_03_kubernetes_initialize(stage_outputs, config, disable_checks) - provision_04_kubernetes_ingress(stage_outputs, config, disable_checks) - provision_ingress_dns( - stage_outputs, - config, - dns_provider=dns_provider, - dns_auto_provision=dns_auto_provision, - disable_prompt=disable_prompt, - disable_checks=disable_checks, - ) - provision_05_kubernetes_keycloak(stage_outputs, config, disable_checks) - - with keycloak_provider_context( - stage_outputs["stages/05-kubernetes-keycloak"]["keycloak_credentials"][ - "value" - ] - ): - provision_06_kubernetes_keycloak_configuration( - stage_outputs, config, disable_checks - ) - provision_07_kubernetes_services(stage_outputs, config, disable_checks) - provision_08_nebari_tf_extensions(stage_outputs, config, disable_checks) - - print("Nebari deployed successfully") + config = schema.Main(**config) + with contextlib.ExitStack() as stack: + for stage in get_available_stages(): + s = stage(output_directory=pathlib.Path('.'), config=config) + stack.enter_context(s.deploy(stage_outputs)) + print("Nebari deployed successfully") print("Services:") for service_name, service in stage_outputs["stages/07-kubernetes-services"][ @@ -247,9 +47,7 @@ def guided_install( f"Kubernetes kubeconfig located at file://{stage_outputs['stages/02-infrastructure']['kubeconfig_filename']['value']}" ) username = "root" - password = ( - config.get("security", {}).get("keycloak", {}).get("initial_root_password", "") - ) + password = config.security.keycloak.initial_root_password if password: print(f"Kubecloak master realm username={username} password={password}") diff --git a/src/_nebari/render.py b/src/_nebari/render.py index 7af263a36..1f00e9305 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -41,7 +41,7 @@ def render_template(output_directory, config_filename, dry_run=False): set_env_vars_in_config(config) contents = {} - config = schema.Main.parse_obj(config) + config = schema.Main(**config) for stage in get_available_stages(): contents.update( stage(output_directory=output_directory, config=config).render() diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 974e3bc23..8cdcfaa5e 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -1,7 +1,9 @@ +import os import contextlib import inspect import pathlib import sys +import tempfile from typing import Any, Dict, List from _nebari.stages.base import NebariTerraformStage @@ -39,6 +41,21 @@ def kubernetes_provider_context(kubernetes_credentials: Dict[str, str]): class KubernetesInfrastructureStage(NebariTerraformStage): + """Generalized method to provision infrastructure. + + After successful deployment the following properties are set on + `stage_outputs[directory]`. + - `kubernetes_credentials` which are sufficient credentials to + connect with the kubernetes provider + - `kubeconfig_filename` which is a path to a kubeconfig that can + be used to connect to a kubernetes cluster + - at least one node running such that resources in the + node_group.general can be scheduled + + At a high level this stage is expected to provision a kubernetes + cluster on a given provider. + """ + name = "02-infrastructure" priority = 20 diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index 64d9fa4c7..3cffc0130 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -17,6 +17,60 @@ TIMEOUT = 10 # seconds +def add_clearml_dns(zone_name, record_name, record_type, ip_or_hostname): + dns_records = [ + f"app.clearml.{record_name}", + f"api.clearml.{record_name}", + f"files.clearml.{record_name}", + ] + + for dns_record in dns_records: + update_record(zone_name, dns_record, record_type, ip_or_hostname) + + +def provision_ingress_dns( + stage_outputs, + config, + dns_provider: str, + dns_auto_provision: bool, + disable_prompt: bool = True, + disable_checks: bool = False, +): + directory = "stages/04-kubernetes-ingress" + + ip_or_name = stage_outputs[directory]["load_balancer_address"]["value"] + ip_or_hostname = ip_or_name["hostname"] or ip_or_name["ip"] + + if dns_auto_provision and dns_provider == "cloudflare": + record_name, zone_name = ( + config["domain"].split(".")[:-2], + config["domain"].split(".")[-2:], + ) + record_name = ".".join(record_name) + zone_name = ".".join(zone_name) + if config["provider"] in {"do", "gcp", "azure"}: + update_record(zone_name, record_name, "A", ip_or_hostname) + if config.get("clearml", {}).get("enabled"): + add_clearml_dns(zone_name, record_name, "A", ip_or_hostname) + + elif config["provider"] == "aws": + update_record(zone_name, record_name, "CNAME", ip_or_hostname) + if config.get("clearml", {}).get("enabled"): + add_clearml_dns(zone_name, record_name, "CNAME", ip_or_hostname) + else: + logger.info( + f"Couldn't update the DNS record for cloud provider: {config['provider']}" + ) + elif not disable_prompt: + input( + f"Take IP Address {ip_or_hostname} and update DNS to point to " + f'"{config["domain"]}" [Press Enter when Complete]' + ) + + if not disable_checks: + checks.check_ingress_dns(stage_outputs, config, disable_prompt) + + def _calculate_node_groups(config: schema.Main): if config.provider == schema.ProviderEnum.aws: return { @@ -41,7 +95,7 @@ def _calculate_node_groups(config: schema.Main): elif config.provider == schema.ProviderEnum.existing: return config.existing.node_selectors else: - return config.local.node_selectors + return config.local.dict()['node_selectors'] def check_ingress_dns(stage_outputs, config, disable_prompt): @@ -125,7 +179,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "name": self.config.project_name, "environment": self.config.namespace, "node_groups": _calculate_node_groups(self.config), - **config.ingress.terraform_overrides, + **self.config.ingress.terraform_overrides, }, **cert_details, } diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py index 41eff6d76..cfd50b30c 100644 --- a/src/_nebari/stages/kubernetes_initialize/__init__.py +++ b/src/_nebari/stages/kubernetes_initialize/__init__.py @@ -30,7 +30,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): ) gpu_node_group_names = [] - elif self.config.provider == schema.ProvderEnum.aws: + elif self.config.provider == schema.ProviderEnum.aws: gpu_enabled = any( node_group.gpu for node_group in self.config.amazon_web_services.node_groups.values() @@ -46,8 +46,8 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "name": self.config.project_name, "environment": self.config.namespace, "cloud-provider": self.config.provider.value, - "aws-region": self.config.amazon_web_services.region, - "external_container_reg": self.config.external_container_reg.enabled, + "aws-region": self.config.amazon_web_services.region if self.config.provider == schema.ProviderEnum.aws else None, + "external_container_reg": self.config.external_container_reg.dict(), "gpu_enabled": gpu_enabled, "gpu_node_group_names": gpu_node_group_names, } diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index 097427c23..4f9e688f5 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -28,7 +28,7 @@ def render(self) -> Dict[str, str]: @contextlib.contextmanager def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): - return {} + yield def check(self, stage_outputs: Dict[str, Dict[str, Any]]) -> bool: pass @@ -37,7 +37,7 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]) -> bool: def destroy( self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool] ): - pass + yield @hookspec diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index 475e47f46..2fc340743 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -18,7 +18,7 @@ "_nebari.stages.nebari_tf_extensions", # subcommands "_nebari.subcommands.dev", - # "_nebari.subcommands.deploy", + "_nebari.subcommands.deploy", # "_nebari.subcommands.destroy", "_nebari.subcommands.keycloak", "_nebari.subcommands.render", diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 88aff7eb5..1a0ba57d7 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -550,7 +550,7 @@ class Ingress(Base): class ExtContainerReg(Base): - enabled: bool + enabled: bool = False access_key_id: typing.Optional[str] secret_access_key: typing.Optional[str] extcr_account: typing.Optional[str] @@ -648,7 +648,7 @@ class Main(Base): prefect: Prefect = Prefect() cdsdashboards: CDSDashboards = CDSDashboards() security: Security = Security() - external_container_reg: typing.Optional[ExtContainerReg] + external_container_reg: ExtContainerReg = ExtContainerReg() default_images: DefaultImages = DefaultImages() storage: Storage = Storage() local: typing.Optional[LocalProvider] @@ -762,6 +762,7 @@ def is_version_accepted(cls, v): @validator("project_name") def _project_name_convention(cls, value: typing.Any, values): project_name_convention(value=value, values=values) + return value def verify(config): From 8c6e7a479170179154424aa3b785e91f5ecb19b0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Jun 2023 04:49:53 +0000 Subject: [PATCH 012/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/deploy.py | 13 +++---------- src/_nebari/stages/infrastructure/__init__.py | 2 +- src/_nebari/stages/kubernetes_ingress/__init__.py | 2 +- .../stages/kubernetes_initialize/__init__.py | 4 +++- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/_nebari/deploy.py b/src/_nebari/deploy.py index c03f697ac..bcd2f70ba 100644 --- a/src/_nebari/deploy.py +++ b/src/_nebari/deploy.py @@ -1,20 +1,13 @@ +import contextlib import logging -import os import pathlib import subprocess import textwrap -import contextlib -from _nebari.provider import terraform -from _nebari.provider.dns.cloudflare import update_record -from _nebari.utils import ( - check_cloud_credentials, - timer, -) from _nebari.stages.base import get_available_stages +from _nebari.utils import check_cloud_credentials, timer from nebari import schema - logger = logging.getLogger(__name__) @@ -33,7 +26,7 @@ def guided_install( config = schema.Main(**config) with contextlib.ExitStack() as stack: for stage in get_available_stages(): - s = stage(output_directory=pathlib.Path('.'), config=config) + s = stage(output_directory=pathlib.Path("."), config=config) stack.enter_context(s.deploy(stage_outputs)) print("Nebari deployed successfully") diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 8cdcfaa5e..91bcf4cdf 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -1,6 +1,6 @@ -import os import contextlib import inspect +import os import pathlib import sys import tempfile diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index 3cffc0130..1f09f4001 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -95,7 +95,7 @@ def _calculate_node_groups(config: schema.Main): elif config.provider == schema.ProviderEnum.existing: return config.existing.node_selectors else: - return config.local.dict()['node_selectors'] + return config.local.dict()["node_selectors"] def check_ingress_dns(stage_outputs, config, disable_prompt): diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py index cfd50b30c..d2263cc35 100644 --- a/src/_nebari/stages/kubernetes_initialize/__init__.py +++ b/src/_nebari/stages/kubernetes_initialize/__init__.py @@ -46,7 +46,9 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "name": self.config.project_name, "environment": self.config.namespace, "cloud-provider": self.config.provider.value, - "aws-region": self.config.amazon_web_services.region if self.config.provider == schema.ProviderEnum.aws else None, + "aws-region": self.config.amazon_web_services.region + if self.config.provider == schema.ProviderEnum.aws + else None, "external_container_reg": self.config.external_container_reg.dict(), "gpu_enabled": gpu_enabled, "gpu_node_group_names": gpu_node_group_names, From cc1601942066029dc8febeda1cd0a05fbeb318e6 Mon Sep 17 00:00:00 2001 From: eskild <42120229+iameskild@users.noreply.github.com> Date: Thu, 15 Jun 2023 15:10:21 +0200 Subject: [PATCH 013/147] [WIP] Add InputVar schema (#1835) --- src/_nebari/cli/main.py | 10 +- src/_nebari/stages/infrastructure/__init__.py | 174 ++++++++++-------- .../infrastructure/template/aws/variables.tf | 5 - .../template/azure/variables.tf | 2 - .../infrastructure/template/gcp/variables.tf | 12 +- .../template/local/variables.tf | 1 - src/nebari/schema.py | 48 ++++- 7 files changed, 141 insertions(+), 111 deletions(-) diff --git a/src/_nebari/cli/main.py b/src/_nebari/cli/main.py index cbc0f03ef..f00045f9f 100644 --- a/src/_nebari/cli/main.py +++ b/src/_nebari/cli/main.py @@ -10,7 +10,6 @@ from ruamel import yaml from typer.core import TyperGroup -from _nebari.cli.dev import app_dev from _nebari.cli.init import ( check_auth_provider_creds, check_cloud_provider_creds, @@ -24,7 +23,11 @@ from _nebari.deploy import deploy_configuration from _nebari.destroy import destroy_configuration from _nebari.render import render_template -from _nebari.schema import ( +from _nebari.subcommands import app_dev +from _nebari.upgrade import do_upgrade +from _nebari.utils import load_yaml +from _nebari.version import __version__ +from nebari.schema import ( AuthenticationEnum, CiEnum, GitRepoEnum, @@ -33,9 +36,6 @@ TerraformStateEnum, verify, ) -from _nebari.upgrade import do_upgrade -from _nebari.utils import load_yaml -from _nebari.version import __version__ SECOND_COMMAND_GROUP_NAME = "Additional Commands" GUIDED_INIT_MSG = ( diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 91bcf4cdf..41e820acd 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -1,10 +1,9 @@ import contextlib import inspect -import os import pathlib import sys import tempfile -from typing import Any, Dict, List +from typing import Any, Dict, List, Union from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( @@ -17,6 +16,46 @@ from nebari.hookspecs import NebariStage, hookimpl +def get_kubeconfig_filename(): + return pathlib.Path(tempfile.gettempdir()) / "NEBARI_KUBECONFIG" + + +# TODO: +# - create schema for node group for each provider + + +class LocalInputVars(schema.Base): + kubeconfig_filename: Union[str, pathlib.Path] = get_kubeconfig_filename() + kube_context: str + + +class ExistingInputVars(schema.Base): + kube_context: str + + +class BaseCloudProviderInputVars(schema.Base): + name: str + environment: str + kubeconfig_filename: str = get_kubeconfig_filename() + + +class DigitalOceanInputVars(BaseCloudProviderInputVars, schema.DigitalOceanProvider): + pass + + +class GCPInputVars(BaseCloudProviderInputVars, schema.GoogleCloudPlatformProvider): + pass + + +class AzureInputVars(BaseCloudProviderInputVars, schema.AzureProvider): + resource_group_name: str + node_resource_group_name: str + + +class AWSInputVars(BaseCloudProviderInputVars, schema.AmazonWebServicesProvider): + pass + + @contextlib.contextmanager def kubernetes_provider_context(kubernetes_credentials: Dict[str, str]): credential_mapping = { @@ -95,91 +134,64 @@ def tf_objects(self) -> List[Dict]: def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): if self.config.provider == schema.ProviderEnum.local: - return { - "kubeconfig_filename": os.path.join( - tempfile.gettempdir(), "NEBARI_KUBECONFIG" - ), - "kube_context": self.config.local.kube_context, - } + return LocalInputVars(kube_context=self.config.local.kube_context).dict() elif self.config.provider == schema.ProviderEnum.existing: - return {"kube_context": self.config.existing.kube_context} + return ExistingInputVars( + kube_context=self.config.existing.kube_context + ).dict() elif self.config.provider == schema.ProviderEnum.do: - return { - "name": self.config.project_name, - "environment": self.config.namespace, - "region": self.config.digital_ocean.region, - "kubernetes_version": self.config.digital_ocean.kubernetes_version, - "node_groups": self.config.digital_ocean.node_groups, - "kubeconfig_filename": os.path.join( - tempfile.gettempdir(), "NEBARI_KUBECONFIG" - ), - **self.config.do.terraform_overrides, - } + return DigitalOceanInputVars( + name=self.config.project_name, + environment=self.config.namespace, + region=self.config.digital_ocean.region, + tags=self.config.digital_ocean.tags, + kubernetes_version=self.config.digital_ocean.kubernetes_version, + node_groups=self.config.digital_ocean.node_groups, + ).dict() elif self.config.provider == schema.ProviderEnum.gcp: - return { - "name": self.config.project_name, - "environment": self.config.namespace, - "region": self.config.google_cloud_platform.region, - "project_id": self.config.google_cloud_platform.project, - "kubernetes_version": self.config.google_cloud_platform.kubernetes_version, - "release_channel": self.config.google_cloud_platform.release_channel, - "node_groups": [ - { - "name": key, - "instance_type": value["instance"], - "min_size": value["min_nodes"], - "max_size": value["max_nodes"], - "guest_accelerators": value["guest_accelerators"] - if "guest_accelerators" in value - else [], - **value, - } - for key, value in self.config.google_cloud_platform.node_groups.items() - ], - "kubeconfig_filename": os.path.join( - tempfile.gettempdir(), "NEBARI_KUBECONFIG" - ), - **self.config.gcp.terraform_overrides, - } + return GCPInputVars( + name=self.config.project_name, + environment=self.config.namespace, + region=self.config.google_cloud_platform.region, + project_id=self.config.google_cloud_platform.project, + availability_zones=self.config.google_cloud_platform.availability_zones, + node_groups=self.config.google_cloud_platform.node_groups, + tags=self.config.google_cloud_platform.tags, + kubernetes_version=self.config.google_cloud_platform.kubernetes_version, + release_channel=self.config.google_cloud_platform.release_channel, + networking_mode=self.config.google_cloud_platform.networking_mode, + network=self.config.google_cloud_platform.network, + subnetwork=self.config.google_cloud_platform.subnetwork, + ip_allocation_policy=self.config.google_cloud_platform.ip_allocation_policy, + master_authorized_networks_config=self.config.google_cloud_platform.master_authorized_networks_config, + private_cluster_config=self.config.google_cloud_platform.private_cluster_config, + ).dict() elif self.config.provider == schema.ProviderEnum.azure: - return { - "name": self.config.project_name, - "environment": self.config.namespace, - "region": self.config.azure.region, - "kubernetes_version": self.config.azure.kubernetes_version, - "node_groups": self.config.azure.node_groups, - "kubeconfig_filename": os.path.join( - tempfile.gettempdir(), "NEBARI_KUBECONFIG" - ), - "resource_group_name": f"{self.config.project_name}-{self.config.namespace}", - "node_resource_group_name": f"{self.config.project_name}-{self.config.namespace}-node-resource-group", - **self.config.azure.terraform_overrides, - } + return AzureInputVars( + name=self.config.project_name, + environment=self.config.namespace, + region=self.config.azure.region, + kubernetes_version=self.config.azure.kubernetes_version, + node_groups=self.config.azure.node_groups, + resource_group_name=f"{self.config.project_name}-{self.config.namespace}", + node_resource_group_name=f"{self.config.project_name}-{self.config.namespace}-node-resource-group", + vnet_subnet_id=self.config.azure.vnet_subnet_id, + private_cluster_enabled=self.config.azure.private_cluster_enabled, + ).dict() elif self.config.provider == schema.ProviderEnum.aws: - return { - "name": self.config.project_name, - "environment": self.config.namespace, - "region": self.config.amazon_web_services.region, - "kubernetes_version": self.config.amazon_web_services.kubernetes_version, - "node_groups": [ - { - "name": key, - "min_size": value["min_nodes"], - "desired_size": max(value["min_nodes"], 1), - "max_size": value["max_nodes"], - "gpu": value.get("gpu", False), - "instance_type": value["instance"], - "single_subnet": value.get("single_subnet", False), - } - for key, value in self.config.amazon_web_services.node_groups.items() - ], - "kubeconfig_filename": os.path.join( - tempfile.gettempdir(), "NEBARI_KUBECONFIG" - ), - **self.config.amazon_web_services.terraform_overrides, - } + return AWSInputVars( + name=self.config.project_name, + environment=self.config.namespace, + existing_subnet_ids=self.config.amazon_web_services.existing_subnet_ids, + existing_security_group_ids=self.config.amazon_web_services.existing_security_group_ids, + region=self.config.amazon_web_services.region, + kubernetes_version=self.config.amazon_web_services.kubernetes_version, + node_groups=self.config.amazon_web_services.node_groups, + availability_zones=self.config.amazon_web_services.availability_zones, + vpc_cidr_block=self.config.amazon_web_services.vpc_cidr_block, + ).dict() else: - return {} + raise ValueError(f"Unknown provider: {self.config.provider}") def check(self, stage_outputs: Dict[str, Dict[str, Any]]): from kubernetes import client, config diff --git a/src/_nebari/stages/infrastructure/template/aws/variables.tf b/src/_nebari/stages/infrastructure/template/aws/variables.tf index a9ab302f1..593827d0a 100644 --- a/src/_nebari/stages/infrastructure/template/aws/variables.tf +++ b/src/_nebari/stages/infrastructure/template/aws/variables.tf @@ -11,13 +11,11 @@ variable "environment" { variable "existing_subnet_ids" { description = "Existing VPC ID to use for Kubernetes resources" type = list(string) - default = null } variable "existing_security_group_id" { description = "Existing security group ID to use for Kubernetes resources" type = string - default = null } variable "region" { @@ -46,19 +44,16 @@ variable "node_groups" { variable "availability_zones" { description = "AWS availability zones within AWS region" type = list(string) - default = [] } variable "vpc_cidr_block" { description = "VPC cidr block for infastructure" type = string - default = "10.10.0.0/16" } variable "kubeconfig_filename" { description = "Kubernetes kubeconfig written to filesystem" type = string - default = null } variable "eks_endpoint_private_access" { diff --git a/src/_nebari/stages/infrastructure/template/azure/variables.tf b/src/_nebari/stages/infrastructure/template/azure/variables.tf index bcbefa688..9616bf2b0 100644 --- a/src/_nebari/stages/infrastructure/template/azure/variables.tf +++ b/src/_nebari/stages/infrastructure/template/azure/variables.tf @@ -30,7 +30,6 @@ variable "node_groups" { variable "kubeconfig_filename" { description = "Kubernetes kubeconfig written to filesystem" type = string - default = null } variable "resource_group_name" { @@ -46,7 +45,6 @@ variable "node_resource_group_name" { variable "vnet_subnet_id" { description = "The ID of a Subnet where the Kubernetes Node Pool should exist. Changing this forces a new resource to be created." type = string - default = null } variable "private_cluster_enabled" { diff --git a/src/_nebari/stages/infrastructure/template/gcp/variables.tf b/src/_nebari/stages/infrastructure/template/gcp/variables.tf index 18607c7b6..e89f82003 100644 --- a/src/_nebari/stages/infrastructure/template/gcp/variables.tf +++ b/src/_nebari/stages/infrastructure/template/gcp/variables.tf @@ -21,25 +21,21 @@ variable "project_id" { variable "availability_zones" { description = "Availability zones to use for nebari deployment" type = list(string) - default = [] } variable "node_groups" { description = "GCP node groups" type = any - default = null } variable "kubeconfig_filename" { description = "Kubernetes kubeconfig written to filesystem" type = string - default = null } variable "tags" { description = "Google Cloud Platform tags to assign to resources" - type = map(string) - default = {} + type = list(string) } variable "kubernetes_version" { @@ -55,19 +51,16 @@ variable "release_channel" { variable "networking_mode" { description = "Determines whether alias IPs or routes will be used for pod IPs in the cluster. Options are VPC_NATIVE or ROUTES." type = string - default = "ROUTES" } variable "network" { description = "Name of the VPC network, where the cluster should be deployed" type = string - default = "default" } variable "subnetwork" { description = "Name of the subnet for deploying cluster into" type = string - default = null } variable "ip_allocation_policy" { @@ -78,7 +71,6 @@ variable "ip_allocation_policy" { cluster_ipv4_cidr_block = string services_ipv4_cidr_block = string })) - default = null } variable "master_authorized_networks_config" { @@ -89,7 +81,6 @@ variable "master_authorized_networks_config" { display_name = string })) })) - default = null } variable "private_cluster_config" { @@ -99,5 +90,4 @@ variable "private_cluster_config" { enable_private_endpoint = bool master_ipv4_cidr_block = string })) - default = null } diff --git a/src/_nebari/stages/infrastructure/template/local/variables.tf b/src/_nebari/stages/infrastructure/template/local/variables.tf index 097bb1959..246632e92 100644 --- a/src/_nebari/stages/infrastructure/template/local/variables.tf +++ b/src/_nebari/stages/infrastructure/template/local/variables.tf @@ -1,7 +1,6 @@ variable "kubeconfig_filename" { description = "Kubernetes kubeconfig written to filesystem" type = string - default = null } variable "kube_context" { diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 1a0ba57d7..a5c250706 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -316,22 +316,55 @@ class AWSNodeGroup(NodeGroup): single_subnet: typing.Optional[bool] = False +class GCPIPAllocationPolicy(Base): + cluster_secondary_range_name: str + services_secondary_range_name: str + cluster_ipv4_cidr_block: str + services_ipv4_cidr_block: str + + +class GCPCIDRBlock(Base): + cidr_block: str + display_name: str + + +class GCPMasterAuthorizedNetworksConfig(Base): + cidr_blocks: typing.List[GCPCIDRBlock] + + +class GCPPrivateClusterConfig(Base): + enable_private_endpoint: bool + enable_private_nodes: bool + master_ipv4_cidr_block: str + + class DigitalOceanProvider(Base): region: str kubernetes_version: str node_groups: typing.Dict[str, NodeGroup] - terraform_overrides: typing.Any + tags: typing.Optional[typing.List[str]] = [] class GoogleCloudPlatformProvider(Base): project: str region: str - zone: typing.Optional[str] # No longer used - availability_zones: typing.Optional[typing.List[str]] # Genuinely optional + availability_zones: typing.Optional[typing.List[str]] = [] kubernetes_version: str release_channel: typing.Optional[str] node_groups: typing.Dict[str, NodeGroup] - terraform_overrides: typing.Any + tags: typing.Optional[typing.List[str]] = [] + networking_mode: str = "ROUTE" + network: str = "default" + subnetwork: typing.Optional[typing.Union[str, None]] = None + ip_allocation_policy: typing.Optional[ + typing.Union[GCPIPAllocationPolicy, None] + ] = None + master_authorized_networks_config: typing.Optional[ + typing.Union[GCPCIDRBlock, None] + ] = None + private_cluster_config: typing.Optional[ + typing.Union[GCPPrivateClusterConfig, None] + ] = None class AzureProvider(Base): @@ -339,7 +372,8 @@ class AzureProvider(Base): kubernetes_version: str node_groups: typing.Dict[str, NodeGroup] storage_account_postfix: str - terraform_overrides: typing.Any + vnet_subnet_id: typing.Optional[typing.Union[str, None]] = None + private_cluster_enabled: bool = False class AmazonWebServicesProvider(Base): @@ -347,7 +381,9 @@ class AmazonWebServicesProvider(Base): availability_zones: typing.Optional[typing.List[str]] kubernetes_version: str node_groups: typing.Dict[str, AWSNodeGroup] - terraform_overrides: typing.Any + existing_subnet_ids: typing.Optional[typing.List[str]] + existing_security_group_ids: typing.Optional[str] + vpc_cidr_block: str = "10.10.0.0/16" class LocalProvider(Base): From 5bc870625a0a7b4495eba249107198237e7beb4a Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 15 Jun 2023 16:46:30 -0400 Subject: [PATCH 014/147] Working local deploy, render, destroy and major simplifications --- src/_nebari/deploy.py | 15 +- src/_nebari/destroy.py | 178 ++--------------- src/_nebari/keycloak.py | 28 ++- src/_nebari/render.py | 86 +-------- src/_nebari/stages/base.py | 21 +- src/_nebari/stages/bootstrap/__init__.py | 65 ++++++- src/_nebari/stages/infrastructure/__init__.py | 8 +- .../stages/kubernetes_keycloak/__init__.py | 2 +- .../__init__.py | 2 +- .../stages/kubernetes_services/__init__.py | 19 +- .../stages/nebari_tf_extensions/__init__.py | 6 +- src/_nebari/subcommands/deploy.py | 19 +- src/_nebari/subcommands/destroy.py | 23 +-- src/_nebari/subcommands/dev.py | 9 +- src/_nebari/subcommands/keycloak.py | 29 +-- src/_nebari/subcommands/render.py | 19 +- src/_nebari/subcommands/validate.py | 12 +- src/_nebari/utils.py | 66 +------ src/nebari/plugins.py | 2 +- src/nebari/schema.py | 180 +++++++++++++----- 20 files changed, 298 insertions(+), 491 deletions(-) diff --git a/src/_nebari/deploy.py b/src/_nebari/deploy.py index bcd2f70ba..c4bb30fc6 100644 --- a/src/_nebari/deploy.py +++ b/src/_nebari/deploy.py @@ -5,25 +5,21 @@ import textwrap from _nebari.stages.base import get_available_stages -from _nebari.utils import check_cloud_credentials, timer +from _nebari.utils import timer from nebari import schema logger = logging.getLogger(__name__) def guided_install( - config, + config: schema.Main, dns_provider, dns_auto_provision, disable_prompt=False, disable_checks=False, skip_remote_state_provision=False, ): - # 01 Check Environment Variables - check_cloud_credentials(config) - stage_outputs = {} - config = schema.Main(**config) with contextlib.ExitStack() as stack: for stage in get_available_stages(): s = stage(output_directory=pathlib.Path("."), config=config) @@ -51,15 +47,14 @@ def guided_install( def deploy_configuration( - config, + config: schema.Main, dns_provider, dns_auto_provision, disable_prompt, disable_checks, skip_remote_state_provision, ): - if config.get("prevent_deploy", False): - # Note if we used the Pydantic model properly, we might get that nebari_config.prevent_deploy always exists but defaults to False + if config.prevent_deploy: raise ValueError( textwrap.dedent( """ @@ -76,7 +71,7 @@ def deploy_configuration( ) ) - logger.info(f'All nebari endpoints will be under https://{config["domain"]}') + logger.info(f'All nebari endpoints will be under https://{config.domain}') if disable_checks: logger.warning( diff --git a/src/_nebari/destroy.py b/src/_nebari/destroy.py index eefd3eefc..61cc68c07 100644 --- a/src/_nebari/destroy.py +++ b/src/_nebari/destroy.py @@ -1,185 +1,35 @@ +from typing import List, Dict import functools import logging -from pathlib import Path +import os +import contextlib +import pathlib from _nebari.provider import terraform -from _nebari.stages import input_vars, state_imports -from _nebari.utils import ( - check_cloud_credentials, - keycloak_provider_context, - kubernetes_provider_context, - timer, -) +from _nebari.stages.base import get_available_stages +from _nebari.utils import timer +from nebari import schema logger = logging.getLogger(__name__) -def gather_stage_outputs(config): +def destroy_stages(config: schema.Main): stage_outputs = {} - - _terraform_init_output = functools.partial( - terraform.deploy, - terraform_init=True, - terraform_import=True, - terraform_apply=False, - terraform_destroy=False, - ) - - if ( - config["provider"] not in {"existing", "local"} - and config["terraform_state"]["type"] == "remote" - ): - stage_outputs["stages/01-terraform-state"] = _terraform_init_output( - directory=Path("stages/01-terraform-state") / config["provider"], - input_vars=input_vars.stage_01_terraform_state(stage_outputs, config), - state_imports=state_imports.stage_01_terraform_state(stage_outputs, config), - ) - - stage_outputs["stages/02-infrastructure"] = _terraform_init_output( - directory=Path("stages/02-infrastructure") / config["provider"], - input_vars=input_vars.stage_02_infrastructure(stage_outputs, config), - ) - - stage_outputs["stages/03-kubernetes-initialize"] = _terraform_init_output( - directory="stages/03-kubernetes-initialize", - input_vars=input_vars.stage_03_kubernetes_initialize(stage_outputs, config), - ) - - stage_outputs["stages/04-kubernetes-ingress"] = _terraform_init_output( - directory="stages/04-kubernetes-ingress", - input_vars=input_vars.stage_04_kubernetes_ingress(stage_outputs, config), - ) - - stage_outputs["stages/05-kubernetes-keycloak"] = _terraform_init_output( - directory="stages/05-kubernetes-keycloak", - input_vars=input_vars.stage_05_kubernetes_keycloak(stage_outputs, config), - ) - - stage_outputs[ - "stages/06-kubernetes-keycloak-configuration" - ] = _terraform_init_output( - directory="stages/06-kubernetes-keycloak-configuration", - input_vars=input_vars.stage_06_kubernetes_keycloak_configuration( - stage_outputs, config - ), - ) - - stage_outputs["stages/07-kubernetes-services"] = _terraform_init_output( - directory="stages/07-kubernetes-services", - input_vars=input_vars.stage_07_kubernetes_services(stage_outputs, config), - ) - - stage_outputs["stages/08-nebari-tf-extensions"] = _terraform_init_output( - directory="stages/08-nebari-tf-extensions", - input_vars=input_vars.stage_08_nebari_tf_extensions(stage_outputs, config), - ) - - return stage_outputs - - -def destroy_stages(stage_outputs, config): - def _terraform_destroy(ignore_errors=False, terraform_apply=False, **kwargs): - try: - terraform.deploy( - terraform_init=True, - terraform_import=True, - terraform_apply=terraform_apply, - terraform_destroy=True, - **kwargs, - ) - except terraform.TerraformException as e: - if not ignore_errors: - raise e - return False - return True - status = {} - - with kubernetes_provider_context( - stage_outputs["stages/02-infrastructure"]["kubernetes_credentials"]["value"] - ): - with keycloak_provider_context( - stage_outputs["stages/05-kubernetes-keycloak"]["keycloak_credentials"][ - "value" - ] - ): - status["stages/08-nebari-tf-extensions"] = _terraform_destroy( - directory="stages/08-nebari-tf-extensions", - input_vars=input_vars.stage_08_nebari_tf_extensions( - stage_outputs, config - ), - ignore_errors=True, - ) - - status["stages/07-kubernetes-services"] = _terraform_destroy( - directory="stages/07-kubernetes-services", - input_vars=input_vars.stage_07_kubernetes_services( - stage_outputs, config - ), - ignore_errors=True, - ) - - status["stages/06-kubernetes-keycloak-configuration"] = _terraform_destroy( - directory="stages/06-kubernetes-keycloak-configuration", - input_vars=input_vars.stage_06_kubernetes_keycloak_configuration( - stage_outputs, config - ), - ignore_errors=True, - ) - - status["stages/05-kubernetes-keycloak"] = _terraform_destroy( - directory="stages/05-kubernetes-keycloak", - input_vars=input_vars.stage_05_kubernetes_keycloak(stage_outputs, config), - ignore_errors=True, - ) - - status["stages/04-kubernetes-ingress"] = _terraform_destroy( - directory="stages/04-kubernetes-ingress", - input_vars=input_vars.stage_04_kubernetes_ingress(stage_outputs, config), - ignore_errors=True, - ) - - status["stages/03-kubernetes-initialize"] = _terraform_destroy( - directory="stages/03-kubernetes-initialize", - input_vars=input_vars.stage_03_kubernetes_initialize(stage_outputs, config), - ignore_errors=True, - ) - - status["stages/02-infrastructure"] = _terraform_destroy( - directory=Path("stages/02-infrastructure") / config["provider"], - input_vars=input_vars.stage_02_infrastructure(stage_outputs, config), - ignore_errors=True, - ) - - if ( - config["provider"] not in {"existing", "local"} - and config["terraform_state"]["type"] == "remote" - ): - status["stages/01-terraform-state"] = _terraform_destroy( - # acl and force_destroy do not import properly - # and only get refreshed properly with an apply - terraform_apply=True, - directory=Path("stages/01-terraform-state") / config["provider"], - input_vars=input_vars.stage_01_terraform_state(stage_outputs, config), - ignore_errors=True, - ) - + with contextlib.ExitStack() as stack: + for stage in get_available_stages(): + s = stage(output_directory=pathlib.Path("."), config=config) + stack.enter_context(s.destroy(stage_outputs, status)) return status -def destroy_configuration(config): +def destroy_configuration(config: schema.Main): logger.info( """Removing all infrastructure, your local files will still remain, you can use 'nebari deploy' to re-install infrastructure using same config file\n""" ) - check_cloud_credentials(config) - - # Populate stage_outputs to determine progress of deployment and - # get credentials to kubernetes and keycloak context - stage_outputs = gather_stage_outputs(config) - with timer(logger, "destroying Nebari"): - status = destroy_stages(stage_outputs, config) + status = destroy_stages(config) for stage_name, success in status.items(): if not success: diff --git a/src/_nebari/keycloak.py b/src/_nebari/keycloak.py index 9f49278ff..1f5709d6e 100644 --- a/src/_nebari/keycloak.py +++ b/src/_nebari/keycloak.py @@ -8,15 +8,12 @@ import rich from _nebari.utils import load_yaml -from nebari.schema import verify +from nebari import schema logger = logging.getLogger(__name__) -def do_keycloak(config_filename, *args): - config = load_yaml(config_filename) - verify(config) - +def do_keycloak(config: schema.Main, *args): # suppress insecure warnings import urllib3 @@ -32,7 +29,7 @@ def do_keycloak(config_filename, *args): username = args[1] password = args[2] if len(args) >= 3 else None - create_user(keycloak_admin, username, password, domain=config["domain"]) + create_user(keycloak_admin, username, password, domain=config.domain) elif args[0] == "listusers": list_users(keycloak_admin) else: @@ -84,18 +81,18 @@ def list_users(keycloak_admin: keycloak.KeycloakAdmin): ) -def get_keycloak_admin_from_config(config): +def get_keycloak_admin_from_config(config: schema.Main): keycloak_server_url = os.environ.get( - "KEYCLOAK_SERVER_URL", f"https://{config['domain']}/auth/" + "KEYCLOAK_SERVER_URL", f"https://{config.domain}/auth/" ) keycloak_username = os.environ.get("KEYCLOAK_ADMIN_USERNAME", "root") keycloak_password = os.environ.get( "KEYCLOAK_ADMIN_PASSWORD", - config.get("security", {}).get("keycloak", {}).get("initial_root_password", ""), + config.security.keycloak.initial_root_password ) - should_verify_tls = config.get("certificate", {}).get("type", "") != "self-signed" + should_verify_tls = config.certificate.type != schema.CertificateEnum.selfsigned try: keycloak_admin = keycloak.KeycloakAdmin( @@ -116,17 +113,14 @@ def get_keycloak_admin_from_config(config): return keycloak_admin -def keycloak_rest_api_call(config=None, request: str = None): +def keycloak_rest_api_call(config: schema.Main = None, request: str = None): """Communicate directly with the Keycloak REST API by passing it a request""" - - config = load_yaml(config) - - keycloak_server_url = f"https://{config['domain']}/auth/" + keycloak_server_url = f"https://{config.domain}/auth/" keycloak_admin_username = os.environ.get("KEYCLOAK_ADMIN_USERNAME", "root") keycloak_admin_password = os.environ.get( "KEYCLOAK_ADMIN_PASSWORD", - config.get("security", {}).get("keycloak", {}).get("initial_root_password", ""), + config.security.keycloak.initial_root_password, ) try: @@ -184,7 +178,7 @@ def keycloak_rest_api_call(config=None, request: str = None): raise e -def export_keycloak_users(config, realm): +def export_keycloak_users(config: schema.Main, realm: str): request = f"GET /{realm}/users" users = keycloak_rest_api_call(config, request=request) diff --git a/src/_nebari/render.py b/src/_nebari/render.py index 1f00e9305..78438d17a 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -15,10 +15,9 @@ from nebari import schema -def render_template(output_directory, config_filename, dry_run=False): +def render_template(output_directory: pathlib.Path, config: schema.Main, dry_run=False): output_directory = pathlib.Path(output_directory).resolve() - - if output_directory == str(pathlib.Path.home()): + if output_directory == pathlib.Path.home(): print("ERROR: Deploying Nebari in home directory is not advised!") sys.exit(1) @@ -26,29 +25,12 @@ def render_template(output_directory, config_filename, dry_run=False): # into it in remove_existing_renders output_directory.mkdir(exist_ok=True, parents=True) - if not config_filename.is_file(): - raise ValueError( - f"cookiecutter configuration={config_filename} is not filename" - ) - - with open(config_filename) as f: - yaml = YAML(typ="safe", pure=True) - config = yaml.load(f) - - # For any config values that start with - # NEBARI_SECRET_, set the values using the - # corresponding env var. - set_env_vars_in_config(config) - contents = {} - config = schema.Main(**config) for stage in get_available_stages(): contents.update( stage(output_directory=output_directory, config=config).render() ) - print(contents.keys()) - new, untracked, updated, deleted = inspect_files( output_base_dir=str(output_directory), ignore_filenames=[ @@ -189,67 +171,3 @@ def hash_file(file_path: str): """ with open(file_path, "rb") as f: return hashlib.sha256(f.read()).hexdigest() - - -def set_env_vars_in_config(config): - """ - - For values in the config starting with 'NEBARI_SECRET_XXX' the environment - variables are searched for the pattern XXX and the config value is - modified. This enables setting secret values that should not be directly - stored in the config file. - - NOTE: variables are most likely written to a file somewhere upon render. In - order to further reduce risk of exposure of any of these variables you might - consider preventing storage of the terraform render output. - """ - private_entries = get_secret_config_entries(config) - for idx in private_entries: - set_nebari_secret(config, idx) - - -def get_secret_config_entries(config, config_idx=None, private_entries=None): - output = private_entries or [] - if config_idx is None: - sub_dict = config - config_idx = [] - else: - sub_dict = get_sub_config(config, config_idx) - - for key, value in sub_dict.items(): - if type(value) is dict: - sub_dict_outputs = get_secret_config_entries( - config, [*config_idx, key], private_entries - ) - output = [*output, *sub_dict_outputs] - else: - if "NEBARI_SECRET_" in str(value): - output = [*output, [*config_idx, key]] - return output - - -def get_sub_config(conf, conf_idx): - sub_config = functools.reduce(dict.__getitem__, conf_idx, conf) - return sub_config - - -def set_sub_config(conf, conf_idx, value): - get_sub_config(conf, conf_idx[:-1])[conf_idx[-1]] = value - - -def set_nebari_secret(config, idx): - placeholder = get_sub_config(config, idx) - secret_var = get_nebari_secret(placeholder) - set_sub_config(config, idx, secret_var) - - -def get_nebari_secret(secret_var): - env_var = secret_var.lstrip("NEBARI_SECRET_") - val = os.environ.get(env_var) - if not val: - raise EnvironmentError( - f"Since '{secret_var}' was found in the" - " Nebari config, the environment variable" - f" '{env_var}' must be set." - ) - return val diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index 1068e128d..ef1f7af30 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -68,7 +68,7 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]): @contextlib.contextmanager def destroy( - self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool] + self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool], ignore_errors: bool = True ): stage_outputs["stages/" + self.name] = terraform.deploy( directory=str(self.output_directory / self.stage_prefix), @@ -79,11 +79,20 @@ def destroy( terraform_destroy=False, ) yield - status["stages/" + self.name] = _terraform_destroy( - directory=str(output_directory / self.stage_prefix), - input_vars=self.input_vars(stage_outputs), - ignore_errors=True, - ) + try: + terraform.deploy( + directory=str(self.output_directory / self.stage_prefix), + input_vars=self.input_vars(stage_outputs), + terraform_init=True, + terraform_import=True, + terraform_apply=False, + terraform_destroy=True, + ) + status["stages/" + self.name] = True + except terraform.TerraformException as e: + if not ignore_errors: + raise e + status["stages/" + self.name] = False def get_available_stages(): diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index c3dfddd80..5c1e507af 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -1,5 +1,5 @@ from inspect import cleandoc -from typing import Dict, List +from typing import Dict, List, Any from _nebari.provider.cicd.github import gen_nebari_linter, gen_nebari_ops from _nebari.provider.cicd.gitlab import gen_gitlab_ci @@ -7,6 +7,66 @@ from nebari.hookspecs import NebariStage, hookimpl +def check_cloud_credentials(config: schema.Main): + if config.provider == schema.ProviderEnum.gcp: + for variable in {"GOOGLE_CREDENTIALS"}: + if variable not in os.environ: + raise ValueError( + f"""Missing the following required environment variable: {variable}\n + Please see the documentation for more information: {GCP_ENV_DOCS}""" + ) + elif config.provider == schema.ProviderEnum.azure: + for variable in { + "ARM_CLIENT_ID", + "ARM_CLIENT_SECRET", + "ARM_SUBSCRIPTION_ID", + "ARM_TENANT_ID", + }: + if variable not in os.environ: + raise ValueError( + f"""Missing the following required environment variable: {variable}\n + Please see the documentation for more information: {AZURE_ENV_DOCS}""" + ) + elif config.provider == schema.ProviderEnum.aws: + for variable in { + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + }: + if variable not in os.environ: + raise ValueError( + f"""Missing the following required environment variable: {variable}\n + Please see the documentation for more information: {AWS_ENV_DOCS}""" + ) + elif config.provider == schema.ProviderEnum.do: + for variable in { + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "SPACES_ACCESS_KEY_ID", + "SPACES_SECRET_ACCESS_KEY", + "DIGITALOCEAN_TOKEN", + }: + if variable not in os.environ: + raise ValueError( + f"""Missing the following required environment variable: {variable}\n + Please see the documentation for more information: {DO_ENV_DOCS}""" + ) + + if os.environ["AWS_ACCESS_KEY_ID"] != os.environ["SPACES_ACCESS_KEY_ID"]: + raise ValueError( + f"""The environment variables AWS_ACCESS_KEY_ID and SPACES_ACCESS_KEY_ID must be equal\n + See {DO_ENV_DOCS} for more information""" + ) + + if ( + os.environ["AWS_SECRET_ACCESS_KEY"] + != os.environ["SPACES_SECRET_ACCESS_KEY"] + ): + raise ValueError( + f"""The environment variables AWS_SECRET_ACCESS_KEY and SPACES_SECRET_ACCESS_KEY must be equal\n + See {DO_ENV_DOCS} for more information""" + ) + + def gen_gitignore(): """ Generate `.gitignore` file. @@ -74,6 +134,9 @@ def render(self) -> Dict[str, str]: contents.update(gen_gitignore()) return contents + def check(self, stage_outputs: Dict[str, Dict[str, Any]]): + check_cloud_credentials(self.config) + @hookimpl def nebari_stage() -> List[NebariStage]: diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 41e820acd..fab0ba6ed 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -3,7 +3,7 @@ import pathlib import sys import tempfile -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Union, Optional from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( @@ -17,7 +17,7 @@ def get_kubeconfig_filename(): - return pathlib.Path(tempfile.gettempdir()) / "NEBARI_KUBECONFIG" + return str(pathlib.Path(tempfile.gettempdir()) / "NEBARI_KUBECONFIG") # TODO: @@ -25,8 +25,8 @@ def get_kubeconfig_filename(): class LocalInputVars(schema.Base): - kubeconfig_filename: Union[str, pathlib.Path] = get_kubeconfig_filename() - kube_context: str + kubeconfig_filename: str = get_kubeconfig_filename() + kube_context: Optional[str] class ExistingInputVars(schema.Base): diff --git a/src/_nebari/stages/kubernetes_keycloak/__init__.py b/src/_nebari/stages/kubernetes_keycloak/__init__.py index 11cb6f50f..50b98359d 100644 --- a/src/_nebari/stages/kubernetes_keycloak/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak/__init__.py @@ -53,7 +53,7 @@ def _calculate_node_groups(config: schema.Main): elif config.provider == schema.ProviderEnum.existing: return config.existing.node_selectors else: - return config.local.node_selectors + return config.local.dict()['node_selectors'] class KubernetesKeycloakStage(NebariTerraformStage): diff --git a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py index 68651568e..3426f314c 100644 --- a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py @@ -23,7 +23,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): return { "realm": realm_id, "realm_display_name": self.config.security.keycloak.realm_display_name, - "authentication": self.config.security.authentication, + "authentication": self.config.security.authentication.dict(), "keycloak_groups": ["superadmin", "admin", "developer", "analyst"] + users_group, "default_groups": ["analyst"] + users_group, diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index 03ef04121..6e3c91005 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -1,3 +1,4 @@ +import json import sys from typing import Any, Dict, List from urllib.parse import urlencode @@ -45,7 +46,7 @@ def _calculate_node_groups(config: schema.Main): elif config.provider == schema.ProviderEnum.existing: return config.existing.node_selectors else: - return config.local.node_selectors + return config.local.dict()['node_selectors'] class KubernetesServicesStage(NebariTerraformStage): @@ -60,7 +61,7 @@ def tf_objects(self) -> List[Dict]: ] def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): - final_logout_uri = f"https://{config['domain']}/hub/login" + final_logout_uri = f"https://{self.config.domain}/hub/login" # Compound any logout URLs from extensions so they are are logged out in succession # when Keycloak and JupyterHub are logged out @@ -86,7 +87,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): ]["value"], "node_groups": _calculate_node_groups(self.config), # conda-store - "conda-store-environments": self.config.environments, + "conda-store-environments": {k: v.dict() for k, v in self.config.environments.items()}, "conda-store-filesystem-storage": self.config.storage.conda_store, "conda-store-service-token-scopes": { "cdsdashboards": { @@ -107,8 +108,8 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "conda-store-extra-config": self.config.conda_store.extra_config, "conda-store-image-tag": self.config.conda_store.image_tag, # jupyterhub - "cdsdashboards": self.config.cdsdashboards, - "jupyterhub-theme": jupyterhub_theme, + "cdsdashboards": self.config.cdsdashboards.dict(), + "jupyterhub-theme": self.config.theme.jupyterhub.dict(), "jupyterhub-image": _split_docker_image_name( self.config.default_images.jupyterhub ), @@ -116,21 +117,21 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "jupyterhub-shared-endpoint": stage_outputs["stages/02-infrastructure"] .get("nfs_endpoint", {}) .get("value"), - "jupyterlab-profiles": self.config.profiles.jupyterlab, + "jupyterlab-profiles": self.config.profiles.dict()['jupyterlab'], "jupyterlab-image": _split_docker_image_name( self.config.default_images.jupyterlab ), "jupyterhub-overrides": [json.dumps(self.config.jupyterhub.overrides)], "jupyterhub-hub-extraEnv": json.dumps( - self.config.jupyterhub.overrides.hub.extraEnv + self.config.jupyterhub.overrides.get('hub', {}).get('extraEnv', []) ), # jupyterlab - "idle-culler-settings": self.config.jupyterlab.idle_culler, + "idle-culler-settings": self.config.jupyterlab.idle_culler.dict(), # dask-gateway "dask-worker-image": _split_docker_image_name( self.config.default_images.dask_worker ), - "dask-gateway-profiles": self.config.profiles.dask_worker, + "dask-gateway-profiles": self.config.profiles.dict()['dask_worker'], # monitoring "monitoring-enabled": self.config.monitoring.enabled, # argo-worfklows diff --git a/src/_nebari/stages/nebari_tf_extensions/__init__.py b/src/_nebari/stages/nebari_tf_extensions/__init__.py index 47d37a71e..ca50def3a 100644 --- a/src/_nebari/stages/nebari_tf_extensions/__init__.py +++ b/src/_nebari/stages/nebari_tf_extensions/__init__.py @@ -27,12 +27,12 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "realm_id": stage_outputs["stages/06-kubernetes-keycloak-configuration"][ "realm_id" ]["value"], - "tf_extensions": self.config.tf_extensions, - "nebari_config_yaml": self.config, + "tf_extensions": [_.dict() for _ in self.config.tf_extensions], + "nebari_config_yaml": self.config.dict(), "keycloak_nebari_bot_password": stage_outputs[ "stages/05-kubernetes-keycloak" ]["keycloak_nebari_bot_password"]["value"], - "helm_extensions": self.config.helm_extensions, + "helm_extensions": [_.dict() for _ in self.config.helm_extensions], } diff --git a/src/_nebari/subcommands/deploy.py b/src/_nebari/subcommands/deploy.py index 881e0839b..86b551697 100644 --- a/src/_nebari/subcommands/deploy.py +++ b/src/_nebari/subcommands/deploy.py @@ -13,13 +13,13 @@ def nebari_subcommand(cli: typer.Typer): @cli.command() def deploy( - config: str = typer.Option( + config_filename: pathlib.Path = typer.Option( ..., "--config", "-c", help="nebari configuration yaml file path", ), - output: str = typer.Option( + output_directory: pathlib.Path = typer.Option( "./", "-o", "--output", @@ -59,22 +59,13 @@ def deploy( """ Deploy the Nebari cluster from your [purple]nebari-config.yaml[/purple] file. """ - config_filename = pathlib.Path(config) - - if not config_filename.is_file(): - raise ValueError( - f"passed in configuration filename={config_filename} must exist" - ) - - config_yaml = load_yaml(config_filename) - - schema.verify(config_yaml) + config = schema.read_configuration(config_filename) if not disable_render: - render_template(output, config) + render_template(output_directory, config) deploy_configuration( - config_yaml, + config, dns_provider=dns_provider, dns_auto_provision=dns_auto_provision, disable_prompt=disable_prompt, diff --git a/src/_nebari/subcommands/destroy.py b/src/_nebari/subcommands/destroy.py index ee27d1044..e47ec17c4 100644 --- a/src/_nebari/subcommands/destroy.py +++ b/src/_nebari/subcommands/destroy.py @@ -11,12 +11,12 @@ @hookimpl def nebari_subcommand(cli: typer.Typer): - @app.command() + @cli.command() def destroy( - config: str = typer.Option( + config_filename: pathlib.Path = typer.Option( ..., "-c", "--config", help="nebari configuration file path" ), - output: str = typer.Option( + output_directory: pathlib.Path = typer.Option( "./", "-o", "--output", @@ -36,22 +36,13 @@ def destroy( """ Destroy the Nebari cluster from your [purple]nebari-config.yaml[/purple] file. """ - - def _run_destroy(config=config, disable_render=disable_render): - config_filename = pathlib.Path(config) - if not config_filename.is_file(): - raise ValueError( - f"passed in configuration filename={config_filename} must exist" - ) - - config_yaml = load_yaml(config_filename) - - schema.verify(config_yaml) + def _run_destroy(config_filename=config_filename, disable_render=disable_render): + config = schema.read_configuration(config_filename) if not disable_render: - render_template(output, config) + render_template(output_directory, config) - destroy_configuration(config_yaml) + destroy_configuration(config) if disable_prompt: _run_destroy() diff --git a/src/_nebari/subcommands/dev.py b/src/_nebari/subcommands/dev.py index 215dcaa4e..b47085fc3 100644 --- a/src/_nebari/subcommands/dev.py +++ b/src/_nebari/subcommands/dev.py @@ -1,5 +1,5 @@ import json -from pathlib import Path +import pathlib import typer @@ -25,7 +25,7 @@ def nebari_subcommand(cli: typer.Typer): @app_dev.command(name="keycloak-api") def keycloak_api( - config_filename: str = typer.Option( + config_filename: pathlib.Path = typer.Option( ..., "-c", "--config", @@ -45,9 +45,6 @@ def keycloak_api( Please use this at your own risk. """ - if isinstance(config_filename, str): - config_filename = Path(config_filename) - + config = schema.read_configuration(config_filename) r = keycloak_rest_api_call(config_filename, request=request) - print(json.dumps(r, indent=4)) diff --git a/src/_nebari/subcommands/keycloak.py b/src/_nebari/subcommands/keycloak.py index 6f33c6788..b64a6c2b5 100644 --- a/src/_nebari/subcommands/keycloak.py +++ b/src/_nebari/subcommands/keycloak.py @@ -1,5 +1,5 @@ import json -from pathlib import Path +import pathlib from typing import Tuple import typer @@ -29,7 +29,7 @@ def add_user( add_users: Tuple[str, str] = typer.Option( ..., "--user", help="Provide both: " ), - config_filename: str = typer.Option( + config_filename: pathlib.Path = typer.Option( ..., "-c", "--config", @@ -37,16 +37,13 @@ def add_user( ), ): """Add a user to Keycloak. User will be automatically added to the [italic]analyst[/italic] group.""" - if isinstance(config_filename, str): - config_filename = Path(config_filename) - args = ["adduser", add_users[0], add_users[1]] - - do_keycloak(config_filename, *args) + config = schema.read_configuration(config_filename) + do_keycloak(config, *args) @app_keycloak.command(name="listusers") def list_users( - config_filename: str = typer.Option( + config_filename: pathlib.Path = typer.Option( ..., "-c", "--config", @@ -54,16 +51,13 @@ def list_users( ) ): """List the users in Keycloak.""" - if isinstance(config_filename, str): - config_filename = Path(config_filename) - args = ["listusers"] - - do_keycloak(config_filename, *args) + config = schema.read_configuration(config_filename) + do_keycloak(config, *args) @app_keycloak.command(name="export-users") def export_users( - config_filename: str = typer.Option( + config_filename: pathlib.Path = typer.Option( ..., "-c", "--config", @@ -76,9 +70,6 @@ def export_users( ), ): """Export the users in Keycloak.""" - if isinstance(config_filename, str): - config_filename = Path(config_filename) - - r = export_keycloak_users(config_filename, realm=realm) - + config = schema.read_configuration(config_filename) + r = export_keycloak_users(config, realm=realm) print(json.dumps(r, indent=4)) diff --git a/src/_nebari/subcommands/render.py b/src/_nebari/subcommands/render.py index 3760c089f..837ad22e6 100644 --- a/src/_nebari/subcommands/render.py +++ b/src/_nebari/subcommands/render.py @@ -3,7 +3,6 @@ import typer from _nebari.render import render_template -from _nebari.utils import load_yaml from nebari import schema from nebari.hookspecs import hookimpl @@ -12,13 +11,13 @@ def nebari_subcommand(cli: typer.Typer): @cli.command(rich_help_panel="Additional Commands") def render( - output: str = typer.Option( + output_directory: pathlib.Path = typer.Option( "./", "-o", "--output", help="output directory", ), - config: str = typer.Option( + config_filename: pathlib.Path = typer.Option( ..., "-c", "--config", @@ -33,15 +32,5 @@ def render( """ Dynamically render the Terraform scripts and other files from your [purple]nebari-config.yaml[/purple] file. """ - config_filename = pathlib.Path(config) - - if not config_filename.is_file(): - raise ValueError( - f"passed in configuration filename={config_filename} must exist" - ) - - config_yaml = load_yaml(config_filename) - - schema.verify(config_yaml) - - render_template(output, config, dry_run=dry_run) + config = schema.read_configuration(config_filename) + render_template(output_directory, config, dry_run=dry_run) diff --git a/src/_nebari/subcommands/validate.py b/src/_nebari/subcommands/validate.py index 3b73c5dbf..fbe2a2b52 100644 --- a/src/_nebari/subcommands/validate.py +++ b/src/_nebari/subcommands/validate.py @@ -12,7 +12,7 @@ def nebari_subcommand(cli: typer.Typer): @cli.command(rich_help_panel="Additional Commands") def validate( - config: str = typer.Option( + config_filename: pathlib.Path = typer.Option( ..., "--config", "-c", @@ -25,18 +25,10 @@ def validate( """ Validate the values in the [purple]nebari-config.yaml[/purple] file are acceptable. """ - config_filename = pathlib.Path(config) - if not config_filename.is_file(): - raise ValueError( - f"Passed in configuration filename={config_filename} must exist." - ) - - config = load_yaml(config_filename) - if enable_commenting: # for PR's only # comment_on_pr(config) pass else: - schema.verify(config) + schema.read_configuration(config_filename) print("[bold purple]Successfully validated configuration.[/bold purple]") diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 04c54216b..206c25a24 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -112,71 +112,7 @@ def kill_process(): ) # Should already have finished because we have drained stdout -def check_cloud_credentials(config): - if config["provider"] == "gcp": - for variable in {"GOOGLE_CREDENTIALS"}: - if variable not in os.environ: - raise ValueError( - f"""Missing the following required environment variable: {variable}\n - Please see the documentation for more information: {GCP_ENV_DOCS}""" - ) - elif config["provider"] == "azure": - for variable in { - "ARM_CLIENT_ID", - "ARM_CLIENT_SECRET", - "ARM_SUBSCRIPTION_ID", - "ARM_TENANT_ID", - }: - if variable not in os.environ: - raise ValueError( - f"""Missing the following required environment variable: {variable}\n - Please see the documentation for more information: {AZURE_ENV_DOCS}""" - ) - elif config["provider"] == "aws": - for variable in { - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - }: - if variable not in os.environ: - raise ValueError( - f"""Missing the following required environment variable: {variable}\n - Please see the documentation for more information: {AWS_ENV_DOCS}""" - ) - elif config["provider"] == "do": - for variable in { - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "SPACES_ACCESS_KEY_ID", - "SPACES_SECRET_ACCESS_KEY", - "DIGITALOCEAN_TOKEN", - }: - if variable not in os.environ: - raise ValueError( - f"""Missing the following required environment variable: {variable}\n - Please see the documentation for more information: {DO_ENV_DOCS}""" - ) - - if os.environ["AWS_ACCESS_KEY_ID"] != os.environ["SPACES_ACCESS_KEY_ID"]: - raise ValueError( - f"""The environment variables AWS_ACCESS_KEY_ID and SPACES_ACCESS_KEY_ID must be equal\n - See {DO_ENV_DOCS} for more information""" - ) - - if ( - os.environ["AWS_SECRET_ACCESS_KEY"] - != os.environ["SPACES_SECRET_ACCESS_KEY"] - ): - raise ValueError( - f"""The environment variables AWS_SECRET_ACCESS_KEY and SPACES_SECRET_ACCESS_KEY must be equal\n - See {DO_ENV_DOCS} for more information""" - ) - elif config["provider"] in ["local", "existing"]: - pass - else: - raise ValueError("Cloud Provider configuration not supported") - - -def load_yaml(config_filename: Path): +def load_yaml(config_filename: pathlib.Path): """ Return yaml dict containing config loaded from config_filename. """ diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index 2fc340743..16ae7a8c1 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -19,7 +19,7 @@ # subcommands "_nebari.subcommands.dev", "_nebari.subcommands.deploy", - # "_nebari.subcommands.destroy", + "_nebari.subcommands.destroy", "_nebari.subcommands.keycloak", "_nebari.subcommands.render", "_nebari.subcommands.support", diff --git a/src/nebari/schema.py b/src/nebari/schema.py index a5c250706..4a3b4c667 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -1,14 +1,21 @@ +import sys +import os +import re import enum import secrets import string import typing +import pathlib +import json from abc import ABC +from ruamel.yaml import YAML import pydantic from pydantic import Field, root_validator, validator from _nebari.utils import namestr_regex, set_docker_image_tag, set_nebari_dask_version from _nebari.version import __version__, rounded_ver_parse +from _nebari import constants class CertificateEnum(str, enum.Enum): @@ -62,6 +69,7 @@ class Base(pydantic.BaseModel): class Config: extra = "forbid" + validate_assignment = True # ============== CI/CD ============= @@ -88,28 +96,28 @@ class HelmExtension(Base): class NebariWorkflowController(Base): - enabled: bool - image_tag: typing.Optional[str] + enabled: bool = True + image_tag: str = constants.DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG class ArgoWorkflows(Base): - enabled: bool - overrides: typing.Optional[typing.Dict] - nebari_workflow_controller: typing.Optional[NebariWorkflowController] + enabled: bool = True + overrides: typing.Dict = {} + nebari_workflow_controller: NebariWorkflowController = NebariWorkflowController() # ============== kbatch ============= class KBatch(Base): - enabled: bool + enabled: bool = True # ============== Monitoring ============= class Monitoring(Base): - enabled: bool + enabled: bool = True # ============== ClearML ============= @@ -117,7 +125,7 @@ class Monitoring(Base): class ClearML(Base): enabled: bool = False - enable_forward_auth: typing.Optional[bool] + enable_forward_auth: bool = False overrides: typing.Dict = {} @@ -128,25 +136,26 @@ class Prefect(Base): enabled: bool = False image: typing.Optional[str] overrides: typing.Dict = {} + token: typing.Optional[str] # =========== Conda-Store ============== class CondaStore(Base): - extra_settings: typing.Optional[typing.Dict[str, typing.Any]] = {} - extra_config: typing.Optional[str] = "" - image_tag: typing.Optional[str] = "" - default_namespace: typing.Optional[str] = "" + extra_settings: typing.Dict[str, typing.Any] = {} + extra_config: str = "" + image_tag: str = constants.DEFAULT_CONDA_STORE_IMAGE_TAG + default_namespace: str = "nebari-git" # ============= Terraform =============== class TerraformState(Base): - type: TerraformStateEnum + type: TerraformStateEnum = TerraformStateEnum.remote backend: typing.Optional[str] - config: typing.Optional[typing.Dict[str, str]] + config: typing.Dict[str, str] = {} # ============ Certificate ============= @@ -399,38 +408,47 @@ class ExistingProvider(Base): # ================= Theme ================== +class JupyterHubTheme(Base): + hub_title: str = "Nebari", + hub_subtitle: str = "Your open source data science platform", + welcome: str = """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""", + logo: str = "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg", + primary_color: str = '#4f4173' + secondary_color: str = '#957da6' + accent_color: str = '#32C574' + text_color: str = '#111111' + h1_color: str = '#652e8e' + h2_color: str = '#652e8e' + version: str = f"v{__version__}" + display_version: bool = True + + class Theme(Base): - jupyterhub: typing.Dict[str, typing.Union[str, list]] = dict( - hub_title="Nebari", - hub_subtitle="Your open source data science platform", - welcome="""Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""", - logo="https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg", - display_version=True, - ) + jupyterhub: JupyterHubTheme = JupyterHubTheme() # ================= Theme ================== class JupyterHub(Base): - overrides: typing.Optional[typing.Dict] + overrides: typing.Dict = {} # ================= JupyterLab ================== class IdleCuller(Base): - terminal_cull_inactive_timeout: typing.Optional[int] - terminal_cull_interval: typing.Optional[int] - kernel_cull_idle_timeout: typing.Optional[int] - kernel_cull_interval: typing.Optional[int] - kernel_cull_connected: typing.Optional[bool] - kernel_cull_busy: typing.Optional[int] - server_shutdown_no_activity_timeout: typing.Optional[int] + terminal_cull_inactive_timeout: int = 15 + terminal_cull_interval: int = 5 + kernel_cull_idle_timeout: int = 15 + kernel_cull_interval: int = 5 + kernel_cull_connected: bool = True + kernel_cull_busy: bool = False + server_shutdown_no_activity_timeout: int = 15 class JupyterLab(Base): - idle_culler: typing.Optional[IdleCuller] + idle_culler: IdleCuller = IdleCuller() # ================== Profiles ================== @@ -572,7 +590,7 @@ class NebariExtension(Base): class Ingress(Base): - terraform_overrides: typing.Any + terraform_overrides: typing.Dict = {} # ======== External Container Registry ======== @@ -678,7 +696,7 @@ class Main(Base): nebari_version: str = __version__ ci_cd: CICD = CICD() domain: str - terraform_state: typing.Optional[TerraformState] + terraform_state: TerraformState = TerraformState() certificate: Certificate = Certificate() helm_extensions: typing.List[HelmExtension] = [] prefect: Prefect = Prefect() @@ -762,18 +780,18 @@ class Main(Base): ], ), } - conda_store: typing.Optional[CondaStore] - argo_workflows: typing.Optional[ArgoWorkflows] - kbatch: typing.Optional[KBatch] - monitoring: typing.Optional[Monitoring] - clearml: typing.Optional[ClearML] - tf_extensions: typing.Optional[typing.List[NebariExtension]] - jupyterhub: typing.Optional[JupyterHub] - jupyterlab: typing.Optional[JupyterLab] + conda_store: CondaStore = CondaStore() + argo_workflows: ArgoWorkflows = ArgoWorkflows() + kbatch: KBatch = KBatch() + monitoring: Monitoring = Monitoring() + clearml: ClearML = ClearML() + tf_extensions: typing.List[NebariExtension] = [] + jupyterhub: JupyterHub = JupyterHub() + jupyterlab: JupyterLab = JupyterLab() prevent_deploy: bool = ( False # Optional, but will be given default value if not present ) - ingress: typing.Optional[Ingress] + ingress: Ingress = Ingress() # If the nebari_version in the schema is old # we must tell the user to first run nebari upgrade @@ -801,10 +819,6 @@ def _project_name_convention(cls, value: typing.Any, values): return value -def verify(config): - return Main(**config) - - def is_version_accepted(v): """ Given a version string, return boolean indicating whether @@ -812,3 +826,79 @@ def is_version_accepted(v): for deployment with the current Nebari package. """ return Main.is_version_accepted(v) + + +def set_nested_attribute(data: typing.Any, attrs: typing.List[str], value: typing.Any): + """Takes an arbitrary set of attributes and accesses the deep + nested object config to set value + + """ + def _get_attr(d: typing.Any, attr: str): + if hasattr(d, '__getitem__'): + if re.fullmatch('\d+', attr): + try: + return d[int(attr)] + except Exception: + return d[attr] + else: + return d[attr] + else: + return getattr(d, attr) + + def _set_attr(d: typing.Any, attr: str, value: typing.Any): + if hasattr(d, '__getitem__'): + if re.fullmatch('\d+', attr): + try: + d[int(attr)] = value + except Exception: + d[attr] = value + else: + d[attr] = value + else: + return setattr(d, attr, value) + + data_pos = data + for attr in attrs[:-1]: + data_pos = _get_attr(data_pos, attr) + _set_attr(data_pos, attrs[-1], value) + + +def set_config_from_environment_variables(config: Main, keyword: str = 'NEBARI_SECRET', separator: str = "__"): + """Setting nebari configuration values from environment variables + + For example `NEBARI_SECRET__ci_cd__branch=master` would set `ci_cd.branch = "master"` + """ + nebari_secrets = [_ for _ in os.environ if _.startswith(keyword + separator)] + for secret in nebari_secrets: + attrs = secret[len(keyword + separator):].split(separator) + try: + set_nested_attribute(config, attrs, os.environ[secret]) + except Exception as e: + print(f'FAILED: setting secret from environment variable={secret} due to the following error\n {e}') + sys.exit(1) + return config + + + +def read_configuration(config_filename: pathlib.Path, read_environment: bool = True): + """Read configuration from multiple sources and apply validation + + """ + filename = pathlib.Path(config_filename) + + yaml = YAML() + yaml.preserve_quotes = True + yaml.default_flow_style = False + + if not filename.is_file(): + raise ValueError( + f"passed in configuration filename={config_filename} does not exist" + ) + + with filename.open() as f: + config = Main(**yaml.load(f.read())) + + if read_environment: + config = set_config_from_environment_variables(config) + + return config From c3b794d7044b8c9e311a5f8c4961d9145e911faf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Jun 2023 20:46:59 +0000 Subject: [PATCH 015/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/deploy.py | 2 +- src/_nebari/destroy.py | 6 +- src/_nebari/keycloak.py | 4 +- src/_nebari/render.py | 2 - src/_nebari/stages/base.py | 5 +- src/_nebari/stages/bootstrap/__init__.py | 2 +- src/_nebari/stages/infrastructure/__init__.py | 2 +- .../stages/kubernetes_keycloak/__init__.py | 2 +- .../stages/kubernetes_services/__init__.py | 12 ++-- src/_nebari/subcommands/deploy.py | 1 - src/_nebari/subcommands/destroy.py | 6 +- src/_nebari/subcommands/dev.py | 2 +- src/_nebari/subcommands/validate.py | 1 - src/nebari/schema.py | 59 ++++++++++--------- 14 files changed, 54 insertions(+), 52 deletions(-) diff --git a/src/_nebari/deploy.py b/src/_nebari/deploy.py index c4bb30fc6..964a3f87e 100644 --- a/src/_nebari/deploy.py +++ b/src/_nebari/deploy.py @@ -71,7 +71,7 @@ def deploy_configuration( ) ) - logger.info(f'All nebari endpoints will be under https://{config.domain}') + logger.info(f"All nebari endpoints will be under https://{config.domain}") if disable_checks: logger.warning( diff --git a/src/_nebari/destroy.py b/src/_nebari/destroy.py index 61cc68c07..1b5d5271a 100644 --- a/src/_nebari/destroy.py +++ b/src/_nebari/destroy.py @@ -1,11 +1,7 @@ -from typing import List, Dict -import functools -import logging -import os import contextlib +import logging import pathlib -from _nebari.provider import terraform from _nebari.stages.base import get_available_stages from _nebari.utils import timer from nebari import schema diff --git a/src/_nebari/keycloak.py b/src/_nebari/keycloak.py index 1f5709d6e..77b0ae46e 100644 --- a/src/_nebari/keycloak.py +++ b/src/_nebari/keycloak.py @@ -7,7 +7,6 @@ import requests import rich -from _nebari.utils import load_yaml from nebari import schema logger = logging.getLogger(__name__) @@ -88,8 +87,7 @@ def get_keycloak_admin_from_config(config: schema.Main): keycloak_username = os.environ.get("KEYCLOAK_ADMIN_USERNAME", "root") keycloak_password = os.environ.get( - "KEYCLOAK_ADMIN_PASSWORD", - config.security.keycloak.initial_root_password + "KEYCLOAK_ADMIN_PASSWORD", config.security.keycloak.initial_root_password ) should_verify_tls = config.certificate.type != schema.CertificateEnum.selfsigned diff --git a/src/_nebari/render.py b/src/_nebari/render.py index 78438d17a..22782133a 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -1,4 +1,3 @@ -import functools import hashlib import os import shutil @@ -8,7 +7,6 @@ from rich import print from rich.table import Table -from ruamel.yaml import YAML from _nebari.deprecate import DEPRECATED_FILE_PATHS from _nebari.stages.base import get_available_stages diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index ef1f7af30..df16612d0 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -68,7 +68,10 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]): @contextlib.contextmanager def destroy( - self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool], ignore_errors: bool = True + self, + stage_outputs: Dict[str, Dict[str, Any]], + status: Dict[str, bool], + ignore_errors: bool = True, ): stage_outputs["stages/" + self.name] = terraform.deploy( directory=str(self.output_directory / self.stage_prefix), diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index 5c1e507af..df580e98b 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -1,5 +1,5 @@ from inspect import cleandoc -from typing import Dict, List, Any +from typing import Any, Dict, List from _nebari.provider.cicd.github import gen_nebari_linter, gen_nebari_ops from _nebari.provider.cicd.gitlab import gen_gitlab_ci diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index fab0ba6ed..3d437eccd 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -3,7 +3,7 @@ import pathlib import sys import tempfile -from typing import Any, Dict, List, Union, Optional +from typing import Any, Dict, List, Optional from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( diff --git a/src/_nebari/stages/kubernetes_keycloak/__init__.py b/src/_nebari/stages/kubernetes_keycloak/__init__.py index 50b98359d..210794069 100644 --- a/src/_nebari/stages/kubernetes_keycloak/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak/__init__.py @@ -53,7 +53,7 @@ def _calculate_node_groups(config: schema.Main): elif config.provider == schema.ProviderEnum.existing: return config.existing.node_selectors else: - return config.local.dict()['node_selectors'] + return config.local.dict()["node_selectors"] class KubernetesKeycloakStage(NebariTerraformStage): diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index 6e3c91005..c224709b3 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -46,7 +46,7 @@ def _calculate_node_groups(config: schema.Main): elif config.provider == schema.ProviderEnum.existing: return config.existing.node_selectors else: - return config.local.dict()['node_selectors'] + return config.local.dict()["node_selectors"] class KubernetesServicesStage(NebariTerraformStage): @@ -87,7 +87,9 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): ]["value"], "node_groups": _calculate_node_groups(self.config), # conda-store - "conda-store-environments": {k: v.dict() for k, v in self.config.environments.items()}, + "conda-store-environments": { + k: v.dict() for k, v in self.config.environments.items() + }, "conda-store-filesystem-storage": self.config.storage.conda_store, "conda-store-service-token-scopes": { "cdsdashboards": { @@ -117,13 +119,13 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "jupyterhub-shared-endpoint": stage_outputs["stages/02-infrastructure"] .get("nfs_endpoint", {}) .get("value"), - "jupyterlab-profiles": self.config.profiles.dict()['jupyterlab'], + "jupyterlab-profiles": self.config.profiles.dict()["jupyterlab"], "jupyterlab-image": _split_docker_image_name( self.config.default_images.jupyterlab ), "jupyterhub-overrides": [json.dumps(self.config.jupyterhub.overrides)], "jupyterhub-hub-extraEnv": json.dumps( - self.config.jupyterhub.overrides.get('hub', {}).get('extraEnv', []) + self.config.jupyterhub.overrides.get("hub", {}).get("extraEnv", []) ), # jupyterlab "idle-culler-settings": self.config.jupyterlab.idle_culler.dict(), @@ -131,7 +133,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "dask-worker-image": _split_docker_image_name( self.config.default_images.dask_worker ), - "dask-gateway-profiles": self.config.profiles.dict()['dask_worker'], + "dask-gateway-profiles": self.config.profiles.dict()["dask_worker"], # monitoring "monitoring-enabled": self.config.monitoring.enabled, # argo-worfklows diff --git a/src/_nebari/subcommands/deploy.py b/src/_nebari/subcommands/deploy.py index 86b551697..18214a28d 100644 --- a/src/_nebari/subcommands/deploy.py +++ b/src/_nebari/subcommands/deploy.py @@ -4,7 +4,6 @@ from _nebari.deploy import deploy_configuration from _nebari.render import render_template -from _nebari.utils import load_yaml from nebari import schema from nebari.hookspecs import hookimpl diff --git a/src/_nebari/subcommands/destroy.py b/src/_nebari/subcommands/destroy.py index e47ec17c4..d38fb5736 100644 --- a/src/_nebari/subcommands/destroy.py +++ b/src/_nebari/subcommands/destroy.py @@ -4,7 +4,6 @@ from _nebari.destroy import destroy_configuration from _nebari.render import render_template -from _nebari.utils import load_yaml from nebari import schema from nebari.hookspecs import hookimpl @@ -36,7 +35,10 @@ def destroy( """ Destroy the Nebari cluster from your [purple]nebari-config.yaml[/purple] file. """ - def _run_destroy(config_filename=config_filename, disable_render=disable_render): + + def _run_destroy( + config_filename=config_filename, disable_render=disable_render + ): config = schema.read_configuration(config_filename) if not disable_render: diff --git a/src/_nebari/subcommands/dev.py b/src/_nebari/subcommands/dev.py index b47085fc3..296d3d219 100644 --- a/src/_nebari/subcommands/dev.py +++ b/src/_nebari/subcommands/dev.py @@ -45,6 +45,6 @@ def keycloak_api( Please use this at your own risk. """ - config = schema.read_configuration(config_filename) + schema.read_configuration(config_filename) r = keycloak_rest_api_call(config_filename, request=request) print(json.dumps(r, indent=4)) diff --git a/src/_nebari/subcommands/validate.py b/src/_nebari/subcommands/validate.py index fbe2a2b52..062cec8a3 100644 --- a/src/_nebari/subcommands/validate.py +++ b/src/_nebari/subcommands/validate.py @@ -3,7 +3,6 @@ import typer from rich import print -from _nebari.utils import load_yaml from nebari import schema from nebari.hookspecs import hookimpl diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 4a3b4c667..807feff9f 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -1,21 +1,20 @@ -import sys +import enum import os +import pathlib import re -import enum import secrets import string +import sys import typing -import pathlib -import json from abc import ABC -from ruamel.yaml import YAML import pydantic from pydantic import Field, root_validator, validator +from ruamel.yaml import YAML +from _nebari import constants from _nebari.utils import namestr_regex, set_docker_image_tag, set_nebari_dask_version from _nebari.version import __version__, rounded_ver_parse -from _nebari import constants class CertificateEnum(str, enum.Enum): @@ -409,16 +408,20 @@ class ExistingProvider(Base): class JupyterHubTheme(Base): - hub_title: str = "Nebari", - hub_subtitle: str = "Your open source data science platform", - welcome: str = """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""", - logo: str = "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg", - primary_color: str = '#4f4173' - secondary_color: str = '#957da6' - accent_color: str = '#32C574' - text_color: str = '#111111' - h1_color: str = '#652e8e' - h2_color: str = '#652e8e' + hub_title: str = ("Nebari",) + hub_subtitle: str = ("Your open source data science platform",) + welcome: str = ( + """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""", + ) + logo: str = ( + "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg", + ) + primary_color: str = "#4f4173" + secondary_color: str = "#957da6" + accent_color: str = "#32C574" + text_color: str = "#111111" + h1_color: str = "#652e8e" + h2_color: str = "#652e8e" version: str = f"v{__version__}" display_version: bool = True @@ -833,9 +836,10 @@ def set_nested_attribute(data: typing.Any, attrs: typing.List[str], value: typin nested object config to set value """ + def _get_attr(d: typing.Any, attr: str): - if hasattr(d, '__getitem__'): - if re.fullmatch('\d+', attr): + if hasattr(d, "__getitem__"): + if re.fullmatch("\d+", attr): try: return d[int(attr)] except Exception: @@ -846,8 +850,8 @@ def _get_attr(d: typing.Any, attr: str): return getattr(d, attr) def _set_attr(d: typing.Any, attr: str, value: typing.Any): - if hasattr(d, '__getitem__'): - if re.fullmatch('\d+', attr): + if hasattr(d, "__getitem__"): + if re.fullmatch("\d+", attr): try: d[int(attr)] = value except Exception: @@ -863,27 +867,28 @@ def _set_attr(d: typing.Any, attr: str, value: typing.Any): _set_attr(data_pos, attrs[-1], value) -def set_config_from_environment_variables(config: Main, keyword: str = 'NEBARI_SECRET', separator: str = "__"): +def set_config_from_environment_variables( + config: Main, keyword: str = "NEBARI_SECRET", separator: str = "__" +): """Setting nebari configuration values from environment variables For example `NEBARI_SECRET__ci_cd__branch=master` would set `ci_cd.branch = "master"` """ nebari_secrets = [_ for _ in os.environ if _.startswith(keyword + separator)] for secret in nebari_secrets: - attrs = secret[len(keyword + separator):].split(separator) + attrs = secret[len(keyword + separator) :].split(separator) try: set_nested_attribute(config, attrs, os.environ[secret]) except Exception as e: - print(f'FAILED: setting secret from environment variable={secret} due to the following error\n {e}') + print( + f"FAILED: setting secret from environment variable={secret} due to the following error\n {e}" + ) sys.exit(1) return config - def read_configuration(config_filename: pathlib.Path, read_environment: bool = True): - """Read configuration from multiple sources and apply validation - - """ + """Read configuration from multiple sources and apply validation""" filename = pathlib.Path(config_filename) yaml = YAML() From c9c8440d3ca52e3174451199e467eb30e6bba5c8 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 15 Jun 2023 17:27:50 -0400 Subject: [PATCH 016/147] Fully working deploy with checks again --- pyproject.toml | 2 +- src/_nebari/deploy.py | 3 +++ src/_nebari/stages/infrastructure/__init__.py | 6 +++--- .../stages/kubernetes_ingress/__init__.py | 20 +++++++++---------- .../stages/kubernetes_initialize/__init__.py | 6 +++--- .../stages/kubernetes_keycloak/__init__.py | 13 ++++++++---- .../__init__.py | 15 +++++++++----- .../stages/kubernetes_services/__init__.py | 6 +++--- src/nebari/__main__.py | 6 ++---- 9 files changed, 44 insertions(+), 33 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 85ac6a1e6..eebff1089 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ Documentation = "https://www.nebari.dev/docs" Source = "https://github.com/nebari-dev/nebari" [project.scripts] -nebari = "_nebari.__main__:main" +nebari = "nebari.__main__:main" [tool.ruff] select = [ diff --git a/src/_nebari/deploy.py b/src/_nebari/deploy.py index 964a3f87e..18d2c38a5 100644 --- a/src/_nebari/deploy.py +++ b/src/_nebari/deploy.py @@ -24,6 +24,9 @@ def guided_install( for stage in get_available_stages(): s = stage(output_directory=pathlib.Path("."), config=config) stack.enter_context(s.deploy(stage_outputs)) + + if not disable_checks: + s.check(stage_outputs) print("Nebari deployed successfully") print("Services:") diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 3d437eccd..5b8136b29 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -207,18 +207,18 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]): api_instance = client.CoreV1Api() result = api_instance.list_namespace() except ApiException: - self.log.error( + print( f"ERROR: After stage={self.name} unable to connect to kubernetes cluster" ) sys.exit(1) if len(result.items) < 1: - self.log.error( + print( f"ERROR: After stage={self.name} no nodes provisioned within kubernetes cluster" ) sys.exit(1) - self.log.info( + print( f"After stage={self.name} kubernetes cluster successfully provisioned" ) diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index 1f09f4001..97139d715 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -43,28 +43,28 @@ def provision_ingress_dns( if dns_auto_provision and dns_provider == "cloudflare": record_name, zone_name = ( - config["domain"].split(".")[:-2], - config["domain"].split(".")[-2:], + config.domain.split(".")[:-2], + config.domain.split(".")[-2:], ) record_name = ".".join(record_name) zone_name = ".".join(zone_name) - if config["provider"] in {"do", "gcp", "azure"}: + if config.provider in {schema.ProviderEnum.do, schema.ProviderEnum.gcp, schema.ProviderEnum.azure}: update_record(zone_name, record_name, "A", ip_or_hostname) - if config.get("clearml", {}).get("enabled"): + if config.clearml.enabled: add_clearml_dns(zone_name, record_name, "A", ip_or_hostname) - elif config["provider"] == "aws": + elif config.provider == schema.ProviderEnum.aws: update_record(zone_name, record_name, "CNAME", ip_or_hostname) - if config.get("clearml", {}).get("enabled"): + if config.clearml.enabled: add_clearml_dns(zone_name, record_name, "CNAME", ip_or_hostname) else: logger.info( - f"Couldn't update the DNS record for cloud provider: {config['provider']}" + f"Couldn't update the DNS record for cloud provider: {config.provider}" ) elif not disable_prompt: input( f"Take IP Address {ip_or_hostname} and update DNS to point to " - f'"{config["domain"]}" [Press Enter when Complete]' + f'"{config.domain}" [Press Enter when Complete]' ) if not disable_checks: @@ -103,7 +103,7 @@ def check_ingress_dns(stage_outputs, config, disable_prompt): ip_or_name = stage_outputs[directory]["load_balancer_address"]["value"] ip = socket.gethostbyname(ip_or_name["hostname"] or ip_or_name["ip"]) - domain_name = config["domain"] + domain_name = config.domain def _attempt_dns_lookup( domain_name, ip, num_attempts=NUM_ATTEMPTS, timeout=TIMEOUT @@ -231,7 +231,7 @@ def _attempt_tcp_connect( ) sys.exit(1) - self.log.info( + print( f"After stage={self.name} kubernetes ingress available on tcp ports={tcp_ports}" ) diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py index d2263cc35..4821e70d3 100644 --- a/src/_nebari/stages/kubernetes_initialize/__init__.py +++ b/src/_nebari/stages/kubernetes_initialize/__init__.py @@ -68,19 +68,19 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]): api_instance = client.CoreV1Api() result = api_instance.list_namespace() except ApiException: - self.log.error( + print( f"ERROR: After stage={self.name} unable to connect to kubernetes cluster" ) sys.exit(1) namespaces = {_.metadata.name for _ in result.items} if self.config.namespace not in namespaces: - self.log.error( + print( f"ERROR: After stage={self.name} namespace={self.config.namespace} not provisioned within kubernetes cluster" ) sys.exit(1) - self.log.info(f"After stage={self.name} kubernetes initialized successfully") + print(f"After stage={self.name} kubernetes initialized successfully") @hookimpl diff --git a/src/_nebari/stages/kubernetes_keycloak/__init__.py b/src/_nebari/stages/kubernetes_keycloak/__init__.py index 210794069..58c0f5f41 100644 --- a/src/_nebari/stages/kubernetes_keycloak/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak/__init__.py @@ -14,6 +14,11 @@ from nebari.hookspecs import NebariStage, hookimpl +# check and retry settings +NUM_ATTEMPTS = 10 +TIMEOUT = 10 # seconds + + @contextlib.contextmanager def keycloak_provider_context(keycloak_credentials: Dict[str, str]): credential_mapping = { @@ -103,12 +108,12 @@ def _attempt_keycloak_connection( client_id=client_id, verify=verify, ) - self.log.info( + print( f"Attempt {i+1} succeeded connecting to keycloak master realm" ) return True except KeycloakError: - self.log.info( + print( f"Attempt {i+1} failed connecting to keycloak master realm" ) time.sleep(timeout) @@ -130,12 +135,12 @@ def _attempt_keycloak_connection( ], verify=False, ): - self.log.error( + print( f"ERROR: unable to connect to keycloak master realm at url={keycloak_url} with root credentials" ) sys.exit(1) - self.log.info("Keycloak service successfully started") + print("Keycloak service successfully started") @contextlib.contextmanager def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): diff --git a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py index 3426f314c..31b3d8a34 100644 --- a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py @@ -6,6 +6,11 @@ from nebari.hookspecs import NebariStage, hookimpl +# check and retry settings +NUM_ATTEMPTS = 10 +TIMEOUT = 10 # seconds + + class KubernetesKeycloakConfigurationStage(NebariTerraformStage): name = "06-kubernetes-keycloak-configuration" priority = 60 @@ -62,16 +67,16 @@ def _attempt_keycloak_connection( ) existing_realms = {_["id"] for _ in realm_admin.get_realms()} if nebari_realm in existing_realms: - self.log.info( + print( f"Attempt {i+1} succeeded connecting to keycloak and nebari realm={nebari_realm} exists" ) return True else: - self.log.info( + print( f"Attempt {i+1} succeeded connecting to keycloak but nebari realm did not exist" ) except KeycloakError: - self.log.info( + print( f"Attempt {i+1} failed connecting to keycloak master realm" ) time.sleep(timeout) @@ -88,12 +93,12 @@ def _attempt_keycloak_connection( ]["value"], verify=False, ): - self.log.error( + print( "ERROR: unable to connect to keycloak master realm and ensure that nebari realm exists" ) sys.exit(1) - self.log.info("Keycloak service successfully started with nebari realm") + print("Keycloak service successfully started with nebari realm") @hookimpl diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index c224709b3..eceba6678 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -175,10 +175,10 @@ def _attempt_connect_url( for i in range(num_attempts): response = requests.get(url, verify=verify, timeout=timeout) if response.status_code < 400: - self.log.info(f"Attempt {i+1} health check succeeded for url={url}") + print(f"Attempt {i+1} health check succeeded for url={url}") return True else: - self.log.info(f"Attempt {i+1} health check failed for url={url}") + print(f"Attempt {i+1} health check failed for url={url}") time.sleep(timeout) return False @@ -186,7 +186,7 @@ def _attempt_connect_url( for service_name, service in services.items(): service_url = service["health_url"] if service_url and not _attempt_connect_url(service_url): - self.log.error( + print( f"ERROR: Service {service_name} DOWN when checking url={service_url}" ) sys.exit(1) diff --git a/src/nebari/__main__.py b/src/nebari/__main__.py index 09030b4cb..29d39600d 100644 --- a/src/nebari/__main__.py +++ b/src/nebari/__main__.py @@ -1,6 +1,4 @@ -import sys - -from _nebari.cli.main import app as main +from _nebari.__main__ import main if __name__ == "__main__": - main(sys.argv[1:]) + main() From e45e2c0c37ebee4df544d6eca669609e47ba7add Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Jun 2023 21:28:13 +0000 Subject: [PATCH 017/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/stages/infrastructure/__init__.py | 4 +--- src/_nebari/stages/kubernetes_ingress/__init__.py | 6 +++++- src/_nebari/stages/kubernetes_keycloak/__init__.py | 5 +---- .../stages/kubernetes_keycloak_configuration/__init__.py | 5 +---- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 5b8136b29..1cecb4012 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -218,9 +218,7 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]): ) sys.exit(1) - print( - f"After stage={self.name} kubernetes cluster successfully provisioned" - ) + print(f"After stage={self.name} kubernetes cluster successfully provisioned") @contextlib.contextmanager def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index 97139d715..40b4909b7 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -48,7 +48,11 @@ def provision_ingress_dns( ) record_name = ".".join(record_name) zone_name = ".".join(zone_name) - if config.provider in {schema.ProviderEnum.do, schema.ProviderEnum.gcp, schema.ProviderEnum.azure}: + if config.provider in { + schema.ProviderEnum.do, + schema.ProviderEnum.gcp, + schema.ProviderEnum.azure, + }: update_record(zone_name, record_name, "A", ip_or_hostname) if config.clearml.enabled: add_clearml_dns(zone_name, record_name, "A", ip_or_hostname) diff --git a/src/_nebari/stages/kubernetes_keycloak/__init__.py b/src/_nebari/stages/kubernetes_keycloak/__init__.py index 58c0f5f41..acad8517e 100644 --- a/src/_nebari/stages/kubernetes_keycloak/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak/__init__.py @@ -13,7 +13,6 @@ from nebari import schema from nebari.hookspecs import NebariStage, hookimpl - # check and retry settings NUM_ATTEMPTS = 10 TIMEOUT = 10 # seconds @@ -113,9 +112,7 @@ def _attempt_keycloak_connection( ) return True except KeycloakError: - print( - f"Attempt {i+1} failed connecting to keycloak master realm" - ) + print(f"Attempt {i+1} failed connecting to keycloak master realm") time.sleep(timeout) return False diff --git a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py index 31b3d8a34..ac131e983 100644 --- a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py @@ -5,7 +5,6 @@ from _nebari.stages.tf_objects import NebariTerraformState from nebari.hookspecs import NebariStage, hookimpl - # check and retry settings NUM_ATTEMPTS = 10 TIMEOUT = 10 # seconds @@ -76,9 +75,7 @@ def _attempt_keycloak_connection( f"Attempt {i+1} succeeded connecting to keycloak but nebari realm did not exist" ) except KeycloakError: - print( - f"Attempt {i+1} failed connecting to keycloak master realm" - ) + print(f"Attempt {i+1} failed connecting to keycloak master realm") time.sleep(timeout) return False From 7ac3609b913868956f114f2421b085fbd8c26a5c Mon Sep 17 00:00:00 2001 From: eskild <42120229+iameskild@users.noreply.github.com> Date: Fri, 16 Jun 2023 01:03:13 +0200 Subject: [PATCH 018/147] Add more InputVars schema (#1836) Co-authored-by: Christopher Ostrouchov --- .../stages/kubernetes_initialize/__init__.py | 44 ++++++------ .../kubernetes_initialize/template/main.tf | 6 +- .../modules/cluster-autoscaler/main.tf | 2 +- .../modules/cluster-autoscaler/variables.tf | 2 +- .../nvidia-installer/aws-nvidia-installer.tf | 2 +- .../nvidia-installer/gcp-nvidia-installer.tf | 2 +- .../modules/nvidia-installer/variables.tf | 4 +- .../template/variables.tf | 6 +- .../stages/kubernetes_keycloak/__init__.py | 29 +++++--- .../kubernetes_keycloak/template/main.tf | 4 +- .../modules/kubernetes/keycloak-helm/main.tf | 6 +- .../kubernetes/keycloak-helm/variables.tf | 4 +- .../kubernetes_keycloak/template/variables.tf | 5 +- .../__init__.py | 30 +++++--- .../stages/terraform_state/__init__.py | 72 +++++++++++++------ 15 files changed, 132 insertions(+), 86 deletions(-) diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py index 4821e70d3..e000b7d53 100644 --- a/src/_nebari/stages/kubernetes_initialize/__init__.py +++ b/src/_nebari/stages/kubernetes_initialize/__init__.py @@ -1,5 +1,5 @@ import sys -from typing import Any, Dict, List +from typing import Any, Dict, List, Union from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( @@ -11,6 +11,16 @@ from nebari.hookspecs import NebariStage, hookimpl +class InputVars(schema.BaseModel): + name: str + environment: str + cloud_provider: str + aws_region: Union[str, None] = None + external_container_reg: Union[schema.ExtContainerReg, None] = None + gpu_enabled: bool = False + gpu_node_group_names: List[str] = [] + + class KubernetesInitializeStage(NebariTerraformStage): name = "03-kubernetes-initialize" priority = 30 @@ -23,36 +33,30 @@ def tf_objects(self) -> List[Dict]: ] def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): + input_vars = InputVars( + name=self.config.project_name, + environment=self.config.namespace, + cloud_provider=self.config.provider.value, + external_container_reg=self.config.external_container_reg.dict(), + ) + if self.config.provider == schema.ProviderEnum.gcp: - gpu_enabled = any( + input_vars.gpu_enabled = any( node_group.guest_accelerators for node_group in self.config.google_cloud_platform.node_groups.values() ) - gpu_node_group_names = [] elif self.config.provider == schema.ProviderEnum.aws: - gpu_enabled = any( + input_vars.gpu_enabled = any( node_group.gpu for node_group in self.config.amazon_web_services.node_groups.values() ) - gpu_node_group_names = [ + input_vars.gpu_node_group_names = [ group for group in self.config.amazon_web_services.node_groups.keys() ] - else: - gpu_enabled = False - gpu_node_group_names = [] - - return { - "name": self.config.project_name, - "environment": self.config.namespace, - "cloud-provider": self.config.provider.value, - "aws-region": self.config.amazon_web_services.region - if self.config.provider == schema.ProviderEnum.aws - else None, - "external_container_reg": self.config.external_container_reg.dict(), - "gpu_enabled": gpu_enabled, - "gpu_node_group_names": gpu_node_group_names, - } + input_vars.aws_region = self.config.amazon_web_services.region + + return input_vars.dict() def check(self, stage_outputs: Dict[str, Dict[str, Any]]): from kubernetes import client, config diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf index 1a0fae924..402c68fb3 100644 --- a/src/_nebari/stages/kubernetes_initialize/template/main.tf +++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf @@ -6,13 +6,13 @@ module "kubernetes-initialization" { } module "kubernetes-autoscaling" { - count = var.cloud-provider == "aws" ? 1 : 0 + count = var.cloud_provider == "aws" ? 1 : 0 source = "./modules/cluster-autoscaler" namespace = var.environment - aws-region = var.aws-region + aws_region = var.aws_region cluster-name = local.cluster_name } @@ -25,7 +25,7 @@ module "nvidia-driver-installer" { source = "./modules/nvidia-installer" - cloud-provider = var.cloud-provider + cloud_provider = var.cloud_provider gpu_enabled = var.gpu_enabled gpu_node_group_names = var.gpu_node_group_names } diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf index 377998163..29f982c86 100644 --- a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf +++ b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf @@ -13,7 +13,7 @@ resource "helm_release" "autoscaler" { } cloudProvider = "aws" - awsRegion = var.aws-region + awsRegion = var.aws_region autoDiscovery = { clusterName = var.cluster-name diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/variables.tf index 312383f9a..a7169abee 100644 --- a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/variables.tf +++ b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/variables.tf @@ -8,7 +8,7 @@ variable "cluster-name" { type = string } -variable "aws-region" { +variable "aws_region" { description = "AWS Region that cluster autoscaler is running" type = string } diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/aws-nvidia-installer.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/aws-nvidia-installer.tf index f8706f7b6..4b1500ec9 100644 --- a/src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/aws-nvidia-installer.tf +++ b/src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/aws-nvidia-installer.tf @@ -1,5 +1,5 @@ resource "kubernetes_daemonset" "aws_nvidia_installer" { - count = var.gpu_enabled && (var.cloud-provider == "aws") ? 1 : 0 + count = var.gpu_enabled && (var.cloud_provider == "aws") ? 1 : 0 metadata { name = "nvidia-device-plugin-daemonset-1.12" namespace = "kube-system" diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/gcp-nvidia-installer.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/gcp-nvidia-installer.tf index 544aed0b8..bb73ac38d 100644 --- a/src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/gcp-nvidia-installer.tf +++ b/src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/gcp-nvidia-installer.tf @@ -1,6 +1,6 @@ # source https://cloud.google.com/kubernetes-engine/docs/how-to/gpus#installing_drivers resource "kubernetes_daemonset" "gcp_nvidia_installer" { - count = var.gpu_enabled && (var.cloud-provider == "gcp") ? 1 : 0 + count = var.gpu_enabled && (var.cloud_provider == "gcp") ? 1 : 0 metadata { name = "nvidia-driver-installer" diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/variables.tf index 1c3e60f3d..9eb9a9b2a 100644 --- a/src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/variables.tf +++ b/src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/variables.tf @@ -8,7 +8,7 @@ variable "gpu_enabled" { default = false } -variable "cloud-provider" { - description = "Name of cloud-provider" +variable "cloud_provider" { + description = "Name of cloud_provider" type = string } diff --git a/src/_nebari/stages/kubernetes_initialize/template/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/variables.tf index 87eaa9ed0..f169f5bcf 100644 --- a/src/_nebari/stages/kubernetes_initialize/template/variables.tf +++ b/src/_nebari/stages/kubernetes_initialize/template/variables.tf @@ -8,12 +8,12 @@ variable "environment" { type = string } -variable "cloud-provider" { +variable "cloud_provider" { description = "Cloud provider being used in deployment" type = string } -variable "aws-region" { +variable "aws_region" { description = "AWS region is cloud provider is AWS" type = string } @@ -25,10 +25,8 @@ variable "external_container_reg" { variable "gpu_enabled" { description = "Enable GPU support" type = bool - default = false } variable "gpu_node_group_names" { description = "Names of node groups with GPU" - default = [] } diff --git a/src/_nebari/stages/kubernetes_keycloak/__init__.py b/src/_nebari/stages/kubernetes_keycloak/__init__.py index acad8517e..ae7061cb0 100644 --- a/src/_nebari/stages/kubernetes_keycloak/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak/__init__.py @@ -1,6 +1,7 @@ import contextlib import json import sys +import time from typing import Any, Dict, List from _nebari.stages.base import NebariTerraformStage @@ -13,9 +14,17 @@ from nebari import schema from nebari.hookspecs import NebariStage, hookimpl -# check and retry settings NUM_ATTEMPTS = 10 -TIMEOUT = 10 # seconds +TIMEOUT = 10 + + +class InputVars(schema.Base): + name: str + environment: str + endpoint: str + initial_root_password: str + overrides: List[str] + node_group: Dict[str, str] @contextlib.contextmanager @@ -72,14 +81,14 @@ def tf_objects(self) -> List[Dict]: ] def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): - return { - "name": self.config.project_name, - "environment": self.config.namespace, - "endpoint": self.config.domain, - "initial-root-password": self.config.security.keycloak.initial_root_password, - "overrides": [json.dumps(self.config.security.keycloak.overrides)], - "node-group": _calculate_node_groups(self.config)["general"], - } + return InputVars( + name=self.config.project_name, + environment=self.config.namespace, + endpoint=self.config.domain, + initial_root_password=self.config.security.keycloak.initial_root_password, + overrides=[json.dumps(self.config.security.keycloak.overrides)], + node_group=_calculate_node_groups(self.config)["general"], + ).dict() def check(self, stage_outputs: Dict[str, Dict[str, Any]]): from keycloak import KeycloakAdmin diff --git a/src/_nebari/stages/kubernetes_keycloak/template/main.tf b/src/_nebari/stages/kubernetes_keycloak/template/main.tf index 51a49ae2f..d22b4ec84 100644 --- a/src/_nebari/stages/kubernetes_keycloak/template/main.tf +++ b/src/_nebari/stages/kubernetes_keycloak/template/main.tf @@ -12,9 +12,9 @@ module "kubernetes-keycloak-helm" { nebari-bot-password = random_password.keycloak-nebari-bot-password.result - initial-root-password = var.initial-root-password + initial_root_password = var.initial_root_password overrides = var.overrides - node-group = var.node-group + node_group = var.node_group } diff --git a/src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/main.tf b/src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/main.tf index d9e804ee9..7e02ea102 100644 --- a/src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/main.tf +++ b/src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/main.tf @@ -11,12 +11,12 @@ resource "helm_release" "keycloak" { file("${path.module}/values.yaml"), jsonencode({ nodeSelector = { - "${var.node-group.key}" = var.node-group.value + "${var.node_group.key}" = var.node_group.value } postgresql = { primary = { nodeSelector = { - "${var.node-group.key}" = var.node-group.value + "${var.node_group.key}" = var.node_group.value } } } @@ -30,7 +30,7 @@ resource "helm_release" "keycloak" { set { name = "initial_root_password" - value = var.initial-root-password + value = var.initial_root_password } } diff --git a/src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/variables.tf b/src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/variables.tf index 1c3648553..90392b1e9 100644 --- a/src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/variables.tf +++ b/src/_nebari/stages/kubernetes_keycloak/template/modules/kubernetes/keycloak-helm/variables.tf @@ -19,12 +19,12 @@ variable "nebari-bot-password" { type = string } -variable "initial-root-password" { +variable "initial_root_password" { description = "initial root password for keycloak" type = string } -variable "node-group" { +variable "node_group" { description = "Node key value pair for bound general resources" type = object({ key = string diff --git a/src/_nebari/stages/kubernetes_keycloak/template/variables.tf b/src/_nebari/stages/kubernetes_keycloak/template/variables.tf index 6dd2241b8..589b0cca0 100644 --- a/src/_nebari/stages/kubernetes_keycloak/template/variables.tf +++ b/src/_nebari/stages/kubernetes_keycloak/template/variables.tf @@ -13,7 +13,7 @@ variable "endpoint" { type = string } -variable "initial-root-password" { +variable "initial_root_password" { description = "Keycloak root user password" type = string } @@ -22,10 +22,9 @@ variable "overrides" { # https://github.com/codecentric/helm-charts/blob/master/charts/keycloak/values.yaml description = "Keycloak helm chart overrides" type = list(string) - default = [] } -variable "node-group" { +variable "node_group" { description = "Node key value pair for bound general resources" type = object({ key = string diff --git a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py index ac131e983..39ca07a59 100644 --- a/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py @@ -1,13 +1,22 @@ import sys +import time from typing import Any, Dict, List from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import NebariTerraformState +from nebari import schema from nebari.hookspecs import NebariStage, hookimpl -# check and retry settings NUM_ATTEMPTS = 10 -TIMEOUT = 10 # seconds +TIMEOUT = 10 + + +class InputVars(schema.Base): + realm: str = "nebari" + realm_display_name: str + authentication: Dict[str, Any] + keycloak_groups: List[str] = ["superadmin", "admin", "developer", "analyst"] + default_groups: List[str] = ["analyst"] class KubernetesKeycloakConfigurationStage(NebariTerraformStage): @@ -20,18 +29,17 @@ def tf_objects(self) -> List[Dict]: ] def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): - realm_id = "nebari" + input_vars = InputVars( + realm_display_name=self.config.security.keycloak.realm_display_name, + authentication=self.config.security.authentication, + ) users_group = ["users"] if self.config.security.shared_users_group else [] - return { - "realm": realm_id, - "realm_display_name": self.config.security.keycloak.realm_display_name, - "authentication": self.config.security.authentication.dict(), - "keycloak_groups": ["superadmin", "admin", "developer", "analyst"] - + users_group, - "default_groups": ["analyst"] + users_group, - } + input_vars.keycloak_groups += users_group + input_vars.default_groups += users_group + + return input_vars.dict() def check(self, stage_outputs: Dict[str, Dict[str, Any]]): directory = "stages/05-kubernetes-keycloak" diff --git a/src/_nebari/stages/terraform_state/__init__.py b/src/_nebari/stages/terraform_state/__init__.py index 10a8adae0..fdb6dcd08 100644 --- a/src/_nebari/stages/terraform_state/__init__.py +++ b/src/_nebari/stages/terraform_state/__init__.py @@ -8,6 +8,29 @@ from nebari.hookspecs import NebariStage, hookimpl +class BaseCloudProviderInputVars(schema.Base): + name: str + namespace: str + + +class DigitalOceanInputVars(BaseCloudProviderInputVars): + region: str + + +class GCPInputVars(BaseCloudProviderInputVars): + region: str + + +class AzureInputVars(BaseCloudProviderInputVars): + region: str + storage_account_postfix: str + state_resource_group_name: str + + +class AWSInputVars(BaseCloudProviderInputVars): + pass + + class TerraformStateStage(NebariTerraformStage): name = "01-terraform-state" priority = 10 @@ -79,32 +102,37 @@ def tf_objects(self) -> List[Dict]: def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): if self.config.provider == schema.ProviderEnum.do: - return { - "name": self.config.project_name, - "namespace": self.config.namespace, - "region": self.config.digital_ocean.region, - } + return DigitalOceanInputVars( + name=self.config.project_name, + namespace=self.config.namespace, + region=self.config.digital_ocean.region, + ).dict() elif self.config.provider == schema.ProviderEnum.gcp: - return { - "name": self.config.project_name, - "namespace": self.config.namespace, - "region": self.config.google_cloud_platform.region, - } + return GCPInputVars( + name=self.config.project_name, + namespace=self.config.namespace, + region=self.config.google_cloud_platform.region, + ).dict() elif self.config.provider == schema.ProviderEnum.aws: - return { - "name": self.config.project_name, - "namespace": self.config.namespace, - } + return AWSInputVars( + name=self.config.project_name, + namespace=self.config.namespace, + ).dict() elif self.config.provider == schema.ProviderEnum.azure: - return { - "name": self.config.project_name, - "namespace": self.config.namespace, - "region": self.config.azure.region, - "storage_account_postfix": self.config.azure.storage_account_postfix, - "state_resource_group_name": f"{self.config.project_name}-{self.config.namespace}-state", - } - else: + return AzureInputVars( + name=self.config.project_name, + namespace=self.config.namespace, + region=self.config.azure.region, + storage_account_postfix=self.config.azure.storage_account_postfix, + state_resource_group_name=f"{self.config.project_name}-{self.config.namespace}-state", + ).dict() + elif ( + self.config.provider == schema.ProviderEnum.local + or self.config.provider == schema.ProviderEnum.existing + ): return {} + else: + ValueError(f"Unknown provider: {self.config.provider}") @hookimpl From 1f86a7ba341f660797935eee313a6a86d8e5d1fd Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 15 Jun 2023 19:04:39 -0400 Subject: [PATCH 019/147] Adding more schema information about providers --- src/_nebari/cli/keycloak.py | 76 ------------------------------------- src/nebari/schema.py | 63 +++++++++++++++++++++++++----- 2 files changed, 53 insertions(+), 86 deletions(-) delete mode 100644 src/_nebari/cli/keycloak.py diff --git a/src/_nebari/cli/keycloak.py b/src/_nebari/cli/keycloak.py deleted file mode 100644 index 3b3511b3e..000000000 --- a/src/_nebari/cli/keycloak.py +++ /dev/null @@ -1,76 +0,0 @@ -import json -from pathlib import Path -from typing import Tuple - -import typer - -from _nebari.keycloak import do_keycloak, export_keycloak_users - -app_keycloak = typer.Typer( - add_completion=False, - no_args_is_help=True, - rich_markup_mode="rich", - context_settings={"help_option_names": ["-h", "--help"]}, -) - - -@app_keycloak.command(name="adduser") -def add_user( - add_users: Tuple[str, str] = typer.Option( - ..., "--user", help="Provide both: " - ), - config_filename: str = typer.Option( - ..., - "-c", - "--config", - help="nebari configuration file path", - ), -): - """Add a user to Keycloak. User will be automatically added to the [italic]analyst[/italic] group.""" - if isinstance(config_filename, str): - config_filename = Path(config_filename) - - args = ["adduser", add_users[0], add_users[1]] - - do_keycloak(config_filename, *args) - - -@app_keycloak.command(name="listusers") -def list_users( - config_filename: str = typer.Option( - ..., - "-c", - "--config", - help="nebari configuration file path", - ) -): - """List the users in Keycloak.""" - if isinstance(config_filename, str): - config_filename = Path(config_filename) - - args = ["listusers"] - - do_keycloak(config_filename, *args) - - -@app_keycloak.command(name="export-users") -def export_users( - config_filename: str = typer.Option( - ..., - "-c", - "--config", - help="nebari configuration file path", - ), - realm: str = typer.Option( - "nebari", - "--realm", - help="realm from which users are to be exported", - ), -): - """Export the users in Keycloak.""" - if isinstance(config_filename, str): - config_filename = Path(config_filename) - - r = export_keycloak_users(config_filename, realm=realm) - - print(json.dumps(r, indent=4)) diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 807feff9f..64082859d 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -347,19 +347,27 @@ class GCPPrivateClusterConfig(Base): class DigitalOceanProvider(Base): - region: str + region: str = "nyc3" kubernetes_version: str - node_groups: typing.Dict[str, NodeGroup] + node_groups: typing.Dict[str, NodeGroup] = { + "general": NodeGroup(instance="g-8vcpu-32gb", min_nodes=1, max_nodes=1), + "user": NodeGroup(instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5), + "worker": NodeGroup(instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5), + } tags: typing.Optional[typing.List[str]] = [] class GoogleCloudPlatformProvider(Base): project: str - region: str + region: str = "us-central1" availability_zones: typing.Optional[typing.List[str]] = [] kubernetes_version: str release_channel: typing.Optional[str] - node_groups: typing.Dict[str, NodeGroup] + node_groups: typing.Dict[str, NodeGroup] = { + "general": NodeGroup(instance="n1-standard-8", min_nodes=1, max_nodes=1), + "user": NodeGroup(instance="n1-standard-4", min_nodes=0, max_nodes=5), + "worker": NodeGroup(instance="n1-standard-4", min_nodes=0, max_nodes=5), + } tags: typing.Optional[typing.List[str]] = [] networking_mode: str = "ROUTE" network: str = "default" @@ -376,19 +384,27 @@ class GoogleCloudPlatformProvider(Base): class AzureProvider(Base): - region: str + region: str = "Central US" kubernetes_version: str - node_groups: typing.Dict[str, NodeGroup] + node_groups: typing.Dict[str, NodeGroup] = { + "general": NodeGroup(instance="Standard_D8_v3", min_nodes=1, max_nodes=1), + "user": NodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), + "worker": NodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), + } storage_account_postfix: str vnet_subnet_id: typing.Optional[typing.Union[str, None]] = None private_cluster_enabled: bool = False class AmazonWebServicesProvider(Base): - region: str + region: str = "us-west-2" availability_zones: typing.Optional[typing.List[str]] kubernetes_version: str - node_groups: typing.Dict[str, AWSNodeGroup] + node_groups: typing.Dict[str, AWSNodeGroup] = { + "general": NodeGroup(instance="m5.2xlarge", min_nodes=1, max_nodes=1), + "user": NodeGroup(instance="m5.xlarge", min_nodes=0, max_nodes=5, single_subnet=False), + "worker": NodeGroup(instance="m5.xlarge", min_nodes=0, max_nodes=5, single_subnet=False), + } existing_subnet_ids: typing.Optional[typing.List[str]] existing_security_group_ids: typing.Optional[str] vpc_cidr_block: str = "10.10.0.0/16" @@ -396,12 +412,20 @@ class AmazonWebServicesProvider(Base): class LocalProvider(Base): kube_context: typing.Optional[str] - node_selectors: typing.Dict[str, KeyValueDict] + node_selectors: typing.Dict[str, KeyValueDict] = { + "general": KeyValueDict(key="kubernetes.io/os", value="linux"), + "user": KeyValueDict(key="kubernetes.io/os", value="linux"), + "worker": KeyValueDict(key="kubernetes.io/os", value="linux"), + } class ExistingProvider(Base): kube_context: typing.Optional[str] - node_selectors: typing.Dict[str, KeyValueDict] + node_selectors: typing.Dict[str, KeyValueDict] = { + "general": KeyValueDict(key="kubernetes.io/os", value="linux"), + "user": KeyValueDict(key="kubernetes.io/os", value="linux"), + "worker": KeyValueDict(key="kubernetes.io/os", value="linux"), + } # ================= Theme ================== @@ -812,6 +836,25 @@ def check_default(cls, v): ) return v + @root_validator(pre=True) + def check_provider(cls, values): + if values['provider'] == ProviderEnum.local and values.get('local') is None: + values['local'] = LocalProvider() + elif values['provider'] == ProviderEnum.existing and values.get('existing') is None: + values['existing'] = ExistingProvider() + elif values['provider'] == ProviderEnum.gcp and values.get('google_cloud_platform') is None: + values['google_cloud_platform'] = GoogleCloudPlatformProvider() + elif values['provider'] == ProviderEnum.aws and values.get('amazon_web_services') is None: + values['amazon_web_services'] = AmazonWebServicesProvider() + elif values['provider'] == ProviderEnum.azure and values.get('azure') is None: + values['azure'] = AzureProvider() + elif values['provider'] == ProviderEnum.do and values.get('digital_ocean') is None: + values['digital_ocean'] = DigitalOceanProvider() + + if sum((_ in values and values[_] is not None) for _ in {'local', 'existing', 'google_cloud_platform', 'amazon_web_services', 'azure', 'digital_ocean'}) != 1: + raise ValueError('multiple providers set or wrong provider fields set') + return values + @classmethod def is_version_accepted(cls, v): return v != "" and rounded_ver_parse(v) == rounded_ver_parse(__version__) From 56b4d38934898fc42fe48879f1cfd40abec4a7b4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Jun 2023 23:04:59 +0000 Subject: [PATCH 020/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/nebari/schema.py | 63 ++++++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 64082859d..8b48b6dee 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -402,8 +402,12 @@ class AmazonWebServicesProvider(Base): kubernetes_version: str node_groups: typing.Dict[str, AWSNodeGroup] = { "general": NodeGroup(instance="m5.2xlarge", min_nodes=1, max_nodes=1), - "user": NodeGroup(instance="m5.xlarge", min_nodes=0, max_nodes=5, single_subnet=False), - "worker": NodeGroup(instance="m5.xlarge", min_nodes=0, max_nodes=5, single_subnet=False), + "user": NodeGroup( + instance="m5.xlarge", min_nodes=0, max_nodes=5, single_subnet=False + ), + "worker": NodeGroup( + instance="m5.xlarge", min_nodes=0, max_nodes=5, single_subnet=False + ), } existing_subnet_ids: typing.Optional[typing.List[str]] existing_security_group_ids: typing.Optional[str] @@ -838,21 +842,46 @@ def check_default(cls, v): @root_validator(pre=True) def check_provider(cls, values): - if values['provider'] == ProviderEnum.local and values.get('local') is None: - values['local'] = LocalProvider() - elif values['provider'] == ProviderEnum.existing and values.get('existing') is None: - values['existing'] = ExistingProvider() - elif values['provider'] == ProviderEnum.gcp and values.get('google_cloud_platform') is None: - values['google_cloud_platform'] = GoogleCloudPlatformProvider() - elif values['provider'] == ProviderEnum.aws and values.get('amazon_web_services') is None: - values['amazon_web_services'] = AmazonWebServicesProvider() - elif values['provider'] == ProviderEnum.azure and values.get('azure') is None: - values['azure'] = AzureProvider() - elif values['provider'] == ProviderEnum.do and values.get('digital_ocean') is None: - values['digital_ocean'] = DigitalOceanProvider() - - if sum((_ in values and values[_] is not None) for _ in {'local', 'existing', 'google_cloud_platform', 'amazon_web_services', 'azure', 'digital_ocean'}) != 1: - raise ValueError('multiple providers set or wrong provider fields set') + if values["provider"] == ProviderEnum.local and values.get("local") is None: + values["local"] = LocalProvider() + elif ( + values["provider"] == ProviderEnum.existing + and values.get("existing") is None + ): + values["existing"] = ExistingProvider() + elif ( + values["provider"] == ProviderEnum.gcp + and values.get("google_cloud_platform") is None + ): + values["google_cloud_platform"] = GoogleCloudPlatformProvider() + elif ( + values["provider"] == ProviderEnum.aws + and values.get("amazon_web_services") is None + ): + values["amazon_web_services"] = AmazonWebServicesProvider() + elif values["provider"] == ProviderEnum.azure and values.get("azure") is None: + values["azure"] = AzureProvider() + elif ( + values["provider"] == ProviderEnum.do + and values.get("digital_ocean") is None + ): + values["digital_ocean"] = DigitalOceanProvider() + + if ( + sum( + (_ in values and values[_] is not None) + for _ in { + "local", + "existing", + "google_cloud_platform", + "amazon_web_services", + "azure", + "digital_ocean", + } + ) + != 1 + ): + raise ValueError("multiple providers set or wrong provider fields set") return values @classmethod From 31a6390cc730eb6a547e2973b4ff6d2122816f96 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 16 Jun 2023 09:52:55 -0400 Subject: [PATCH 021/147] Fixes to schema to allow for local deployment --- src/_nebari/stages/kubernetes_initialize/__init__.py | 2 +- src/nebari/schema.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py index e000b7d53..202d8b875 100644 --- a/src/_nebari/stages/kubernetes_initialize/__init__.py +++ b/src/_nebari/stages/kubernetes_initialize/__init__.py @@ -11,7 +11,7 @@ from nebari.hookspecs import NebariStage, hookimpl -class InputVars(schema.BaseModel): +class InputVars(schema.Base): name: str environment: str cloud_provider: str diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 8b48b6dee..9ed449d57 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -436,13 +436,13 @@ class ExistingProvider(Base): class JupyterHubTheme(Base): - hub_title: str = ("Nebari",) - hub_subtitle: str = ("Your open source data science platform",) + hub_title: str = "Nebari" + hub_subtitle: str = "Your open source data science platform" welcome: str = ( - """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""", + """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""" ) logo: str = ( - "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg", + "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg" ) primary_color: str = "#4f4173" secondary_color: str = "#957da6" @@ -451,7 +451,7 @@ class JupyterHubTheme(Base): h1_color: str = "#652e8e" h2_color: str = "#652e8e" version: str = f"v{__version__}" - display_version: bool = True + display_version: str = "True" # limitation of theme everything is a str class Theme(Base): @@ -840,7 +840,7 @@ def check_default(cls, v): ) return v - @root_validator(pre=True) + @root_validator def check_provider(cls, values): if values["provider"] == ProviderEnum.local and values.get("local") is None: values["local"] = LocalProvider() From 76f8d6d2dde89933c180466a7063cdd3f22e3129 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Jun 2023 13:53:24 +0000 Subject: [PATCH 022/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/nebari/schema.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 9ed449d57..2cf5f2dc4 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -438,12 +438,8 @@ class ExistingProvider(Base): class JupyterHubTheme(Base): hub_title: str = "Nebari" hub_subtitle: str = "Your open source data science platform" - welcome: str = ( - """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""" - ) - logo: str = ( - "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg" - ) + welcome: str = """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""" + logo: str = "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg" primary_color: str = "#4f4173" secondary_color: str = "#957da6" accent_color: str = "#32C574" From 9896875484f48b00211672aa4ffb1de52e75a524 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 16 Jun 2023 10:16:20 -0400 Subject: [PATCH 023/147] Apply validation on the kubernetes version --- src/nebari/schema.py | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 2cf5f2dc4..a985cbbf6 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -15,6 +15,7 @@ from _nebari import constants from _nebari.utils import namestr_regex, set_docker_image_tag, set_nebari_dask_version from _nebari.version import __version__, rounded_ver_parse +from _nebari.provider.cloud import azure_cloud, amazon_web_services, digital_ocean, google_cloud class CertificateEnum(str, enum.Enum): @@ -348,7 +349,7 @@ class GCPPrivateClusterConfig(Base): class DigitalOceanProvider(Base): region: str = "nyc3" - kubernetes_version: str + kubernetes_version: str = Field(default_factory=lambda: digital_ocean.kubernetes_versions()[-1]) node_groups: typing.Dict[str, NodeGroup] = { "general": NodeGroup(instance="g-8vcpu-32gb", min_nodes=1, max_nodes=1), "user": NodeGroup(instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5), @@ -356,12 +357,20 @@ class DigitalOceanProvider(Base): } tags: typing.Optional[typing.List[str]] = [] + @validator('kubernetes_version') + def _validate_kubernetes_version(cls, value): + available_kubernetes_versions = digital_ocean.kubernetes_versions() + if value not in available_kubernetes_versions: + raise ValueError(f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available.") + return value + class GoogleCloudPlatformProvider(Base): project: str region: str = "us-central1" availability_zones: typing.Optional[typing.List[str]] = [] - kubernetes_version: str + kubernetes_version: str = Field(default_factory=lambda: google_cloud.kubernetes_versions()[-1]) + release_channel: typing.Optional[str] node_groups: typing.Dict[str, NodeGroup] = { "general": NodeGroup(instance="n1-standard-8", min_nodes=1, max_nodes=1), @@ -382,10 +391,18 @@ class GoogleCloudPlatformProvider(Base): typing.Union[GCPPrivateClusterConfig, None] ] = None + @validator('kubernetes_version') + def _validate_kubernetes_version(cls, value): + available_kubernetes_versions = google_cloud.kubernetes_versions() + if value not in available_kubernetes_versions: + raise ValueError(f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available.") + return value + class AzureProvider(Base): region: str = "Central US" - kubernetes_version: str + kubernetes_version: str = Field(default_factory=lambda: azure_cloud.kubernetes_versions()[-1]) + node_groups: typing.Dict[str, NodeGroup] = { "general": NodeGroup(instance="Standard_D8_v3", min_nodes=1, max_nodes=1), "user": NodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), @@ -395,11 +412,20 @@ class AzureProvider(Base): vnet_subnet_id: typing.Optional[typing.Union[str, None]] = None private_cluster_enabled: bool = False + @validator('kubernetes_version') + def _validate_kubernetes_version(cls, value): + available_kubernetes_versions = azure_cloud.kubernetes_versions() + if value not in available_kubernetes_versions: + raise ValueError(f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available.") + return value + + class AmazonWebServicesProvider(Base): region: str = "us-west-2" availability_zones: typing.Optional[typing.List[str]] - kubernetes_version: str + kubernetes_version: str = Field(default_factory=lambda: amazon_web_services.kubernetes_versions()[-1]) + node_groups: typing.Dict[str, AWSNodeGroup] = { "general": NodeGroup(instance="m5.2xlarge", min_nodes=1, max_nodes=1), "user": NodeGroup( @@ -413,6 +439,14 @@ class AmazonWebServicesProvider(Base): existing_security_group_ids: typing.Optional[str] vpc_cidr_block: str = "10.10.0.0/16" + @validator('kubernetes_version') + def _validate_kubernetes_version(cls, value): + available_kubernetes_versions = amazon_web_services.kubernetes_versions() + if value not in available_kubernetes_versions: + raise ValueError(f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available.") + return value + + class LocalProvider(Base): kube_context: typing.Optional[str] From e70dd2bb4022841b5179c211e29441c6ad090575 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Jun 2023 14:16:40 +0000 Subject: [PATCH 024/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/nebari/schema.py | 49 ++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/src/nebari/schema.py b/src/nebari/schema.py index a985cbbf6..dfe29e242 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -13,9 +13,14 @@ from ruamel.yaml import YAML from _nebari import constants +from _nebari.provider.cloud import ( + amazon_web_services, + azure_cloud, + digital_ocean, + google_cloud, +) from _nebari.utils import namestr_regex, set_docker_image_tag, set_nebari_dask_version from _nebari.version import __version__, rounded_ver_parse -from _nebari.provider.cloud import azure_cloud, amazon_web_services, digital_ocean, google_cloud class CertificateEnum(str, enum.Enum): @@ -349,7 +354,9 @@ class GCPPrivateClusterConfig(Base): class DigitalOceanProvider(Base): region: str = "nyc3" - kubernetes_version: str = Field(default_factory=lambda: digital_ocean.kubernetes_versions()[-1]) + kubernetes_version: str = Field( + default_factory=lambda: digital_ocean.kubernetes_versions()[-1] + ) node_groups: typing.Dict[str, NodeGroup] = { "general": NodeGroup(instance="g-8vcpu-32gb", min_nodes=1, max_nodes=1), "user": NodeGroup(instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5), @@ -357,11 +364,13 @@ class DigitalOceanProvider(Base): } tags: typing.Optional[typing.List[str]] = [] - @validator('kubernetes_version') + @validator("kubernetes_version") def _validate_kubernetes_version(cls, value): available_kubernetes_versions = digital_ocean.kubernetes_versions() if value not in available_kubernetes_versions: - raise ValueError(f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available.") + raise ValueError( + f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." + ) return value @@ -369,7 +378,9 @@ class GoogleCloudPlatformProvider(Base): project: str region: str = "us-central1" availability_zones: typing.Optional[typing.List[str]] = [] - kubernetes_version: str = Field(default_factory=lambda: google_cloud.kubernetes_versions()[-1]) + kubernetes_version: str = Field( + default_factory=lambda: google_cloud.kubernetes_versions()[-1] + ) release_channel: typing.Optional[str] node_groups: typing.Dict[str, NodeGroup] = { @@ -391,17 +402,21 @@ class GoogleCloudPlatformProvider(Base): typing.Union[GCPPrivateClusterConfig, None] ] = None - @validator('kubernetes_version') + @validator("kubernetes_version") def _validate_kubernetes_version(cls, value): available_kubernetes_versions = google_cloud.kubernetes_versions() if value not in available_kubernetes_versions: - raise ValueError(f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available.") + raise ValueError( + f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." + ) return value class AzureProvider(Base): region: str = "Central US" - kubernetes_version: str = Field(default_factory=lambda: azure_cloud.kubernetes_versions()[-1]) + kubernetes_version: str = Field( + default_factory=lambda: azure_cloud.kubernetes_versions()[-1] + ) node_groups: typing.Dict[str, NodeGroup] = { "general": NodeGroup(instance="Standard_D8_v3", min_nodes=1, max_nodes=1), @@ -412,19 +427,22 @@ class AzureProvider(Base): vnet_subnet_id: typing.Optional[typing.Union[str, None]] = None private_cluster_enabled: bool = False - @validator('kubernetes_version') + @validator("kubernetes_version") def _validate_kubernetes_version(cls, value): available_kubernetes_versions = azure_cloud.kubernetes_versions() if value not in available_kubernetes_versions: - raise ValueError(f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available.") + raise ValueError( + f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." + ) return value - class AmazonWebServicesProvider(Base): region: str = "us-west-2" availability_zones: typing.Optional[typing.List[str]] - kubernetes_version: str = Field(default_factory=lambda: amazon_web_services.kubernetes_versions()[-1]) + kubernetes_version: str = Field( + default_factory=lambda: amazon_web_services.kubernetes_versions()[-1] + ) node_groups: typing.Dict[str, AWSNodeGroup] = { "general": NodeGroup(instance="m5.2xlarge", min_nodes=1, max_nodes=1), @@ -439,15 +457,16 @@ class AmazonWebServicesProvider(Base): existing_security_group_ids: typing.Optional[str] vpc_cidr_block: str = "10.10.0.0/16" - @validator('kubernetes_version') + @validator("kubernetes_version") def _validate_kubernetes_version(cls, value): available_kubernetes_versions = amazon_web_services.kubernetes_versions() if value not in available_kubernetes_versions: - raise ValueError(f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available.") + raise ValueError( + f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." + ) return value - class LocalProvider(Base): kube_context: typing.Optional[str] node_selectors: typing.Dict[str, KeyValueDict] = { From 0b7ebc022c12bbcde160429efbae1485ee38e0a6 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 16 Jun 2023 13:23:21 -0400 Subject: [PATCH 025/147] Fixing the upgrade command --- src/_nebari/subcommands/upgrade.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/_nebari/subcommands/upgrade.py b/src/_nebari/subcommands/upgrade.py index b1c173af4..53cd0e28c 100644 --- a/src/_nebari/subcommands/upgrade.py +++ b/src/_nebari/subcommands/upgrade.py @@ -10,7 +10,7 @@ def nebari_subcommand(cli: typer.Typer): @cli.command(rich_help_panel="Additional Commands") def upgrade( - config: str = typer.Option( + config_filename: pathlib = typer.Option( ..., "-c", "--config", @@ -29,7 +29,6 @@ def upgrade( See the project [green]RELEASE.md[/green] for details. """ - config_filename = pathlib.Path(config) if not config_filename.is_file(): raise ValueError( f"passed in configuration filename={config_filename} must exist" From 04010d87f2642e8e0b5f406e7213f37bff343d29 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 16 Jun 2023 13:45:52 -0400 Subject: [PATCH 026/147] Fixing support and upgrde commands --- src/_nebari/subcommands/support.py | 33 ++++++++++++------------------ src/_nebari/subcommands/upgrade.py | 2 +- src/_nebari/upgrade.py | 6 +++--- src/nebari/plugins.py | 2 +- 4 files changed, 18 insertions(+), 25 deletions(-) diff --git a/src/_nebari/subcommands/support.py b/src/_nebari/subcommands/support.py index c85b4d56d..fce080e1e 100644 --- a/src/_nebari/subcommands/support.py +++ b/src/_nebari/subcommands/support.py @@ -1,26 +1,19 @@ +import pathlib +from zipfile import ZipFile + import typer +import kubernetes.config +import kubernetes.client +from nebari import schema from nebari.hookspecs import hookimpl -def get_config_namespace(config): - config_filename = Path(config) - if not config_filename.is_file(): - raise ValueError( - f"passed in configuration filename={config_filename} must exist" - ) - - with config_filename.open() as f: - config = yaml.safe_load(f.read()) - - return config["namespace"] - - @hookimpl def nebari_subcommand(cli: typer.Typer): @cli.command(rich_help_panel="Additional Commands") def support( - config_filename: str = typer.Option( + config_filename: pathlib.Path = typer.Option( ..., "-c", "--config", @@ -39,17 +32,17 @@ def support( The Nebari team recommends k9s to manage and inspect the state of the cluster. However, this command occasionally helpful for debugging purposes should the logs need to be shared. """ - kube_config.load_kube_config() + kubernetes.config.kube_config.load_kube_config() - v1 = client.CoreV1Api() + v1 = kubernetes.client.CoreV1Api() - namespace = get_config_namespace(config=config_filename) + namespace = schema.read_configuration(config_filename).namespace pods = v1.list_namespaced_pod(namespace=namespace) for pod in pods.items: - Path(f"./log/{namespace}").mkdir(parents=True, exist_ok=True) - path = Path(f"./log/{namespace}/{pod.metadata.name}.txt") + pathlib.Path(f"./log/{namespace}").mkdir(parents=True, exist_ok=True) + path = pathlib.Path(f"./log/{namespace}/{pod.metadata.name}.txt") with path.open(mode="wt") as file: try: file.write( @@ -83,6 +76,6 @@ def support( raise e with ZipFile(output, "w") as zip: - for file in list(Path(f"./log/{namespace}").glob("*.txt")): + for file in list(pathlib.Path(f"./log/{namespace}").glob("*.txt")): print(file) zip.write(file) diff --git a/src/_nebari/subcommands/upgrade.py b/src/_nebari/subcommands/upgrade.py index 53cd0e28c..53d6cfabf 100644 --- a/src/_nebari/subcommands/upgrade.py +++ b/src/_nebari/subcommands/upgrade.py @@ -10,7 +10,7 @@ def nebari_subcommand(cli: typer.Typer): @cli.command(rich_help_panel="Additional Commands") def upgrade( - config_filename: pathlib = typer.Option( + config_filename: pathlib.Path = typer.Option( ..., "-c", "--config", diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 2f53492ae..1d05d5637 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -10,7 +10,7 @@ from pydantic.error_wrappers import ValidationError from rich.prompt import Prompt -from nebari.schema import is_version_accepted, verify +from nebari import schema from .utils import backup_config_file, load_yaml, yaml from .version import __version__, rounded_ver_parse @@ -32,13 +32,13 @@ def do_upgrade(config_filename, attempt_fixes=False): return try: - verify(config) + schema.read_configuration(config_filename) rich.print( f"Your config file [purple]{config_filename}[/purple] appears to be already up-to-date for Nebari version [green]{__version__}[/green]" ) return except (ValidationError, ValueError) as e: - if is_version_accepted(config.get("nebari_version", "")): + if schema.is_version_accepted(config.get("nebari_version", "")): # There is an unrelated validation problem rich.print( f"Your config file [purple]{config_filename}[/purple] appears to be already up-to-date for Nebari version [green]{__version__}[/green] but there is another validation error.\n" diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index 16ae7a8c1..69a66c494 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -23,7 +23,7 @@ "_nebari.subcommands.keycloak", "_nebari.subcommands.render", "_nebari.subcommands.support", - # "_nebari.subcommands.upgrade", + "_nebari.subcommands.upgrade", "_nebari.subcommands.validate", ] From a051ec5bac193d7098972ab23e8feb1ebcc8438c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Jun 2023 17:46:10 +0000 Subject: [PATCH 027/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/subcommands/support.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_nebari/subcommands/support.py b/src/_nebari/subcommands/support.py index fce080e1e..c3a82d40e 100644 --- a/src/_nebari/subcommands/support.py +++ b/src/_nebari/subcommands/support.py @@ -1,9 +1,9 @@ import pathlib from zipfile import ZipFile -import typer -import kubernetes.config import kubernetes.client +import kubernetes.config +import typer from nebari import schema from nebari.hookspecs import hookimpl From 97d14644c9e3990b639c1adb43dc0364a5dac818 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 16 Jun 2023 16:06:01 -0400 Subject: [PATCH 028/147] More work to get init working again --- src/_nebari/cli/init.py | 52 +-- src/_nebari/initialize.py | 503 +++-------------------- src/_nebari/provider/oauth/auth0.py | 2 +- src/_nebari/stages/bootstrap/__init__.py | 61 +-- src/_nebari/subcommands/init.py | 404 +++++++++++++----- src/_nebari/utils.py | 96 +++-- src/nebari/plugins.py | 1 + src/nebari/schema.py | 97 ++++- 8 files changed, 533 insertions(+), 683 deletions(-) diff --git a/src/_nebari/cli/init.py b/src/_nebari/cli/init.py index 0e86c26b8..35eae6f8a 100644 --- a/src/_nebari/cli/init.py +++ b/src/_nebari/cli/init.py @@ -1,22 +1,13 @@ import os import re -from pathlib import Path +import pathlib import questionary import rich import typer from _nebari.initialize import render_config -from _nebari.schema import ( - AuthenticationEnum, - CiEnum, - GitRepoEnum, - InitInputs, - ProviderEnum, - TerraformStateEnum, - project_name_convention, -) -from _nebari.utils import NEBARI_DASK_VERSION, NEBARI_IMAGE_TAG, yaml +from nebari import schema MISSING_CREDS_TEMPLATE = "Unable to locate your {provider} credentials, refer to this guide on how to generate them:\n\n[green]\t{link_to_docs}[/green]\n\n" LINKS_TO_DOCS_TEMPLATE = ( @@ -46,19 +37,19 @@ def enum_to_list(enum_cls): return [e.value.lower() for e in enum_cls] -def handle_init(inputs: InitInputs): +def handle_init(inputs: schema.InitInputs): """ Take the inputs from the `nebari init` command, render the config and write it to a local yaml file. """ - if NEBARI_IMAGE_TAG: - print( - f"Modifying the image tags for the `default_images`, setting tags to: {NEBARI_IMAGE_TAG}" - ) + # if NEBARI_IMAGE_TAG: + # print( + # f"Modifying the image tags for the `default_images`, setting tags to: {NEBARI_IMAGE_TAG}" + # ) - if NEBARI_DASK_VERSION: - print( - f"Modifying the version of the `nebari_dask` package, setting version to: {NEBARI_DASK_VERSION}" - ) + # if NEBARI_DASK_VERSION: + # print( + # f"Modifying the version of the `nebari_dask` package, setting version to: {NEBARI_DASK_VERSION}" + # ) # this will force the `set_kubernetes_version` to grab the latest version if inputs.kubernetes_version == "latest": @@ -81,8 +72,7 @@ def handle_init(inputs: InitInputs): ) try: - with open("nebari-config.yaml", "x") as f: - yaml.dump(config, f) + schema.write_configuration(pathlib.Path("nebari-config.yaml"), config, mode='x') except FileExistsError: raise ValueError( "A nebari-config.yaml file already exists. Please move or delete it and try again." @@ -97,7 +87,7 @@ def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: str): cloud_provider = cloud_provider.lower() # AWS - if cloud_provider == ProviderEnum.aws.value.lower() and ( + if cloud_provider == schema.ProviderEnum.aws.value.lower() and ( not os.environ.get("AWS_ACCESS_KEY_ID") or not os.environ.get("AWS_SECRET_ACCESS_KEY") ): @@ -117,7 +107,7 @@ def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: str): ) # GCP - elif cloud_provider == ProviderEnum.gcp.value.lower() and ( + elif cloud_provider == schema.ProviderEnum.gcp.value.lower() and ( not os.environ.get("GOOGLE_CREDENTIALS") or not os.environ.get("PROJECT_ID") ): rich.print( @@ -136,7 +126,7 @@ def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: str): ) # DO - elif cloud_provider == ProviderEnum.do.value.lower() and ( + elif cloud_provider == schema.ProviderEnum.do.value.lower() and ( not os.environ.get("DIGITALOCEAN_TOKEN") or not os.environ.get("SPACES_ACCESS_KEY_ID") or not os.environ.get("SPACES_SECRET_ACCESS_KEY") @@ -163,7 +153,7 @@ def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: str): os.environ["AWS_SECRET_ACCESS_KEY"] = os.getenv("AWS_SECRET_ACCESS_KEY") # AZURE - elif cloud_provider == ProviderEnum.azure.value.lower() and ( + elif cloud_provider == schema.ProviderEnum.azure.value.lower() and ( not os.environ.get("ARM_CLIENT_ID") or not os.environ.get("ARM_CLIENT_SECRET") or not os.environ.get("ARM_SUBSCRIPTION_ID") @@ -251,7 +241,7 @@ def check_auth_provider_creds(ctx: typer.Context, auth_provider: str): def check_project_name(ctx: typer.Context, project_name: str): """Validate the project_name is acceptable. Depends on `cloud_provider`.""" - project_name_convention( + schema.project_name_convention( project_name.lower(), {"provider": ctx.params["cloud_provider"]} ) @@ -291,7 +281,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): qmark = " " disable_checks = os.environ.get("NEBARI_DISABLE_INIT_CHECKS", False) - if Path("nebari-config.yaml").exists(): + if pathlib.Path("nebari-config.yaml").exists(): raise ValueError( "A nebari-config.yaml file already exists. Please move or delete it and try again." ) @@ -314,7 +304,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): ) # pull in default values for each of the below - inputs = InitInputs() + inputs = schema.InitInputs() # CLOUD PROVIDER rich.print( @@ -345,7 +335,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): - Letters from A to Z (upper and lower case) and numbers - Maximum accepted length of the name string is 16 characters """ - if inputs.cloud_provider == ProviderEnum.aws.value.lower(): + if inputs.cloud_provider == schema.ProviderEnum.aws.value.lower(): name_guidelines += "- Should NOT start with the string `aws`\n" elif inputs.cloud_provider == ProviderEnum.azure.value.lower(): name_guidelines += "- Should NOT contain `-`\n" @@ -396,7 +386,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): if not disable_checks: check_auth_provider_creds(ctx, auth_provider=inputs.auth_provider) - if inputs.auth_provider.lower() == AuthenticationEnum.auth0.value.lower(): + if inputs.auth_provider.lower() == schemaAuthenticationEnum.auth0.value.lower(): inputs.auth_auto_provision = questionary.confirm( "Would you like us to auto provision the Auth0 Machine-to-Machine app?", default=False, diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index d176c9e20..b1aa67ea0 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -12,458 +12,96 @@ from _nebari.provider import git from _nebari.provider.cicd import github from _nebari.provider.oauth.auth0 import create_client -from _nebari.utils import ( - check_cloud_credentials, - namestr_regex, - set_docker_image_tag, - set_kubernetes_version, - set_nebari_dask_version, -) +from _nebari.utils import check_cloud_credentials -from .version import __version__ +from _nebari.version import __version__ +from nebari import schema logger = logging.getLogger(__name__) WELCOME_HEADER_TEXT = "Your open source data science platform, hosted" -def base_configuration(): - nebari_image_tag = set_docker_image_tag() - return { - "project_name": None, - "provider": None, - "domain": None, - "certificate": { - "type": "self-signed", - }, - "security": { - "authentication": None, - }, - "default_images": { - "jupyterhub": f"quay.io/nebari/nebari-jupyterhub:{nebari_image_tag}", - "jupyterlab": f"quay.io/nebari/nebari-jupyterlab:{nebari_image_tag}", - "dask_worker": f"quay.io/nebari/nebari-dask-worker:{nebari_image_tag}", - }, - "storage": {"conda_store": "200Gi", "shared_filesystem": "200Gi"}, - "theme": { - "jupyterhub": { - "hub_title": None, - "hub_subtitle": None, - "welcome": None, - "logo": "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg", - "display_version": True, - } - }, - "helm_extensions": [], - "monitoring": { - "enabled": True, - }, - "argo_workflows": { - "enabled": True, - }, - "kbatch": { - "enabled": True, - }, - "cdsdashboards": { - "enabled": True, - "cds_hide_user_named_servers": True, - "cds_hide_user_dashboard_servers": False, - }, - } - - -def default_environments(): - nebari_dask_version = set_nebari_dask_version() - return { - "environment-dask.yaml": { - "name": "dask", - "channels": ["conda-forge"], - "dependencies": [ - "python=3.10.8", - "ipykernel=6.21.0", - "ipywidgets==7.7.1", - f"nebari-dask =={nebari_dask_version}", - "python-graphviz=0.20.1", - "pyarrow=10.0.1", - "s3fs=2023.1.0", - "gcsfs=2023.1.0", - "numpy=1.23.5", - "numba=0.56.4", - "pandas=1.5.3", - { - "pip": [ - "kbatch==0.4.1", - ], - }, - ], - }, - "environment-dashboard.yaml": { - "name": "dashboard", - "channels": ["conda-forge"], - "dependencies": [ - "python=3.10", - "cdsdashboards-singleuser=0.6.3", - "cufflinks-py=0.17.3", - "dash=2.8.1", - "geopandas=0.12.2", - "geopy=2.3.0", - "geoviews=1.9.6", - "gunicorn=20.1.0", - "holoviews=1.15.4", - "ipykernel=6.21.2", - "ipywidgets=8.0.4", - "jupyter=1.0.0", - "jupyterlab=3.6.1", - "jupyter_bokeh=3.0.5", - "matplotlib=3.7.0", - f"nebari-dask=={nebari_dask_version}", - "nodejs=18.12.1", - "numpy", - "openpyxl=3.1.1", - "pandas=1.5.3", - "panel=0.14.3", - "param=1.12.3", - "plotly=5.13.0", - "python-graphviz=0.20.1", - "rich=13.3.1", - "streamlit=1.9.0", - "sympy=1.11.1", - "voila=0.4.0", - "pip=23.0", - { - "pip": [ - "streamlit-image-comparison==0.0.3", - "noaa-coops==0.2.1", - "dash_core_components==2.0.0", - "dash_html_components==2.0.0", - ], - }, - ], - }, - } - - -def __getattr__(name): - if name == "nebari_image_tag": - return set_docker_image_tag() - elif name == "nebari_dask_version": - return set_nebari_dask_version() - elif name == "BASE_CONFIGURATION": - return base_configuration() - elif name == "DEFAULT_ENVIRONMENTS": - return default_environments() - - -CICD_CONFIGURATION = { - "type": "PLACEHOLDER", - "branch": "main", - "commit_render": True, -} - -AUTH_PASSWORD = { - "type": "password", -} - -AUTH_OAUTH_GITHUB = { - "type": "GitHub", - "config": { - "client_id": "PLACEHOLDER", - "client_secret": "PLACEHOLDER", - }, -} - -AUTH_OAUTH_AUTH0 = { - "type": "Auth0", - "config": { - "client_id": "PLACEHOLDER", - "client_secret": "PLACEHOLDER", - "auth0_subdomain": "PLACEHOLDER", - }, -} - -LOCAL = { - "node_selectors": { - "general": { - "key": "kubernetes.io/os", - "value": "linux", - }, - "user": { - "key": "kubernetes.io/os", - "value": "linux", - }, - "worker": { - "key": "kubernetes.io/os", - "value": "linux", - }, - } -} - -EXISTING = { - "node_selectors": { - "general": { - "key": "kubernetes.io/os", - "value": "linux", - }, - "user": { - "key": "kubernetes.io/os", - "value": "linux", - }, - "worker": { - "key": "kubernetes.io/os", - "value": "linux", - }, - } -} - -DIGITAL_OCEAN = { - "region": "nyc3", - "kubernetes_version": "PLACEHOLDER", - "node_groups": { - "general": {"instance": "g-8vcpu-32gb", "min_nodes": 1, "max_nodes": 1}, - "user": {"instance": "g-4vcpu-16gb", "min_nodes": 1, "max_nodes": 5}, - "worker": {"instance": "g-4vcpu-16gb", "min_nodes": 1, "max_nodes": 5}, - }, -} -# Digital Ocean image slugs are listed here https://slugs.do-api.dev/ - -GOOGLE_PLATFORM = { - "project": "PLACEHOLDER", - "region": "us-central1", - "kubernetes_version": "PLACEHOLDER", - "node_groups": { - "general": {"instance": "n1-standard-8", "min_nodes": 1, "max_nodes": 1}, - "user": {"instance": "n1-standard-4", "min_nodes": 0, "max_nodes": 5}, - "worker": {"instance": "n1-standard-4", "min_nodes": 0, "max_nodes": 5}, - }, -} - -AZURE = { - "region": "Central US", - "kubernetes_version": "PLACEHOLDER", - "node_groups": { - "general": { - "instance": "Standard_D8_v3", - "min_nodes": 1, - "max_nodes": 1, - }, - "user": {"instance": "Standard_D4_v3", "min_nodes": 0, "max_nodes": 5}, - "worker": { - "instance": "Standard_D4_v3", - "min_nodes": 0, - "max_nodes": 5, - }, - }, - "storage_account_postfix": "".join( - random.choices("abcdefghijklmnopqrstuvwxyz0123456789", k=8) - ), -} - -AMAZON_WEB_SERVICES = { - "region": "us-west-2", - "kubernetes_version": "PLACEHOLDER", - "node_groups": { - "general": {"instance": "m5.2xlarge", "min_nodes": 1, "max_nodes": 1}, - "user": { - "instance": "m5.xlarge", - "min_nodes": 1, - "max_nodes": 5, - "single_subnet": False, - }, - "worker": { - "instance": "m5.xlarge", - "min_nodes": 1, - "max_nodes": 5, - "single_subnet": False, - }, - }, -} - -DEFAULT_PROFILES = { - "jupyterlab": [ - { - "display_name": "Small Instance", - "description": "Stable environment with 2 cpu / 8 GB ram", - "default": True, - "kubespawner_override": { - "cpu_limit": 2, - "cpu_guarantee": 1.5, - "mem_limit": "8G", - "mem_guarantee": "5G", - }, - }, - { - "display_name": "Medium Instance", - "description": "Stable environment with 4 cpu / 16 GB ram", - "kubespawner_override": { - "cpu_limit": 4, - "cpu_guarantee": 3, - "mem_limit": "16G", - "mem_guarantee": "10G", - }, - }, - ], - "dask_worker": { - "Small Worker": { - "worker_cores_limit": 2, - "worker_cores": 1.5, - "worker_memory_limit": "8G", - "worker_memory": "5G", - "worker_threads": 2, - }, - "Medium Worker": { - "worker_cores_limit": 4, - "worker_cores": 3, - "worker_memory_limit": "16G", - "worker_memory": "10G", - "worker_threads": 4, - }, - }, -} - - def render_config( - project_name, - nebari_domain, - cloud_provider, - ci_provider, - repository, - auth_provider, - namespace=None, - repository_auto_provision=False, - auth_auto_provision=False, - terraform_state=None, - kubernetes_version=None, - disable_prompt=False, - ssl_cert_email=None, + project_name: str, + nebari_domain: str, + cloud_provider: schema.ProviderEnum = schema.ProviderEnum.local, + ci_provider: schema.CiEnum = schema.CiEnum.none, + repository: str = None, + auth_provider: schema.AuthenticationEnum = schema.AuthenticationEnum.password, + namespace: str = "dev", + repository_auto_provision: bool = False, + auth_auto_provision: bool = False, + terraform_state: schema.TerraformStateEnum = schema.TerraformStateEnum.remote, + kubernetes_version: str = None, + disable_prompt: bool = False, + ssl_cert_email: str = None, ): - config = base_configuration().copy() - config["provider"] = cloud_provider - - if ci_provider is not None and ci_provider != "none": - config["ci_cd"] = CICD_CONFIGURATION.copy() - config["ci_cd"]["type"] = ci_provider - - if terraform_state is not None: - config["terraform_state"] = {"type": terraform_state} - if project_name is None and not disable_prompt: project_name = input("Provide project name: ") - config["project_name"] = project_name - - if not re.match(namestr_regex, project_name): - raise ValueError( - "project name should contain only letters and hyphens/underscores (but not at the start or end)" - ) - - if namespace is not None: - config["namespace"] = namespace - - if not re.match(namestr_regex, namespace): - raise ValueError( - "namespace should contain only letters and hyphens/underscores (but not at the start or end)" - ) if nebari_domain is None and not disable_prompt: nebari_domain = input("Provide domain: ") - config["domain"] = nebari_domain - - # In nebari_version only use major.minor.patch version - drop any pre/post/dev suffixes - config["nebari_version"] = __version__ - # Generate default password for Keycloak root user and also example-user if using password auth - default_password = "".join( - secrets.choice(string.ascii_letters + string.digits) for i in range(16) + config = schema.Main( + project_name=project_name, + domain=nebari_domain, + provider=cloud_provider, + namespace=namespace, + nebari_version=__version__ ) + config.ci_cd.type = ci_provider + config.terraform_state.type = terraform_state # Save default password to file default_password_filename = Path(tempfile.gettempdir()) / "NEBARI_DEFAULT_PASSWORD" with open(default_password_filename, "w") as f: - f.write(default_password) - default_password_filename.chmod(0o700) + f.write(config.security.keycloak.initial_root_password) + os.chmod(default_password_filename, 0o700) - config["theme"]["jupyterhub"]["hub_title"] = f"Nebari - { project_name }" - config["theme"]["jupyterhub"][ - "welcome" - ] = """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""" + config.theme.jupyterhub.hub_title = f"Nebari - { project_name }" + config.theme.jupyterhub.welcome = """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""" - if auth_provider == "github": - config["security"]["authentication"] = AUTH_OAUTH_GITHUB.copy() + config.security.authentication.type = auth_provider + if config.security.authentication.type == schema.AuthenticationEnum.github: if not disable_prompt: - config["security"]["authentication"]["config"]["client_id"] = input( - "Github client_id: " + config.security.authentication.config = schema.GithubConfig( + client_id = input("Github client_id: "), + client_secret = input("Github client_secret: "), ) - config["security"]["authentication"]["config"]["client_secret"] = input( - "Github client_secret: " + elif config.security.authentication.type == schema.AuthenticationEnum.auth0: + if auth_auto_provision: + auth0_config = create_client(config.domain, config.project_name) + config.security.authentication.config = schema.Auth0Config(**auth0_config) + else: + config.security.authentication.config = schema.Auth0Config( + client_id = input("Auth0 client_id: "), + client_secret = input("Auth0 client_secret: "), + auth0_subdomain = input("Auth0 subdomain: "), ) - elif auth_provider == "auth0": - config["security"]["authentication"] = AUTH_OAUTH_AUTH0.copy() - - elif auth_provider == "password": - config["security"]["authentication"] = AUTH_PASSWORD.copy() - - # Always use default password for keycloak root - config["security"].setdefault("keycloak", {})[ - "initial_root_password" - ] = default_password - - if cloud_provider == "do": - config["theme"]["jupyterhub"][ - "hub_subtitle" - ] = f"{WELCOME_HEADER_TEXT} on Digital Ocean" - config["digital_ocean"] = DIGITAL_OCEAN.copy() - set_kubernetes_version(config, kubernetes_version, cloud_provider) - - elif cloud_provider == "gcp": - config["theme"]["jupyterhub"][ - "hub_subtitle" - ] = f"{WELCOME_HEADER_TEXT} on Google Cloud Platform" - config["google_cloud_platform"] = GOOGLE_PLATFORM.copy() - set_kubernetes_version(config, kubernetes_version, cloud_provider) + if config.provider == schema.ProviderEnum.do: + config.theme.jupyterhub.hub_subtitle = f"{WELCOME_HEADER_TEXT} on Digital Ocean" + elif config.provider == schema.ProviderEnum.gcp: + config.theme.jupyterhub.hub_subtitle = f"{WELCOME_HEADER_TEXT} on Google Cloud Platform" if "PROJECT_ID" in os.environ: - config["google_cloud_platform"]["project"] = os.environ["PROJECT_ID"] + config.google_cloud_platform.project = os.environ["PROJECT_ID"] elif not disable_prompt: - config["google_cloud_platform"]["project"] = input( + config.google_cloud_platform.project = input( "Enter Google Cloud Platform Project ID: " ) - - elif cloud_provider == "azure": - config["theme"]["jupyterhub"][ - "hub_subtitle" - ] = f"{WELCOME_HEADER_TEXT} on Azure" - config["azure"] = AZURE.copy() - set_kubernetes_version(config, kubernetes_version, cloud_provider) - - elif cloud_provider == "aws": - config["theme"]["jupyterhub"][ - "hub_subtitle" - ] = f"{WELCOME_HEADER_TEXT} on Amazon Web Services" - config["amazon_web_services"] = AMAZON_WEB_SERVICES.copy() - set_kubernetes_version(config, kubernetes_version, cloud_provider) - if "AWS_DEFAULT_REGION" in os.environ: - config["amazon_web_services"]["region"] = os.environ["AWS_DEFAULT_REGION"] - + elif config.provider == schema.ProviderEnum.azure: + config.theme.jupyterhub.hub_subtitle = f"{WELCOME_HEADER_TEXT} on Azure" + elif config.provider == schema.ProviderEnum.aws: + config.theme.jupyterhub.hub_subtitle = f"{WELCOME_HEADER_TEXT} on Amazon Web Services" elif cloud_provider == "existing": - config["theme"]["jupyterhub"]["hub_subtitle"] = WELCOME_HEADER_TEXT - config["existing"] = EXISTING.copy() - + config.theme.jupyterhub.hub_subtitle = WELCOME_HEADER_TEXT elif cloud_provider == "local": - config["theme"]["jupyterhub"]["hub_subtitle"] = WELCOME_HEADER_TEXT - config["local"] = LOCAL.copy() - - config["profiles"] = DEFAULT_PROFILES.copy() - config["environments"] = default_environments().copy() - - if ssl_cert_email is not None: - config["certificate"] = { - "type": "lets-encrypt", - "acme_email": ssl_cert_email, - "acme_server": "https://acme-v02.api.letsencrypt.org/directory", - } + config.theme.jupyterhub.hub_subtitle = WELCOME_HEADER_TEXT - if auth_auto_provision: - if auth_provider == "auth0": - auth0_auto_provision(config) + if ssl_cert_email: + config.certificate.type = CertificateEnum.letsencrypt + config.certificate.acme_email = ssl_cert_email if repository_auto_provision: GITHUB_REGEX = "(https://)?github.com/([^/]+)/([^/]+)/?" @@ -481,7 +119,7 @@ def render_config( return config -def github_auto_provision(config, owner, repo): +def github_auto_provision(config: schema.Main, owner: str, repo: str): check_cloud_credentials( config ) # We may need env vars such as AWS_ACCESS_KEY_ID depending on provider @@ -498,8 +136,8 @@ def github_auto_provision(config, owner, repo): github.create_repository( owner, repo, - description=f'Nebari {config["project_name"]}-{config["provider"]}', - homepage=f'https://{config["domain"]}', + description=f'Nebari {config.project_name}-{config.provider}', + homepage=f'https://{config.domain}', ) except requests.exceptions.HTTPError as he: raise ValueError( @@ -510,7 +148,7 @@ def github_auto_provision(config, owner, repo): try: # Secrets - if config["provider"] == "do": + if config.provider == schema.ProviderEnum.do: for name in { "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", @@ -519,17 +157,17 @@ def github_auto_provision(config, owner, repo): "DIGITALOCEAN_TOKEN", }: github.update_secret(owner, repo, name, os.environ[name]) - elif config["provider"] == "aws": + elif config.provider == schema.ProviderEnum.aws: for name in { "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", }: github.update_secret(owner, repo, name, os.environ[name]) - elif config["provider"] == "gcp": + elif config.provider == schema.ProviderEnum.gcp: github.update_secret(owner, repo, "PROJECT_ID", os.environ["PROJECT_ID"]) with open(os.environ["GOOGLE_CREDENTIALS"]) as f: github.update_secret(owner, repo, "GOOGLE_CREDENTIALS", f.read()) - elif config["provider"] == "azure": + elif config.provider == schema.ProviderEnum.azure: for name in { "ARM_CLIENT_ID", "ARM_CLIENT_SECRET", @@ -552,16 +190,3 @@ def git_repository_initialize(git_repository): if not git.is_git_repo(Path.cwd()): git.initialize_git(Path.cwd()) git.add_git_remote(git_repository, path=Path.cwd(), remote_name="origin") - - -def auth0_auto_provision(config): - auth0_config = create_client(config["domain"], config["project_name"]) - config["security"]["authentication"]["config"]["client_id"] = auth0_config[ - "client_id" - ] - config["security"]["authentication"]["config"]["client_secret"] = auth0_config[ - "client_secret" - ] - config["security"]["authentication"]["config"]["auth0_subdomain"] = auth0_config[ - "auth0_subdomain" - ] diff --git a/src/_nebari/provider/oauth/auth0.py b/src/_nebari/provider/oauth/auth0.py index 525811052..dd714ec48 100644 --- a/src/_nebari/provider/oauth/auth0.py +++ b/src/_nebari/provider/oauth/auth0.py @@ -7,7 +7,7 @@ logger = logging.getLogger(__name__) -def create_client(jupyterhub_endpoint, project_name, reuse_existing=True): +def create_client(jupyterhub_endpoint: str, project_name: str, reuse_existing=True): for variable in {"AUTH0_DOMAIN", "AUTH0_CLIENT_ID", "AUTH0_CLIENT_SECRET"}: if variable not in os.environ: raise ValueError(f"Required environment variable={variable} not defined") diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index df580e98b..eb25a2536 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -3,70 +3,11 @@ from _nebari.provider.cicd.github import gen_nebari_linter, gen_nebari_ops from _nebari.provider.cicd.gitlab import gen_gitlab_ci +from _nebari.utils import check_cloud_credentials from nebari import schema from nebari.hookspecs import NebariStage, hookimpl -def check_cloud_credentials(config: schema.Main): - if config.provider == schema.ProviderEnum.gcp: - for variable in {"GOOGLE_CREDENTIALS"}: - if variable not in os.environ: - raise ValueError( - f"""Missing the following required environment variable: {variable}\n - Please see the documentation for more information: {GCP_ENV_DOCS}""" - ) - elif config.provider == schema.ProviderEnum.azure: - for variable in { - "ARM_CLIENT_ID", - "ARM_CLIENT_SECRET", - "ARM_SUBSCRIPTION_ID", - "ARM_TENANT_ID", - }: - if variable not in os.environ: - raise ValueError( - f"""Missing the following required environment variable: {variable}\n - Please see the documentation for more information: {AZURE_ENV_DOCS}""" - ) - elif config.provider == schema.ProviderEnum.aws: - for variable in { - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - }: - if variable not in os.environ: - raise ValueError( - f"""Missing the following required environment variable: {variable}\n - Please see the documentation for more information: {AWS_ENV_DOCS}""" - ) - elif config.provider == schema.ProviderEnum.do: - for variable in { - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "SPACES_ACCESS_KEY_ID", - "SPACES_SECRET_ACCESS_KEY", - "DIGITALOCEAN_TOKEN", - }: - if variable not in os.environ: - raise ValueError( - f"""Missing the following required environment variable: {variable}\n - Please see the documentation for more information: {DO_ENV_DOCS}""" - ) - - if os.environ["AWS_ACCESS_KEY_ID"] != os.environ["SPACES_ACCESS_KEY_ID"]: - raise ValueError( - f"""The environment variables AWS_ACCESS_KEY_ID and SPACES_ACCESS_KEY_ID must be equal\n - See {DO_ENV_DOCS} for more information""" - ) - - if ( - os.environ["AWS_SECRET_ACCESS_KEY"] - != os.environ["SPACES_SECRET_ACCESS_KEY"] - ): - raise ValueError( - f"""The environment variables AWS_SECRET_ACCESS_KEY and SPACES_SECRET_ACCESS_KEY must be equal\n - See {DO_ENV_DOCS} for more information""" - ) - - def gen_gitignore(): """ Generate `.gitignore` file. diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index fb464a9a6..d7df3bb68 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -1,108 +1,312 @@ from _nebari.cli.init import ( - check_auth_provider_creds, - check_cloud_provider_creds, check_project_name, check_ssl_cert_email, - enum_to_list, guided_init_wizard, handle_init, ) +import rich -@app.command() -def init( - cloud_provider: str = typer.Argument( - "local", - help=f"options: {enum_to_list(ProviderEnum)}", - callback=check_cloud_provider_creds, - is_eager=True, - ), - # Although this unused below, the functionality is contained in the callback. Thus, - # this attribute cannot be removed. - guided_init: bool = typer.Option( - False, - help=GUIDED_INIT_MSG, - callback=guided_init_wizard, - is_eager=True, - ), - project_name: str = typer.Option( - ..., - "--project-name", - "--project", - "-p", - callback=check_project_name, - ), - domain_name: str = typer.Option( - ..., - "--domain-name", - "--domain", - "-d", - ), - namespace: str = typer.Option( - "dev", - ), - auth_provider: str = typer.Option( - "password", - help=f"options: {enum_to_list(AuthenticationEnum)}", - callback=check_auth_provider_creds, - ), - auth_auto_provision: bool = typer.Option( - False, - ), - repository: str = typer.Option( - None, - help=f"options: {enum_to_list(GitRepoEnum)}", - ), - repository_auto_provision: bool = typer.Option( - False, - ), - ci_provider: str = typer.Option( - None, - help=f"options: {enum_to_list(CiEnum)}", - ), - terraform_state: str = typer.Option( - "remote", help=f"options: {enum_to_list(TerraformStateEnum)}" - ), - kubernetes_version: str = typer.Option( - "latest", - ), - ssl_cert_email: str = typer.Option( - None, - callback=check_ssl_cert_email, - ), - disable_prompt: bool = typer.Option( - False, - is_eager=True, - ), -): - """ - Create and initialize your [purple]nebari-config.yaml[/purple] file. - - This command will create and initialize your [purple]nebari-config.yaml[/purple] :sparkles: - - This file contains all your Nebari cluster configuration details and, - is used as input to later commands such as [green]nebari render[/green], [green]nebari deploy[/green], etc. - - If you're new to Nebari, we recommend you use the Guided Init wizard. - To get started simply run: - - [green]nebari init --guided-init[/green] - - """ - inputs = InitInputs() - - inputs.cloud_provider = cloud_provider - inputs.project_name = project_name - inputs.domain_name = domain_name - inputs.namespace = namespace - inputs.auth_provider = auth_provider - inputs.auth_auto_provision = auth_auto_provision - inputs.repository = repository - inputs.repository_auto_provision = repository_auto_provision - inputs.ci_provider = ci_provider - inputs.terraform_state = terraform_state - inputs.kubernetes_version = kubernetes_version - inputs.ssl_cert_email = ssl_cert_email - inputs.disable_prompt = disable_prompt - - handle_init(inputs) +import os +import json +import pathlib +from typing import Tuple + +import typer + +from _nebari.keycloak import do_keycloak, export_keycloak_users +from nebari.hookspecs import hookimpl +from nebari import schema + +MISSING_CREDS_TEMPLATE = "Unable to locate your {provider} credentials, refer to this guide on how to generate them:\n\n[green]\t{link_to_docs}[/green]\n\n" +LINKS_TO_DOCS_TEMPLATE = ( + "For more details, refer to the Nebari docs:\n\n\t[green]{link_to_docs}[/green]\n\n" +) + +# links to external docs +CREATE_AWS_CREDS = ( + "https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html" +) +CREATE_GCP_CREDS = ( + "https://cloud.google.com/iam/docs/creating-managing-service-accounts" +) +CREATE_DO_CREDS = ( + "https://docs.digitalocean.com/reference/api/create-personal-access-token" +) +CREATE_AZURE_CREDS = "https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/service_principal_client_secret#creating-a-service-principal-in-the-azure-portal" +CREATE_AUTH0_CREDS = "https://auth0.com/docs/get-started/auth0-overview/create-applications/machine-to-machine-apps" +CREATE_GITHUB_OAUTH_CREDS = "https://docs.github.com/en/developers/apps/building-oauth-apps/creating-an-oauth-app" + +# links to Nebari docs +DOCS_HOME = "https://nebari.dev/docs/" +CHOOSE_CLOUD_PROVIDER = "https://nebari.dev/docs/get-started/deploy" + +GUIDED_INIT_MSG = ( + "[bold green]START HERE[/bold green] - this will guide you step-by-step " + "to generate your [purple]nebari-config.yaml[/purple]. " + "It is an [i]alternative[/i] to passing the options listed below." +) + +def enum_to_list(enum_cls): + return ', '.join([e.value.lower() for e in enum_cls]) + + +def check_auth_provider_creds(ctx: typer.Context, auth_provider: str): + """Validate the the necessary auth provider credentials have been set as environment variables.""" + if ctx.params.get("disable_prompt"): + return auth_provider + + auth_provider = auth_provider.lower() + + # Auth0 + if auth_provider == schema.AuthenticationEnum.auth0.value.lower() and ( + not os.environ.get("AUTH0_CLIENT_ID") + or not os.environ.get("AUTH0_CLIENT_SECRET") + or not os.environ.get("AUTH0_DOMAIN") + ): + rich.print( + MISSING_CREDS_TEMPLATE.format( + provider="Auth0", link_to_docs=CREATE_AUTH0_CREDS + ) + ) + + os.environ["AUTH0_CLIENT_ID"] = typer.prompt( + "Paste your AUTH0_CLIENT_ID", + hide_input=True, + ) + os.environ["AUTH0_CLIENT_SECRET"] = typer.prompt( + "Paste your AUTH0_CLIENT_SECRET", + hide_input=True, + ) + os.environ["AUTH0_DOMAIN"] = typer.prompt( + "Paste your AUTH0_DOMAIN", + hide_input=True, + ) + + # GitHub + elif auth_provider == schema.AuthenticationEnum.github.value.lower() and ( + not os.environ.get("GITHUB_CLIENT_ID") + or not os.environ.get("GITHUB_CLIENT_SECRET") + ): + rich.print( + MISSING_CREDS_TEMPLATE.format( + provider="GitHub OAuth App", link_to_docs=CREATE_GITHUB_OAUTH_CREDS + ) + ) + + os.environ["GITHUB_CLIENT_ID"] = typer.prompt( + "Paste your GITHUB_CLIENT_ID", + hide_input=True, + ) + os.environ["GITHUB_CLIENT_SECRET"] = typer.prompt( + "Paste your GITHUB_CLIENT_SECRET", + hide_input=True, + ) + + return auth_provider + + +def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: str): + """Validate that the necessary cloud credentials have been set as environment variables.""" + if ctx.params.get("disable_prompt"): + return cloud_provider + + cloud_provider = cloud_provider.lower() + + # AWS + if cloud_provider == schema.ProviderEnum.aws.value.lower() and ( + not os.environ.get("AWS_ACCESS_KEY_ID") + or not os.environ.get("AWS_SECRET_ACCESS_KEY") + ): + rich.print( + MISSING_CREDS_TEMPLATE.format( + provider="Amazon Web Services", link_to_docs=CREATE_AWS_CREDS + ) + ) + + os.environ["AWS_ACCESS_KEY_ID"] = typer.prompt( + "Paste your AWS_ACCESS_KEY_ID", + hide_input=True, + ) + os.environ["AWS_SECRET_ACCESS_KEY"] = typer.prompt( + "Paste your AWS_SECRET_ACCESS_KEY", + hide_input=True, + ) + + # GCP + elif cloud_provider == schema.ProviderEnum.gcp.value.lower() and ( + not os.environ.get("GOOGLE_CREDENTIALS") or not os.environ.get("PROJECT_ID") + ): + rich.print( + MISSING_CREDS_TEMPLATE.format( + provider="Google Cloud Provider", link_to_docs=CREATE_GCP_CREDS + ) + ) + + os.environ["GOOGLE_CREDENTIALS"] = typer.prompt( + "Paste your GOOGLE_CREDENTIALS", + hide_input=True, + ) + os.environ["PROJECT_ID"] = typer.prompt( + "Paste your PROJECT_ID", + hide_input=True, + ) + + # DO + elif cloud_provider == schema.ProviderEnum.do.value.lower() and ( + not os.environ.get("DIGITALOCEAN_TOKEN") + or not os.environ.get("SPACES_ACCESS_KEY_ID") + or not os.environ.get("SPACES_SECRET_ACCESS_KEY") + ): + rich.print( + MISSING_CREDS_TEMPLATE.format( + provider="Digital Ocean", link_to_docs=CREATE_DO_CREDS + ) + ) + + os.environ["DIGITALOCEAN_TOKEN"] = typer.prompt( + "Paste your DIGITALOCEAN_TOKEN", + hide_input=True, + ) + os.environ["SPACES_ACCESS_KEY_ID"] = typer.prompt( + "Paste your SPACES_ACCESS_KEY_ID", + hide_input=True, + ) + os.environ["SPACES_SECRET_ACCESS_KEY"] = typer.prompt( + "Paste your SPACES_SECRET_ACCESS_KEY", + hide_input=True, + ) + os.environ["AWS_ACCESS_KEY_ID"] = os.getenv("SPACES_ACCESS_KEY_ID") + os.environ["AWS_SECRET_ACCESS_KEY"] = os.getenv("AWS_SECRET_ACCESS_KEY") + + # AZURE + elif cloud_provider == schema.ProviderEnum.azure.value.lower() and ( + not os.environ.get("ARM_CLIENT_ID") + or not os.environ.get("ARM_CLIENT_SECRET") + or not os.environ.get("ARM_SUBSCRIPTION_ID") + or not os.environ.get("ARM_TENANT_ID") + ): + rich.print( + MISSING_CREDS_TEMPLATE.format( + provider="Azure", link_to_docs=CREATE_AZURE_CREDS + ) + ) + os.environ["ARM_CLIENT_ID"] = typer.prompt( + "Paste your ARM_CLIENT_ID", + hide_input=True, + ) + os.environ["ARM_SUBSCRIPTION_ID"] = typer.prompt( + "Paste your ARM_SUBSCRIPTION_ID", + hide_input=True, + ) + os.environ["ARM_TENANT_ID"] = typer.prompt( + "Paste your ARM_TENANT_ID", + hide_input=True, + ) + os.environ["ARM_CLIENT_SECRET"] = typer.prompt( + "Paste your ARM_CLIENT_SECRET", + hide_input=True, + ) + + return cloud_provider + + +@hookimpl +def nebari_subcommand(cli: typer.Typer): + @cli.command() + def init( + cloud_provider: schema.ProviderEnum = typer.Argument( + schema.ProviderEnum.local, + help=f"options: {enum_to_list(schema.ProviderEnum)}", + callback=check_cloud_provider_creds, + is_eager=True, + ), + # Although this unused below, the functionality is contained in the callback. Thus, + # this attribute cannot be removed. + guided_init: bool = typer.Option( + False, + help=GUIDED_INIT_MSG, + callback=guided_init_wizard, + is_eager=True, + ), + project_name: str = typer.Option( + ..., + "--project-name", + "--project", + "-p", + callback=check_project_name, + ), + domain_name: str = typer.Option( + ..., + "--domain-name", + "--domain", + "-d", + ), + namespace: str = typer.Option( + "dev", + ), + auth_provider: schema.AuthenticationEnum = typer.Option( + schema.AuthenticationEnum.password, + help=f"options: {enum_to_list(schema.AuthenticationEnum)}", + callback=check_auth_provider_creds, + ), + auth_auto_provision: bool = typer.Option( + False, + ), + repository: schema.GitRepoEnum = typer.Option( + None, + help=f"options: {enum_to_list(schema.GitRepoEnum)}", + ), + repository_auto_provision: bool = typer.Option( + False, + ), + ci_provider: schema.CiEnum = typer.Option( + schema.CiEnum.none, + help=f"options: {enum_to_list(schema.CiEnum)}", + ), + terraform_state: schema.TerraformStateEnum = typer.Option( + schema.TerraformStateEnum.remote, help=f"options: {enum_to_list(schema.TerraformStateEnum)}" + ), + kubernetes_version: str = typer.Option( + "latest", + ), + ssl_cert_email: str = typer.Option( + None, + callback=check_ssl_cert_email, + ), + disable_prompt: bool = typer.Option( + False, + is_eager=True, + ), + ): + """ + Create and initialize your [purple]nebari-config.yaml[/purple] file. + + This command will create and initialize your [purple]nebari-config.yaml[/purple] :sparkles: + + This file contains all your Nebari cluster configuration details and, + is used as input to later commands such as [green]nebari render[/green], [green]nebari deploy[/green], etc. + + If you're new to Nebari, we recommend you use the Guided Init wizard. + To get started simply run: + + [green]nebari init --guided-init[/green] + + """ + inputs = schema.InitInputs() + + inputs.cloud_provider = cloud_provider + inputs.project_name = project_name + inputs.domain_name = domain_name + inputs.namespace = namespace + inputs.auth_provider = auth_provider + inputs.auth_auto_provision = auth_auto_provision + inputs.repository = repository + inputs.repository_auto_provision = repository_auto_provision + inputs.ci_provider = ci_provider + inputs.terraform_state = terraform_state + inputs.kubernetes_version = kubernetes_version + inputs.ssl_cert_email = ssl_cert_email + inputs.disable_prompt = disable_prompt + + handle_init(inputs) diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 206c25a24..19576970e 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -19,12 +19,11 @@ digital_ocean, google_cloud, ) +from nebari import schema # environment variable overrides NEBARI_K8S_VERSION = os.getenv("NEBARI_K8S_VERSION", None) NEBARI_GH_BRANCH = os.getenv("NEBARI_GH_BRANCH", None) -NEBARI_IMAGE_TAG = os.getenv("NEBARI_IMAGE_TAG", None) -NEBARI_DASK_VERSION = os.getenv("NEBARI_DASK_VERSION", None) DO_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-do" AWS_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-aws" @@ -33,9 +32,6 @@ CONDA_FORGE_CHANNEL_DATA_URL = "https://conda.anaconda.org/conda-forge/channeldata.json" -# Regex for suitable project names -namestr_regex = r"^[A-Za-z][A-Za-z\-_]*[A-Za-z]$" - # Create a ruamel object with our favored config, for universal use yaml = YAML() yaml.preserve_quotes = True @@ -142,6 +138,66 @@ def backup_config_file(filename: Path, extrasuffix: str = ""): print(f"Backing up {filename} as {backup_filename}") +def check_cloud_credentials(config: schema.Main): + if config.provider == schema.ProviderEnum.gcp: + for variable in {"GOOGLE_CREDENTIALS"}: + if variable not in os.environ: + raise ValueError( + f"""Missing the following required environment variable: {variable}\n + Please see the documentation for more information: {GCP_ENV_DOCS}""" + ) + elif config.provider == schema.ProviderEnum.azure: + for variable in { + "ARM_CLIENT_ID", + "ARM_CLIENT_SECRET", + "ARM_SUBSCRIPTION_ID", + "ARM_TENANT_ID", + }: + if variable not in os.environ: + raise ValueError( + f"""Missing the following required environment variable: {variable}\n + Please see the documentation for more information: {AZURE_ENV_DOCS}""" + ) + elif config.provider == schema.ProviderEnum.aws: + for variable in { + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + }: + if variable not in os.environ: + raise ValueError( + f"""Missing the following required environment variable: {variable}\n + Please see the documentation for more information: {AWS_ENV_DOCS}""" + ) + elif config.provider == schema.ProviderEnum.do: + for variable in { + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "SPACES_ACCESS_KEY_ID", + "SPACES_SECRET_ACCESS_KEY", + "DIGITALOCEAN_TOKEN", + }: + if variable not in os.environ: + raise ValueError( + f"""Missing the following required environment variable: {variable}\n + Please see the documentation for more information: {DO_ENV_DOCS}""" + ) + + if os.environ["AWS_ACCESS_KEY_ID"] != os.environ["SPACES_ACCESS_KEY_ID"]: + raise ValueError( + f"""The environment variables AWS_ACCESS_KEY_ID and SPACES_ACCESS_KEY_ID must be equal\n + See {DO_ENV_DOCS} for more information""" + ) + + if ( + os.environ["AWS_SECRET_ACCESS_KEY"] + != os.environ["SPACES_SECRET_ACCESS_KEY"] + ): + raise ValueError( + f"""The environment variables AWS_SECRET_ACCESS_KEY and SPACES_SECRET_ACCESS_KEY must be equal\n + See {DO_ENV_DOCS} for more information""" + ) + + def set_kubernetes_version( config, kubernetes_version, cloud_provider, grab_latest_version=True ): @@ -276,33 +332,3 @@ def deep_merge(*args): return [*d1, *d2] else: # if they don't match use left one return d1 - - -def set_docker_image_tag() -> str: - """Set docker image tag for `jupyterlab`, `jupyterhub`, and `dask-worker`.""" - - if NEBARI_IMAGE_TAG: - return NEBARI_IMAGE_TAG - - return DEFAULT_NEBARI_IMAGE_TAG - - -def set_nebari_dask_version() -> str: - """Set version of `nebari-dask` meta package.""" - - if NEBARI_DASK_VERSION: - return NEBARI_DASK_VERSION - - return DEFAULT_NEBARI_DASK_VERSION - - -def is_relative_to(self: Path, other: Path, /) -> bool: - """Compatibility function to bring ``Path.is_relative_to`` to Python 3.8""" - if sys.version_info[:2] >= (3, 9): - return self.is_relative_to(other) - - try: - self.relative_to(other) - return True - except ValueError: - return False diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index 69a66c494..ce2b772ad 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -17,6 +17,7 @@ "_nebari.stages.kubernetes_services", "_nebari.stages.nebari_tf_extensions", # subcommands + "_nebari.subcommands.init", "_nebari.subcommands.dev", "_nebari.subcommands.deploy", "_nebari.subcommands.destroy", diff --git a/src/nebari/schema.py b/src/nebari/schema.py index dfe29e242..13e490680 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -10,7 +10,11 @@ import pydantic from pydantic import Field, root_validator, validator -from ruamel.yaml import YAML + +from ruamel.yaml import YAML, yaml_object +yaml = YAML() +yaml.preserve_quotes = True +yaml.default_flow_style = False from _nebari import constants from _nebari.provider.cloud import ( @@ -19,23 +23,52 @@ digital_ocean, google_cloud, ) -from _nebari.utils import namestr_regex, set_docker_image_tag, set_nebari_dask_version from _nebari.version import __version__, rounded_ver_parse +# Regex for suitable project names +namestr_regex = r"^[A-Za-z][A-Za-z\-_]*[A-Za-z]$" + + +def random_secure_string(length: int = 32, chars: str = string.ascii_lowercase + string.digits): + return "".join( + secrets.choice(chars) for i in range(length) + ) + + +def set_docker_image_tag() -> str: + """Set docker image tag for `jupyterlab`, `jupyterhub`, and `dask-worker`.""" + return os.environ.get('NEBARI_IMAGE_TAG', constants.DEFAULT_NEBARI_IMAGE_TAG) + + +def set_nebari_dask_version() -> str: + """Set version of `nebari-dask` meta package.""" + return os.environ.get('NEBARI_DASK_VERSION', constants.DEFAULT_NEBARI_DASK_VERSION) + +@yaml_object(yaml) class CertificateEnum(str, enum.Enum): letsencrypt = "lets-encrypt" selfsigned = "self-signed" existing = "existing" disabled = "disabled" + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_str(node.value) + +@yaml_object(yaml) class TerraformStateEnum(str, enum.Enum): remote = "remote" local = "local" existing = "existing" + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_str(node.value) + +@yaml_object(yaml) class ProviderEnum(str, enum.Enum): local = "local" existing = "existing" @@ -44,30 +77,54 @@ class ProviderEnum(str, enum.Enum): gcp = "gcp" azure = "azure" + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_str(node.value) + +@yaml_object(yaml) class GitRepoEnum(str, enum.Enum): github = "github.com" gitlab = "gitlab.com" + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_str(node.value) + +@yaml_object(yaml) class CiEnum(str, enum.Enum): github_actions = "github-actions" gitlab_ci = "gitlab-ci" none = "none" + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_str(node.value) + +@yaml_object(yaml) class AuthenticationEnum(str, enum.Enum): password = "password" github = "GitHub" auth0 = "Auth0" custom = "custom" + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_str(node.value) + +@yaml_object(yaml) class AccessEnum(str, enum.Enum): all = "all" yaml = "yaml" keycloak = "keycloak" + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_str(node.value) + class Base(pydantic.BaseModel): ... @@ -172,7 +229,7 @@ class Certificate(Base): secret_name: typing.Optional[str] # lets-encrypt acme_email: typing.Optional[str] - acme_server: typing.Optional[str] + acme_server: str = "https://acme-v02.api.letsencrypt.org/directory" # ========== Default Images ============== @@ -255,14 +312,9 @@ class GitHubAuthentication(Authentication): # ================= Keycloak ================== -def random_password(length: int = 32): - return "".join( - secrets.choice(string.ascii_letters + string.digits) for i in range(16) - ) - class Keycloak(Base): - initial_root_password: str = Field(default_factory=random_password) + initial_root_password: str = Field(default_factory=random_secure_string) overrides: typing.Dict = {} realm_display_name: str = "Nebari" @@ -357,6 +409,7 @@ class DigitalOceanProvider(Base): kubernetes_version: str = Field( default_factory=lambda: digital_ocean.kubernetes_versions()[-1] ) + # Digital Ocean image slugs are listed here https://slugs.do-api.dev/ node_groups: typing.Dict[str, NodeGroup] = { "general": NodeGroup(instance="g-8vcpu-32gb", min_nodes=1, max_nodes=1), "user": NodeGroup(instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5), @@ -423,7 +476,7 @@ class AzureProvider(Base): "user": NodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), "worker": NodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), } - storage_account_postfix: str + storage_account_postfix: str = Field(default_factory=random_secure_string) vnet_subnet_id: typing.Optional[typing.Union[str, None]] = None private_cluster_enabled: bool = False @@ -438,7 +491,7 @@ def _validate_kubernetes_version(cls, value): class AmazonWebServicesProvider(Base): - region: str = "us-west-2" + region: str = Field(default_factory=lambda: os.environ.get("AWS_DEFAULT_REGION", "us-west-2")) availability_zones: typing.Optional[typing.List[str]] kubernetes_version: str = Field( default_factory=lambda: amazon_web_services.kubernetes_versions()[-1] @@ -447,10 +500,10 @@ class AmazonWebServicesProvider(Base): node_groups: typing.Dict[str, AWSNodeGroup] = { "general": NodeGroup(instance="m5.2xlarge", min_nodes=1, max_nodes=1), "user": NodeGroup( - instance="m5.xlarge", min_nodes=0, max_nodes=5, single_subnet=False + instance="m5.xlarge", min_nodes=1, max_nodes=5, single_subnet=False ), "worker": NodeGroup( - instance="m5.xlarge", min_nodes=0, max_nodes=5, single_subnet=False + instance="m5.xlarge", min_nodes=1, max_nodes=5, single_subnet=False ), } existing_subnet_ids: typing.Optional[typing.List[str]] @@ -754,16 +807,16 @@ def project_name_convention(value: typing.Any, values): class InitInputs(Base): - cloud_provider: typing.Type[ProviderEnum] = "local" + cloud_provider: ProviderEnum = ProviderEnum.local project_name: str = "" domain_name: str = "" namespace: typing.Optional[letter_dash_underscore_pydantic] = "dev" - auth_provider: typing.Type[AuthenticationEnum] = "password" + auth_provider: AuthenticationEnum = AuthenticationEnum.password auth_auto_provision: bool = False repository: typing.Union[str, None] = None repository_auto_provision: bool = False - ci_provider: typing.Optional[CiEnum] = None - terraform_state: typing.Optional[TerraformStateEnum] = "remote" + ci_provider: CiEnum = CiEnum.none + terraform_state: TerraformStateEnum = TerraformStateEnum.remote kubernetes_version: typing.Union[str, None] = None ssl_cert_email: typing.Union[str, None] = None disable_prompt: bool = False @@ -773,6 +826,7 @@ class Main(Base): provider: ProviderEnum = ProviderEnum.local project_name: str namespace: letter_dash_underscore_pydantic = "dev" + # In nebari_version only use major.minor.patch version - drop any pre/post/dev suffixes nebari_version: str = __version__ ci_cd: CICD = CICD() domain: str @@ -1028,3 +1082,12 @@ def read_configuration(config_filename: pathlib.Path, read_environment: bool = T config = set_config_from_environment_variables(config) return config + + +def write_configuration(config_filename: pathlib.Path, config: Main, mode: str = 'w'): + yaml = YAML() + yaml.preserve_quotes = True + yaml.default_flow_style = False + + with config_filename.open(mode) as f: + yaml.dump(config.dict(exclude_unset=True), f) From 9eb56c69a64ea7f36bbca93b91f06f3313cd1090 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Jun 2023 20:06:18 +0000 Subject: [PATCH 029/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/cli/init.py | 4 ++-- src/_nebari/initialize.py | 28 ++++++++++++++-------------- src/_nebari/subcommands/init.py | 24 ++++++++++-------------- src/_nebari/utils.py | 1 - src/nebari/schema.py | 21 ++++++++++++--------- 5 files changed, 38 insertions(+), 40 deletions(-) diff --git a/src/_nebari/cli/init.py b/src/_nebari/cli/init.py index 35eae6f8a..40f791cc5 100644 --- a/src/_nebari/cli/init.py +++ b/src/_nebari/cli/init.py @@ -1,6 +1,6 @@ import os -import re import pathlib +import re import questionary import rich @@ -72,7 +72,7 @@ def handle_init(inputs: schema.InitInputs): ) try: - schema.write_configuration(pathlib.Path("nebari-config.yaml"), config, mode='x') + schema.write_configuration(pathlib.Path("nebari-config.yaml"), config, mode="x") except FileExistsError: raise ValueError( "A nebari-config.yaml file already exists. Please move or delete it and try again." diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index b1aa67ea0..32dc9d318 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -1,9 +1,6 @@ import logging import os -import random import re -import secrets -import string import tempfile from pathlib import Path @@ -13,7 +10,6 @@ from _nebari.provider.cicd import github from _nebari.provider.oauth.auth0 import create_client from _nebari.utils import check_cloud_credentials - from _nebari.version import __version__ from nebari import schema @@ -48,7 +44,7 @@ def render_config( domain=nebari_domain, provider=cloud_provider, namespace=namespace, - nebari_version=__version__ + nebari_version=__version__, ) config.ci_cd.type = ci_provider config.terraform_state.type = terraform_state @@ -66,8 +62,8 @@ def render_config( if config.security.authentication.type == schema.AuthenticationEnum.github: if not disable_prompt: config.security.authentication.config = schema.GithubConfig( - client_id = input("Github client_id: "), - client_secret = input("Github client_secret: "), + client_id=input("Github client_id: "), + client_secret=input("Github client_secret: "), ) elif config.security.authentication.type == schema.AuthenticationEnum.auth0: if auth_auto_provision: @@ -75,15 +71,17 @@ def render_config( config.security.authentication.config = schema.Auth0Config(**auth0_config) else: config.security.authentication.config = schema.Auth0Config( - client_id = input("Auth0 client_id: "), - client_secret = input("Auth0 client_secret: "), - auth0_subdomain = input("Auth0 subdomain: "), + client_id=input("Auth0 client_id: "), + client_secret=input("Auth0 client_secret: "), + auth0_subdomain=input("Auth0 subdomain: "), ) if config.provider == schema.ProviderEnum.do: config.theme.jupyterhub.hub_subtitle = f"{WELCOME_HEADER_TEXT} on Digital Ocean" elif config.provider == schema.ProviderEnum.gcp: - config.theme.jupyterhub.hub_subtitle = f"{WELCOME_HEADER_TEXT} on Google Cloud Platform" + config.theme.jupyterhub.hub_subtitle = ( + f"{WELCOME_HEADER_TEXT} on Google Cloud Platform" + ) if "PROJECT_ID" in os.environ: config.google_cloud_platform.project = os.environ["PROJECT_ID"] elif not disable_prompt: @@ -93,7 +91,9 @@ def render_config( elif config.provider == schema.ProviderEnum.azure: config.theme.jupyterhub.hub_subtitle = f"{WELCOME_HEADER_TEXT} on Azure" elif config.provider == schema.ProviderEnum.aws: - config.theme.jupyterhub.hub_subtitle = f"{WELCOME_HEADER_TEXT} on Amazon Web Services" + config.theme.jupyterhub.hub_subtitle = ( + f"{WELCOME_HEADER_TEXT} on Amazon Web Services" + ) elif cloud_provider == "existing": config.theme.jupyterhub.hub_subtitle = WELCOME_HEADER_TEXT elif cloud_provider == "local": @@ -136,8 +136,8 @@ def github_auto_provision(config: schema.Main, owner: str, repo: str): github.create_repository( owner, repo, - description=f'Nebari {config.project_name}-{config.provider}', - homepage=f'https://{config.domain}', + description=f"Nebari {config.project_name}-{config.provider}", + homepage=f"https://{config.domain}", ) except requests.exceptions.HTTPError as he: raise ValueError( diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index d7df3bb68..ccc12a7c3 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -1,22 +1,16 @@ +import os + +import rich +import typer + from _nebari.cli.init import ( check_project_name, check_ssl_cert_email, guided_init_wizard, handle_init, ) - -import rich - -import os -import json -import pathlib -from typing import Tuple - -import typer - -from _nebari.keycloak import do_keycloak, export_keycloak_users -from nebari.hookspecs import hookimpl from nebari import schema +from nebari.hookspecs import hookimpl MISSING_CREDS_TEMPLATE = "Unable to locate your {provider} credentials, refer to this guide on how to generate them:\n\n[green]\t{link_to_docs}[/green]\n\n" LINKS_TO_DOCS_TEMPLATE = ( @@ -47,8 +41,9 @@ "It is an [i]alternative[/i] to passing the options listed below." ) + def enum_to_list(enum_cls): - return ', '.join([e.value.lower() for e in enum_cls]) + return ", ".join([e.value.lower() for e in enum_cls]) def check_auth_provider_creds(ctx: typer.Context, auth_provider: str): @@ -265,7 +260,8 @@ def init( help=f"options: {enum_to_list(schema.CiEnum)}", ), terraform_state: schema.TerraformStateEnum = typer.Option( - schema.TerraformStateEnum.remote, help=f"options: {enum_to_list(schema.TerraformStateEnum)}" + schema.TerraformStateEnum.remote, + help=f"options: {enum_to_list(schema.TerraformStateEnum)}", ), kubernetes_version: str = typer.Option( "latest", diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 19576970e..26fc44489 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -12,7 +12,6 @@ from ruamel.yaml import YAML -from _nebari.constants import DEFAULT_NEBARI_DASK_VERSION, DEFAULT_NEBARI_IMAGE_TAG from _nebari.provider.cloud import ( amazon_web_services, azure_cloud, diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 13e490680..edace6932 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -10,8 +10,8 @@ import pydantic from pydantic import Field, root_validator, validator - from ruamel.yaml import YAML, yaml_object + yaml = YAML() yaml.preserve_quotes = True yaml.default_flow_style = False @@ -29,20 +29,20 @@ namestr_regex = r"^[A-Za-z][A-Za-z\-_]*[A-Za-z]$" -def random_secure_string(length: int = 32, chars: str = string.ascii_lowercase + string.digits): - return "".join( - secrets.choice(chars) for i in range(length) - ) +def random_secure_string( + length: int = 32, chars: str = string.ascii_lowercase + string.digits +): + return "".join(secrets.choice(chars) for i in range(length)) def set_docker_image_tag() -> str: """Set docker image tag for `jupyterlab`, `jupyterhub`, and `dask-worker`.""" - return os.environ.get('NEBARI_IMAGE_TAG', constants.DEFAULT_NEBARI_IMAGE_TAG) + return os.environ.get("NEBARI_IMAGE_TAG", constants.DEFAULT_NEBARI_IMAGE_TAG) def set_nebari_dask_version() -> str: """Set version of `nebari-dask` meta package.""" - return os.environ.get('NEBARI_DASK_VERSION', constants.DEFAULT_NEBARI_DASK_VERSION) + return os.environ.get("NEBARI_DASK_VERSION", constants.DEFAULT_NEBARI_DASK_VERSION) @yaml_object(yaml) @@ -313,6 +313,7 @@ class GitHubAuthentication(Authentication): # ================= Keycloak ================== + class Keycloak(Base): initial_root_password: str = Field(default_factory=random_secure_string) overrides: typing.Dict = {} @@ -491,7 +492,9 @@ def _validate_kubernetes_version(cls, value): class AmazonWebServicesProvider(Base): - region: str = Field(default_factory=lambda: os.environ.get("AWS_DEFAULT_REGION", "us-west-2")) + region: str = Field( + default_factory=lambda: os.environ.get("AWS_DEFAULT_REGION", "us-west-2") + ) availability_zones: typing.Optional[typing.List[str]] kubernetes_version: str = Field( default_factory=lambda: amazon_web_services.kubernetes_versions()[-1] @@ -1084,7 +1087,7 @@ def read_configuration(config_filename: pathlib.Path, read_environment: bool = T return config -def write_configuration(config_filename: pathlib.Path, config: Main, mode: str = 'w'): +def write_configuration(config_filename: pathlib.Path, config: Main, mode: str = "w"): yaml = YAML() yaml.preserve_quotes = True yaml.default_flow_style = False From 613b82df7f2862f41cd1d0cfc73e3ffff07a2e22 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 16 Jun 2023 16:24:17 -0400 Subject: [PATCH 030/147] Make cli entirely extension subcommands --- src/_nebari/__main__.py | 2 +- src/_nebari/cli.py | 41 ++ src/_nebari/cli/__init__.py | 0 src/_nebari/cli/init.py | 562 ---------------------------- src/_nebari/cli/main.py | 442 ---------------------- src/_nebari/initialize.py | 2 +- src/_nebari/subcommands/__init__.py | 41 -- src/_nebari/subcommands/init.py | 367 +++++++++++++++++- src/nebari/__main__.py | 8 +- src/nebari/schema.py | 8 +- 10 files changed, 412 insertions(+), 1061 deletions(-) create mode 100644 src/_nebari/cli.py delete mode 100644 src/_nebari/cli/__init__.py delete mode 100644 src/_nebari/cli/init.py delete mode 100644 src/_nebari/cli/main.py diff --git a/src/_nebari/__main__.py b/src/_nebari/__main__.py index ca69c5354..b18eaf428 100644 --- a/src/_nebari/__main__.py +++ b/src/_nebari/__main__.py @@ -1,4 +1,4 @@ -from _nebari.subcommands import create_cli +from _nebari.cli import create_cli def main(): diff --git a/src/_nebari/cli.py b/src/_nebari/cli.py new file mode 100644 index 000000000..15df491e0 --- /dev/null +++ b/src/_nebari/cli.py @@ -0,0 +1,41 @@ +from typing import Optional + +import typer +from typer.core import TyperGroup + +from _nebari.version import __version__ +from nebari.plugins import pm + + +class OrderCommands(TyperGroup): + def list_commands(self, ctx: typer.Context): + """Return list of commands in the order appear.""" + return list(self.commands) + + +def create_cli(): + app = typer.Typer( + cls=OrderCommands, + help="Nebari CLI 🪴", + add_completion=False, + no_args_is_help=True, + rich_markup_mode="rich", + context_settings={"help_option_names": ["-h", "--help"]}, + ) + pm.hook.nebari_subcommand(cli=app) + + @app.callback(invoke_without_command=True) + def version( + version: Optional[bool] = typer.Option( + None, + "-V", + "--version", + help="Nebari version number", + is_eager=True, + ), + ): + if version: + print(__version__) + raise typer.Exit() + + return app diff --git a/src/_nebari/cli/__init__.py b/src/_nebari/cli/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/_nebari/cli/init.py b/src/_nebari/cli/init.py deleted file mode 100644 index 40f791cc5..000000000 --- a/src/_nebari/cli/init.py +++ /dev/null @@ -1,562 +0,0 @@ -import os -import pathlib -import re - -import questionary -import rich -import typer - -from _nebari.initialize import render_config -from nebari import schema - -MISSING_CREDS_TEMPLATE = "Unable to locate your {provider} credentials, refer to this guide on how to generate them:\n\n[green]\t{link_to_docs}[/green]\n\n" -LINKS_TO_DOCS_TEMPLATE = ( - "For more details, refer to the Nebari docs:\n\n\t[green]{link_to_docs}[/green]\n\n" -) - -# links to external docs -CREATE_AWS_CREDS = ( - "https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html" -) -CREATE_GCP_CREDS = ( - "https://cloud.google.com/iam/docs/creating-managing-service-accounts" -) -CREATE_DO_CREDS = ( - "https://docs.digitalocean.com/reference/api/create-personal-access-token" -) -CREATE_AZURE_CREDS = "https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/service_principal_client_secret#creating-a-service-principal-in-the-azure-portal" -CREATE_AUTH0_CREDS = "https://auth0.com/docs/get-started/auth0-overview/create-applications/machine-to-machine-apps" -CREATE_GITHUB_OAUTH_CREDS = "https://docs.github.com/en/developers/apps/building-oauth-apps/creating-an-oauth-app" - -# links to Nebari docs -DOCS_HOME = "https://nebari.dev/docs/" -CHOOSE_CLOUD_PROVIDER = "https://nebari.dev/docs/get-started/deploy" - - -def enum_to_list(enum_cls): - return [e.value.lower() for e in enum_cls] - - -def handle_init(inputs: schema.InitInputs): - """ - Take the inputs from the `nebari init` command, render the config and write it to a local yaml file. - """ - # if NEBARI_IMAGE_TAG: - # print( - # f"Modifying the image tags for the `default_images`, setting tags to: {NEBARI_IMAGE_TAG}" - # ) - - # if NEBARI_DASK_VERSION: - # print( - # f"Modifying the version of the `nebari_dask` package, setting version to: {NEBARI_DASK_VERSION}" - # ) - - # this will force the `set_kubernetes_version` to grab the latest version - if inputs.kubernetes_version == "latest": - inputs.kubernetes_version = None - - config = render_config( - cloud_provider=inputs.cloud_provider, - project_name=inputs.project_name, - nebari_domain=inputs.domain_name, - namespace=inputs.namespace, - auth_provider=inputs.auth_provider, - auth_auto_provision=inputs.auth_auto_provision, - ci_provider=inputs.ci_provider, - repository=inputs.repository, - repository_auto_provision=inputs.repository_auto_provision, - kubernetes_version=inputs.kubernetes_version, - terraform_state=inputs.terraform_state, - ssl_cert_email=inputs.ssl_cert_email, - disable_prompt=inputs.disable_prompt, - ) - - try: - schema.write_configuration(pathlib.Path("nebari-config.yaml"), config, mode="x") - except FileExistsError: - raise ValueError( - "A nebari-config.yaml file already exists. Please move or delete it and try again." - ) - - -def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: str): - """Validate that the necessary cloud credentials have been set as environment variables.""" - if ctx.params.get("disable_prompt"): - return cloud_provider - - cloud_provider = cloud_provider.lower() - - # AWS - if cloud_provider == schema.ProviderEnum.aws.value.lower() and ( - not os.environ.get("AWS_ACCESS_KEY_ID") - or not os.environ.get("AWS_SECRET_ACCESS_KEY") - ): - rich.print( - MISSING_CREDS_TEMPLATE.format( - provider="Amazon Web Services", link_to_docs=CREATE_AWS_CREDS - ) - ) - - os.environ["AWS_ACCESS_KEY_ID"] = typer.prompt( - "Paste your AWS_ACCESS_KEY_ID", - hide_input=True, - ) - os.environ["AWS_SECRET_ACCESS_KEY"] = typer.prompt( - "Paste your AWS_SECRET_ACCESS_KEY", - hide_input=True, - ) - - # GCP - elif cloud_provider == schema.ProviderEnum.gcp.value.lower() and ( - not os.environ.get("GOOGLE_CREDENTIALS") or not os.environ.get("PROJECT_ID") - ): - rich.print( - MISSING_CREDS_TEMPLATE.format( - provider="Google Cloud Provider", link_to_docs=CREATE_GCP_CREDS - ) - ) - - os.environ["GOOGLE_CREDENTIALS"] = typer.prompt( - "Paste your GOOGLE_CREDENTIALS", - hide_input=True, - ) - os.environ["PROJECT_ID"] = typer.prompt( - "Paste your PROJECT_ID", - hide_input=True, - ) - - # DO - elif cloud_provider == schema.ProviderEnum.do.value.lower() and ( - not os.environ.get("DIGITALOCEAN_TOKEN") - or not os.environ.get("SPACES_ACCESS_KEY_ID") - or not os.environ.get("SPACES_SECRET_ACCESS_KEY") - ): - rich.print( - MISSING_CREDS_TEMPLATE.format( - provider="Digital Ocean", link_to_docs=CREATE_DO_CREDS - ) - ) - - os.environ["DIGITALOCEAN_TOKEN"] = typer.prompt( - "Paste your DIGITALOCEAN_TOKEN", - hide_input=True, - ) - os.environ["SPACES_ACCESS_KEY_ID"] = typer.prompt( - "Paste your SPACES_ACCESS_KEY_ID", - hide_input=True, - ) - os.environ["SPACES_SECRET_ACCESS_KEY"] = typer.prompt( - "Paste your SPACES_SECRET_ACCESS_KEY", - hide_input=True, - ) - os.environ["AWS_ACCESS_KEY_ID"] = os.getenv("SPACES_ACCESS_KEY_ID") - os.environ["AWS_SECRET_ACCESS_KEY"] = os.getenv("AWS_SECRET_ACCESS_KEY") - - # AZURE - elif cloud_provider == schema.ProviderEnum.azure.value.lower() and ( - not os.environ.get("ARM_CLIENT_ID") - or not os.environ.get("ARM_CLIENT_SECRET") - or not os.environ.get("ARM_SUBSCRIPTION_ID") - or not os.environ.get("ARM_TENANT_ID") - ): - rich.print( - MISSING_CREDS_TEMPLATE.format( - provider="Azure", link_to_docs=CREATE_AZURE_CREDS - ) - ) - os.environ["ARM_CLIENT_ID"] = typer.prompt( - "Paste your ARM_CLIENT_ID", - hide_input=True, - ) - os.environ["ARM_SUBSCRIPTION_ID"] = typer.prompt( - "Paste your ARM_SUBSCRIPTION_ID", - hide_input=True, - ) - os.environ["ARM_TENANT_ID"] = typer.prompt( - "Paste your ARM_TENANT_ID", - hide_input=True, - ) - os.environ["ARM_CLIENT_SECRET"] = typer.prompt( - "Paste your ARM_CLIENT_SECRET", - hide_input=True, - ) - - return cloud_provider - - -def check_auth_provider_creds(ctx: typer.Context, auth_provider: str): - """Validate the the necessary auth provider credentials have been set as environment variables.""" - if ctx.params.get("disable_prompt"): - return auth_provider - - auth_provider = auth_provider.lower() - - # Auth0 - if auth_provider == AuthenticationEnum.auth0.value.lower() and ( - not os.environ.get("AUTH0_CLIENT_ID") - or not os.environ.get("AUTH0_CLIENT_SECRET") - or not os.environ.get("AUTH0_DOMAIN") - ): - rich.print( - MISSING_CREDS_TEMPLATE.format( - provider="Auth0", link_to_docs=CREATE_AUTH0_CREDS - ) - ) - - os.environ["AUTH0_CLIENT_ID"] = typer.prompt( - "Paste your AUTH0_CLIENT_ID", - hide_input=True, - ) - os.environ["AUTH0_CLIENT_SECRET"] = typer.prompt( - "Paste your AUTH0_CLIENT_SECRET", - hide_input=True, - ) - os.environ["AUTH0_DOMAIN"] = typer.prompt( - "Paste your AUTH0_DOMAIN", - hide_input=True, - ) - - # GitHub - elif auth_provider == AuthenticationEnum.github.value.lower() and ( - not os.environ.get("GITHUB_CLIENT_ID") - or not os.environ.get("GITHUB_CLIENT_SECRET") - ): - rich.print( - MISSING_CREDS_TEMPLATE.format( - provider="GitHub OAuth App", link_to_docs=CREATE_GITHUB_OAUTH_CREDS - ) - ) - - os.environ["GITHUB_CLIENT_ID"] = typer.prompt( - "Paste your GITHUB_CLIENT_ID", - hide_input=True, - ) - os.environ["GITHUB_CLIENT_SECRET"] = typer.prompt( - "Paste your GITHUB_CLIENT_SECRET", - hide_input=True, - ) - - return auth_provider - - -def check_project_name(ctx: typer.Context, project_name: str): - """Validate the project_name is acceptable. Depends on `cloud_provider`.""" - schema.project_name_convention( - project_name.lower(), {"provider": ctx.params["cloud_provider"]} - ) - - return project_name - - -def check_ssl_cert_email(ctx: typer.Context, ssl_cert_email: str): - """Validate the email used for SSL cert is in a valid format.""" - if ssl_cert_email and not re.match("^[^ @]+@[^ @]+\\.[^ @]+$", ssl_cert_email): - raise ValueError("ssl-cert-email should be a valid email address") - - return ssl_cert_email - - -def check_repository_creds(ctx: typer.Context, git_provider: str): - """Validate the necessary Git provider (GitHub) credentials are set.""" - - if ( - git_provider == GitRepoEnum.github.value.lower() - and not os.environ.get("GITHUB_USERNAME") - or not os.environ.get("GITHUB_TOKEN") - ): - os.environ["GITHUB_USERNAME"] = typer.prompt( - "Paste your GITHUB_USERNAME", - hide_input=True, - ) - os.environ["GITHUB_TOKEN"] = typer.prompt( - "Paste your GITHUB_TOKEN", - hide_input=True, - ) - - -def guided_init_wizard(ctx: typer.Context, guided_init: str): - """ - Guided Init Wizard is a user-friendly questionnaire used to help generate the `nebari-config.yaml`. - """ - qmark = " " - disable_checks = os.environ.get("NEBARI_DISABLE_INIT_CHECKS", False) - - if pathlib.Path("nebari-config.yaml").exists(): - raise ValueError( - "A nebari-config.yaml file already exists. Please move or delete it and try again." - ) - - if not guided_init: - return guided_init - - try: - rich.print( - ( - "\n\t[bold]Welcome to the Guided Init wizard![/bold]\n\n" - "You will be asked a few questions to generate your [purple]nebari-config.yaml[/purple]. " - f"{LINKS_TO_DOCS_TEMPLATE.format(link_to_docs=DOCS_HOME)}" - ) - ) - - if disable_checks: - rich.print( - "⚠️ Attempting to use the Guided Init wizard without any validation checks. There is no guarantee values provided will work! ⚠️\n\n" - ) - - # pull in default values for each of the below - inputs = schema.InitInputs() - - # CLOUD PROVIDER - rich.print( - ( - "\n 🪴 Nebari runs on a Kubernetes cluster: Where do you want this Kubernetes cluster deployed? " - "is where you want this Kubernetes cluster deployed. " - f"{LINKS_TO_DOCS_TEMPLATE.format(link_to_docs=CHOOSE_CLOUD_PROVIDER)}" - "\n\t❗️ [purple]local[/purple] requires Docker and Kubernetes running on your local machine. " - "[italic]Currently only available on Linux OS.[/italic]" - "\n\t❗️ [purple]existing[/purple] refers to an existing Kubernetes cluster that Nebari can be deployed on.\n" - ) - ) - # try: - inputs.cloud_provider = questionary.select( - "Where would you like to deploy your Nebari cluster?", - choices=enum_to_list(ProviderEnum), - qmark=qmark, - ).unsafe_ask() - - if not disable_checks: - check_cloud_provider_creds(ctx, cloud_provider=inputs.cloud_provider) - - # specific context needed when `check_project_name` is called - ctx.params["cloud_provider"] = inputs.cloud_provider - - name_guidelines = """ - The project name must adhere to the following requirements: - - Letters from A to Z (upper and lower case) and numbers - - Maximum accepted length of the name string is 16 characters - """ - if inputs.cloud_provider == schema.ProviderEnum.aws.value.lower(): - name_guidelines += "- Should NOT start with the string `aws`\n" - elif inputs.cloud_provider == ProviderEnum.azure.value.lower(): - name_guidelines += "- Should NOT contain `-`\n" - - # PROJECT NAME - rich.print( - ( - f"\n 🪴 Next, give your Nebari instance a project name. This name is what your Kubernetes cluster will be referred to as.\n{name_guidelines}\n" - ) - ) - inputs.project_name = questionary.text( - "What project name would you like to use?", - qmark=qmark, - validate=lambda text: True if len(text) > 0 else "Please enter a value", - ).unsafe_ask() - - if not disable_checks: - check_project_name(ctx, inputs.project_name) - - # DOMAIN NAME - rich.print( - ( - "\n\n 🪴 Great! Now you need to provide a valid domain name (i.e. the URL) to access your Nebri instance. " - "This should be a domain that you own.\n\n" - ) - ) - inputs.domain_name = questionary.text( - "What domain name would you like to use?", - qmark=qmark, - validate=lambda text: True if len(text) > 0 else "Please enter a value", - ).unsafe_ask() - - # AUTH PROVIDER - rich.print( - ( - # TODO once docs are updated, add links for more details - "\n\n 🪴 Nebari comes with [green]Keycloak[/green], an open-source identity and access management tool. This is how users and permissions " - "are managed on the platform. To connect Keycloak with an identity provider, you can select one now.\n\n" - "\n\t❗️ [purple]password[/purple] is the default option and is not connected to any external identity provider.\n" - ) - ) - inputs.auth_provider = questionary.select( - "What authentication provider would you like?", - choices=enum_to_list(AuthenticationEnum), - qmark=qmark, - ).unsafe_ask() - - if not disable_checks: - check_auth_provider_creds(ctx, auth_provider=inputs.auth_provider) - - if inputs.auth_provider.lower() == schemaAuthenticationEnum.auth0.value.lower(): - inputs.auth_auto_provision = questionary.confirm( - "Would you like us to auto provision the Auth0 Machine-to-Machine app?", - default=False, - qmark=qmark, - auto_enter=False, - ).unsafe_ask() - - elif inputs.auth_provider.lower() == AuthenticationEnum.github.value.lower(): - rich.print( - ( - ":warning: If you haven't done so already, please ensure the following:\n" - f"The `Homepage URL` is set to: [green]https://{inputs.domain_name}[/green]\n" - f"The `Authorization callback URL` is set to: [green]https://{inputs.domain_name}/auth/realms/nebari/broker/github/endpoint[/green]\n\n" - ) - ) - - # GITOPS - REPOSITORY, CICD - rich.print( - ( - "\n\n 🪴 This next section is [italic]optional[/italic] but recommended. If you wish to adopt a GitOps approach to managing this platform, " - "we will walk you through a set of questions to get that setup. With this setup, Nebari will use GitHub Actions workflows (or GitLab equivalent) " - "to automatically handle the future deployments of your infrastructure.\n\n" - ) - ) - if questionary.confirm( - "Would you like to adopt a GitOps approach to managing Nebari?", - default=False, - qmark=qmark, - auto_enter=False, - ).unsafe_ask(): - repo_url = "http://{git_provider}/{org_name}/{repo_name}" - - git_provider = questionary.select( - "Which git provider would you like to use?", - choices=enum_to_list(GitRepoEnum), - qmark=qmark, - ).unsafe_ask() - - org_name = questionary.text( - f"Which user or organization will this repository live under? ({repo_url.format(git_provider=git_provider, org_name='', repo_name='')})", - qmark=qmark, - ).unsafe_ask() - - repo_name = questionary.text( - f"And what will the name of this repository be? ({repo_url.format(git_provider=git_provider, org_name=org_name, repo_name='')})", - qmark=qmark, - ).unsafe_ask() - - inputs.repository = repo_url.format( - git_provider=git_provider, org_name=org_name, repo_name=repo_name - ) - - if git_provider == GitRepoEnum.github.value.lower(): - inputs.repository_auto_provision = questionary.confirm( - f"Would you like nebari to create a remote repository on {git_provider}?", - default=False, - qmark=qmark, - auto_enter=False, - ).unsafe_ask() - - if not disable_checks and inputs.repository_auto_provision: - check_repository_creds(ctx, git_provider) - - if git_provider == GitRepoEnum.github.value.lower(): - inputs.ci_provider = CiEnum.github_actions.value.lower() - elif git_provider == GitRepoEnum.gitlab.value.lower(): - inputs.ci_provider = CiEnum.gitlab_ci.value.lower() - - # SSL CERTIFICATE - rich.print( - ( - "\n\n 🪴 This next section is [italic]optional[/italic] but recommended. If you want your Nebari domain to use a Let's Encrypt SSL certificate, " - "all we need is an email address from you.\n\n" - ) - ) - ssl_cert = questionary.confirm( - "Would you like to add a Let's Encrypt SSL certificate to your domain?", - default=False, - qmark=qmark, - auto_enter=False, - ).unsafe_ask() - - if ssl_cert: - inputs.ssl_cert_email = questionary.text( - "Which email address should Let's Encrypt associate the certificate with?", - qmark=qmark, - ).unsafe_ask() - - if not disable_checks: - check_ssl_cert_email(ctx, ssl_cert_email=inputs.ssl_cert_email) - - # ADVANCED FEATURES - rich.print( - ( - # TODO once docs are updated, add links for more info on these changes - "\n\n 🪴 This next section is [italic]optional[/italic] and includes advanced configuration changes to the " - "Terraform state, Kubernetes Namespace and Kubernetes version." - "\n ⚠️ caution is advised!\n\n" - ) - ) - if questionary.confirm( - "Would you like to make advanced configuration changes?", - default=False, - qmark=qmark, - auto_enter=False, - ).unsafe_ask(): - # TERRAFORM STATE - inputs.terraform_state = questionary.select( - "Where should the Terraform State be provisioned?", - choices=enum_to_list(TerraformStateEnum), - qmark=qmark, - ).unsafe_ask() - - # NAMESPACE - inputs.namespace = questionary.text( - "What would you like the main Kubernetes namespace to be called?", - default=inputs.namespace, - qmark=qmark, - ).unsafe_ask() - - # KUBERNETES VERSION - inputs.kubernetes_version = questionary.text( - "Which Kubernetes version would you like to use (if none provided; latest version will be installed)?", - qmark=qmark, - ).unsafe_ask() - - handle_init(inputs) - - rich.print( - ( - "\n\n\t:sparkles: [bold]Congratulations[/bold], you have generated the all important [purple]nebari-config.yaml[/purple] file :sparkles:\n\n" - "You can always make changes to your [purple]nebari-config.yaml[/purple] file by editing the file directly.\n" - "If you do make changes to it you can ensure it's still a valid configuration by running:\n\n" - "\t[green]nebari validate --config path/to/nebari-config.yaml[/green]\n\n" - ) - ) - - base_cmd = f"nebari init {inputs.cloud_provider}" - - def if_used(key, model=inputs, ignore_list=["cloud_provider"]): - if key not in ignore_list: - b = "--{key} {value}" - value = getattr(model, key) - if isinstance(value, str) and (value != "" or value is not None): - return b.format(key=key, value=value).replace("_", "-") - if isinstance(value, bool) and value: - return b.format(key=key, value=value).replace("_", "-") - - cmds = " ".join( - [_ for _ in [if_used(_) for _ in inputs.dict().keys()] if _ is not None] - ) - - rich.print( - ( - "For reference, if the previous Guided Init answers were converted into a direct [green]nebari init[/green] command, it would be:\n\n" - f"\t[green]{base_cmd} {cmds}[/green]\n\n" - ) - ) - - rich.print( - ( - "You can now deploy your Nebari instance with:\n\n" - "\t[green]nebari deploy -c nebari-config.yaml[/green]\n\n" - "For more information, run [green]nebari deploy --help[/green] or check out the documentation: " - "[green]https://www.nebari.dev/docs/how-tos/[/green]" - ) - ) - - except KeyboardInterrupt: - rich.print("\nUser quit the Guided Init.\n\n ") - raise typer.Exit() - - raise typer.Exit() diff --git a/src/_nebari/cli/main.py b/src/_nebari/cli/main.py deleted file mode 100644 index f00045f9f..000000000 --- a/src/_nebari/cli/main.py +++ /dev/null @@ -1,442 +0,0 @@ -from pathlib import Path -from typing import Optional -from zipfile import ZipFile - -import typer -from click import Context -from kubernetes import client -from kubernetes import config as kube_config -from rich import print -from ruamel import yaml -from typer.core import TyperGroup - -from _nebari.cli.init import ( - check_auth_provider_creds, - check_cloud_provider_creds, - check_project_name, - check_ssl_cert_email, - enum_to_list, - guided_init_wizard, - handle_init, -) -from _nebari.cli.keycloak import app_keycloak -from _nebari.deploy import deploy_configuration -from _nebari.destroy import destroy_configuration -from _nebari.render import render_template -from _nebari.subcommands import app_dev -from _nebari.upgrade import do_upgrade -from _nebari.utils import load_yaml -from _nebari.version import __version__ -from nebari.schema import ( - AuthenticationEnum, - CiEnum, - GitRepoEnum, - InitInputs, - ProviderEnum, - TerraformStateEnum, - verify, -) - -SECOND_COMMAND_GROUP_NAME = "Additional Commands" -GUIDED_INIT_MSG = ( - "[bold green]START HERE[/bold green] - this will guide you step-by-step " - "to generate your [purple]nebari-config.yaml[/purple]. " - "It is an [i]alternative[/i] to passing the options listed below." -) -KEYCLOAK_COMMAND_MSG = ( - "Interact with the Nebari Keycloak identity and access management tool." -) -DEV_COMMAND_MSG = "Development tools and advanced features." - - -def path_callback(value: str) -> Path: - return Path(value).expanduser().resolve() - - -def config_path_callback(value: str) -> Path: - value = path_callback(value) - if not value.is_file(): - raise ValueError(f"Passed configuration path {value} does not exist!") - return value - - -CONFIG_PATH_OPTION: Path = typer.Option( - ..., - "--config", - "-c", - help="nebari configuration yaml file path, please pass in as -c/--config flag", - callback=config_path_callback, -) - -OUTPUT_PATH_OPTION: Path = typer.Option( - Path.cwd(), - "-o", - "--output", - help="output directory", - callback=path_callback, -) - - -class OrderCommands(TyperGroup): - def list_commands(self, ctx: Context): - """Return list of commands in the order appear.""" - return list(self.commands) - - -app = typer.Typer( - cls=OrderCommands, - help="Nebari CLI 🪴", - add_completion=False, - no_args_is_help=True, - rich_markup_mode="rich", - context_settings={"help_option_names": ["-h", "--help"]}, -) -app.add_typer( - app_keycloak, - name="keycloak", - help=KEYCLOAK_COMMAND_MSG, - rich_help_panel=SECOND_COMMAND_GROUP_NAME, -) -app.add_typer( - app_dev, - name="dev", - help=DEV_COMMAND_MSG, - rich_help_panel=SECOND_COMMAND_GROUP_NAME, -) - - -@app.callback(invoke_without_command=True) -def version( - version: Optional[bool] = typer.Option( - None, - "-V", - "--version", - help="Nebari version number", - is_eager=True, - ), -): - if version: - print(__version__) - raise typer.Exit() - - -@app.command() -def init( - cloud_provider: str = typer.Argument( - "local", - help=f"options: {enum_to_list(ProviderEnum)}", - callback=check_cloud_provider_creds, - is_eager=True, - ), - # Although this unused below, the functionality is contained in the callback. Thus, - # this attribute cannot be removed. - guided_init: bool = typer.Option( - False, - help=GUIDED_INIT_MSG, - callback=guided_init_wizard, - is_eager=True, - ), - project_name: str = typer.Option( - ..., - "--project-name", - "--project", - "-p", - callback=check_project_name, - ), - domain_name: str = typer.Option( - ..., - "--domain-name", - "--domain", - "-d", - ), - namespace: str = typer.Option( - "dev", - ), - auth_provider: str = typer.Option( - "password", - help=f"options: {enum_to_list(AuthenticationEnum)}", - callback=check_auth_provider_creds, - ), - auth_auto_provision: bool = typer.Option( - False, - ), - repository: str = typer.Option( - None, - help=f"options: {enum_to_list(GitRepoEnum)}", - ), - repository_auto_provision: bool = typer.Option( - False, - ), - ci_provider: str = typer.Option( - None, - help=f"options: {enum_to_list(CiEnum)}", - ), - terraform_state: str = typer.Option( - "remote", help=f"options: {enum_to_list(TerraformStateEnum)}" - ), - kubernetes_version: str = typer.Option( - "latest", - ), - ssl_cert_email: str = typer.Option( - None, - callback=check_ssl_cert_email, - ), - disable_prompt: bool = typer.Option( - False, - is_eager=True, - ), -): - """ - Create and initialize your [purple]nebari-config.yaml[/purple] file. - - This command will create and initialize your [purple]nebari-config.yaml[/purple] :sparkles: - - This file contains all your Nebari cluster configuration details and, - is used as input to later commands such as [green]nebari render[/green], [green]nebari deploy[/green], etc. - - If you're new to Nebari, we recommend you use the Guided Init wizard. - To get started simply run: - - [green]nebari init --guided-init[/green] - - """ - inputs = InitInputs() - - inputs.cloud_provider = cloud_provider - inputs.project_name = project_name - inputs.domain_name = domain_name - inputs.namespace = namespace - inputs.auth_provider = auth_provider - inputs.auth_auto_provision = auth_auto_provision - inputs.repository = repository - inputs.repository_auto_provision = repository_auto_provision - inputs.ci_provider = ci_provider - inputs.terraform_state = terraform_state - inputs.kubernetes_version = kubernetes_version - inputs.ssl_cert_email = ssl_cert_email - inputs.disable_prompt = disable_prompt - - handle_init(inputs) - - -@app.command(rich_help_panel=SECOND_COMMAND_GROUP_NAME) -def validate( - config_path=CONFIG_PATH_OPTION, - enable_commenting: bool = typer.Option( - False, "--enable-commenting", help="Toggle PR commenting on GitHub Actions" - ), -): - """ - Validate the values in the [purple]nebari-config.yaml[/purple] file are acceptable. - """ - config = load_yaml(config_path) - - if enable_commenting: - # for PR's only - # comment_on_pr(config) - pass - else: - verify(config) - print("[bold purple]Successfully validated configuration.[/bold purple]") - - -@app.command(rich_help_panel=SECOND_COMMAND_GROUP_NAME) -def render( - output_path=OUTPUT_PATH_OPTION, - config_path=CONFIG_PATH_OPTION, - dry_run: bool = typer.Option( - False, - "--dry-run", - help="simulate rendering files without actually writing or updating any files", - ), -): - """ - Dynamically render the Terraform scripts and other files from your [purple]nebari-config.yaml[/purple] file. - """ - config = load_yaml(config_path) - - verify(config) - - render_template(output_path, config_path, dry_run=dry_run) - - -@app.command() -def deploy( - config_path=CONFIG_PATH_OPTION, - output_path=OUTPUT_PATH_OPTION, - dns_provider: str = typer.Option( - False, - "--dns-provider", - help="dns provider to use for registering domain name mapping", - ), - dns_auto_provision: bool = typer.Option( - False, - "--dns-auto-provision", - help="Attempt to automatically provision DNS, currently only available for `cloudflare`", - ), - disable_prompt: bool = typer.Option( - False, - "--disable-prompt", - help="Disable human intervention", - ), - disable_render: bool = typer.Option( - False, - "--disable-render", - help="Disable auto-rendering in deploy stage", - ), - disable_checks: bool = typer.Option( - False, - "--disable-checks", - help="Disable the checks performed after each stage", - ), - skip_remote_state_provision: bool = typer.Option( - False, - "--skip-remote-state-provision", - help="Skip terraform state deployment which is often required in CI once the terraform remote state bootstrapping phase is complete", - ), -): - """ - Deploy the Nebari cluster from your [purple]nebari-config.yaml[/purple] file. - """ - config = load_yaml(config_path) - - verify(config) - - if not disable_render: - render_template(output_path, config_path) - - deploy_configuration( - config, - dns_provider=dns_provider, - dns_auto_provision=dns_auto_provision, - disable_prompt=disable_prompt, - disable_checks=disable_checks, - skip_remote_state_provision=skip_remote_state_provision, - ) - - -@app.command() -def destroy( - config_path=CONFIG_PATH_OPTION, - output_path=OUTPUT_PATH_OPTION, - disable_render: bool = typer.Option( - False, - "--disable-render", - help="Disable auto-rendering before destroy", - ), - disable_prompt: bool = typer.Option( - False, - "--disable-prompt", - help="Destroy entire Nebari cluster without confirmation request. Suggested for CI use.", - ), -): - """ - Destroy the Nebari cluster from your [purple]nebari-config.yaml[/purple] file. - """ - - def _run_destroy(config_path=config_path, disable_render=disable_render): - config = load_yaml(config_path) - - verify(config) - - if not disable_render: - render_template(output_path, config_path) - - destroy_configuration(config) - - if disable_prompt: - _run_destroy() - elif typer.confirm("Are you sure you want to destroy your Nebari cluster?"): - _run_destroy() - else: - raise typer.Abort() - - -@app.command(rich_help_panel=SECOND_COMMAND_GROUP_NAME) -def upgrade( - config_path=CONFIG_PATH_OPTION, - attempt_fixes: bool = typer.Option( - False, - "--attempt-fixes", - help="Attempt to fix the config for any incompatibilities between your old and new Nebari versions.", - ), -): - """ - Upgrade your [purple]nebari-config.yaml[/purple]. - - Upgrade your [purple]nebari-config.yaml[/purple] after an nebari upgrade. If necessary, prompts users to perform manual upgrade steps required for the deploy process. - - See the project [green]RELEASE.md[/green] for details. - """ - do_upgrade(config_path, attempt_fixes=attempt_fixes) - - -@app.command(rich_help_panel=SECOND_COMMAND_GROUP_NAME) -def support( - config_path=CONFIG_PATH_OPTION, - output_path=OUTPUT_PATH_OPTION, -): - """ - Support tool to write all Kubernetes logs locally and compress them into a zip file. - - The Nebari team recommends k9s to manage and inspect the state of the cluster. - However, this command occasionally helpful for debugging purposes should the logs need to be shared. - """ - kube_config.load_kube_config() - - v1 = client.CoreV1Api() - - namespace = get_config_namespace(config_path) - - pods = v1.list_namespaced_pod(namespace=namespace) - - for pod in pods.items: - Path(f"./log/{namespace}").mkdir(parents=True, exist_ok=True) - path = Path(f"./log/{namespace}/{pod.metadata.name}.txt") - with path.open(mode="wt") as file: - try: - file.write( - "%s\t%s\t%s\n" - % ( - pod.status.pod_ip, - namespace, - pod.metadata.name, - ) - ) - - # some pods are running multiple containers - containers = [ - _.name if len(pod.spec.containers) > 1 else None - for _ in pod.spec.containers - ] - - for container in containers: - if container is not None: - file.write(f"Container: {container}\n") - file.write( - v1.read_namespaced_pod_log( - name=pod.metadata.name, - namespace=namespace, - container=container, - ) - ) - - except client.exceptions.ApiException as e: - file.write("%s not available" % pod.metadata.name) - raise e - - with ZipFile(output_path, "w") as zip: - for file in list(Path(f"./log/{namespace}").glob("*.txt")): - print(file) - zip.write(file) - - -def get_config_namespace(config_path): - with open(config_path) as f: - config = yaml.safe_load(f.read()) - - return config["namespace"] - - -if __name__ == "__main__": - app() diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index 32dc9d318..e864636b5 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -100,7 +100,7 @@ def render_config( config.theme.jupyterhub.hub_subtitle = WELCOME_HEADER_TEXT if ssl_cert_email: - config.certificate.type = CertificateEnum.letsencrypt + config.certificate.type = schema.CertificateEnum.letsencrypt config.certificate.acme_email = ssl_cert_email if repository_auto_provision: diff --git a/src/_nebari/subcommands/__init__.py b/src/_nebari/subcommands/__init__.py index 15df491e0..e69de29bb 100644 --- a/src/_nebari/subcommands/__init__.py +++ b/src/_nebari/subcommands/__init__.py @@ -1,41 +0,0 @@ -from typing import Optional - -import typer -from typer.core import TyperGroup - -from _nebari.version import __version__ -from nebari.plugins import pm - - -class OrderCommands(TyperGroup): - def list_commands(self, ctx: typer.Context): - """Return list of commands in the order appear.""" - return list(self.commands) - - -def create_cli(): - app = typer.Typer( - cls=OrderCommands, - help="Nebari CLI 🪴", - add_completion=False, - no_args_is_help=True, - rich_markup_mode="rich", - context_settings={"help_option_names": ["-h", "--help"]}, - ) - pm.hook.nebari_subcommand(cli=app) - - @app.callback(invoke_without_command=True) - def version( - version: Optional[bool] = typer.Option( - None, - "-V", - "--version", - help="Nebari version number", - is_eager=True, - ), - ): - if version: - print(__version__) - raise typer.Exit() - - return app diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index ccc12a7c3..c51098bf0 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -1,16 +1,18 @@ +import rich + +import re import os +import json +import pathlib +from typing import Tuple -import rich +import questionary import typer -from _nebari.cli.init import ( - check_project_name, - check_ssl_cert_email, - guided_init_wizard, - handle_init, -) -from nebari import schema +from _nebari.keycloak import do_keycloak, export_keycloak_users +from _nebari.initialize import render_config from nebari.hookspecs import hookimpl +from nebari import schema MISSING_CREDS_TEMPLATE = "Unable to locate your {provider} credentials, refer to this guide on how to generate them:\n\n[green]\t{link_to_docs}[/green]\n\n" LINKS_TO_DOCS_TEMPLATE = ( @@ -43,7 +45,66 @@ def enum_to_list(enum_cls): - return ", ".join([e.value.lower() for e in enum_cls]) + return [e.value for e in enum_cls] + + +def handle_init(inputs: schema.InitInputs): + """ + Take the inputs from the `nebari init` command, render the config and write it to a local yaml file. + """ + # if NEBARI_IMAGE_TAG: + # print( + # f"Modifying the image tags for the `default_images`, setting tags to: {NEBARI_IMAGE_TAG}" + # ) + + # if NEBARI_DASK_VERSION: + # print( + # f"Modifying the version of the `nebari_dask` package, setting version to: {NEBARI_DASK_VERSION}" + # ) + + # this will force the `set_kubernetes_version` to grab the latest version + if inputs.kubernetes_version == "latest": + inputs.kubernetes_version = None + + config = render_config( + cloud_provider=inputs.cloud_provider, + project_name=inputs.project_name, + nebari_domain=inputs.domain_name, + namespace=inputs.namespace, + auth_provider=inputs.auth_provider, + auth_auto_provision=inputs.auth_auto_provision, + ci_provider=inputs.ci_provider, + repository=inputs.repository, + repository_auto_provision=inputs.repository_auto_provision, + kubernetes_version=inputs.kubernetes_version, + terraform_state=inputs.terraform_state, + ssl_cert_email=inputs.ssl_cert_email, + disable_prompt=inputs.disable_prompt, + ) + + try: + schema.write_configuration(pathlib.Path("nebari-config.yaml"), config, mode='x') + except FileExistsError: + raise ValueError( + "A nebari-config.yaml file already exists. Please move or delete it and try again." + ) + + +def check_project_name(ctx: typer.Context, project_name: str): + """Validate the project_name is acceptable. Depends on `cloud_provider`.""" + schema.project_name_convention( + project_name.lower(), {"provider": ctx.params["cloud_provider"]} + ) + + return project_name + + +def check_ssl_cert_email(ctx: typer.Context, ssl_cert_email: str): + """Validate the email used for SSL cert is in a valid format.""" + if ssl_cert_email and not re.match("^[^ @]+@[^ @]+\\.[^ @]+$", ssl_cert_email): + raise ValueError("ssl-cert-email should be a valid email address") + + return ssl_cert_email def check_auth_provider_creds(ctx: typer.Context, auth_provider: str): @@ -306,3 +367,291 @@ def init( inputs.disable_prompt = disable_prompt handle_init(inputs) + + +def guided_init_wizard(ctx: typer.Context, guided_init: str): + """ + Guided Init Wizard is a user-friendly questionnaire used to help generate the `nebari-config.yaml`. + """ + qmark = " " + disable_checks = os.environ.get("NEBARI_DISABLE_INIT_CHECKS", False) + + if pathlib.Path("nebari-config.yaml").exists(): + raise ValueError( + "A nebari-config.yaml file already exists. Please move or delete it and try again." + ) + + if not guided_init: + return guided_init + + try: + rich.print( + ( + "\n\t[bold]Welcome to the Guided Init wizard![/bold]\n\n" + "You will be asked a few questions to generate your [purple]nebari-config.yaml[/purple]. " + f"{LINKS_TO_DOCS_TEMPLATE.format(link_to_docs=DOCS_HOME)}" + ) + ) + + if disable_checks: + rich.print( + "⚠️ Attempting to use the Guided Init wizard without any validation checks. There is no guarantee values provided will work! ⚠️\n\n" + ) + + # pull in default values for each of the below + inputs = schema.InitInputs() + + # CLOUD PROVIDER + rich.print( + ( + "\n 🪴 Nebari runs on a Kubernetes cluster: Where do you want this Kubernetes cluster deployed? " + "is where you want this Kubernetes cluster deployed. " + f"{LINKS_TO_DOCS_TEMPLATE.format(link_to_docs=CHOOSE_CLOUD_PROVIDER)}" + "\n\t❗️ [purple]local[/purple] requires Docker and Kubernetes running on your local machine. " + "[italic]Currently only available on Linux OS.[/italic]" + "\n\t❗️ [purple]existing[/purple] refers to an existing Kubernetes cluster that Nebari can be deployed on.\n" + ) + ) + # try: + inputs.cloud_provider = questionary.select( + "Where would you like to deploy your Nebari cluster?", + choices=enum_to_list(schema.ProviderEnum), + qmark=qmark, + ).unsafe_ask() + + if not disable_checks: + check_cloud_provider_creds(ctx, cloud_provider=inputs.cloud_provider) + + # specific context needed when `check_project_name` is called + ctx.params["cloud_provider"] = inputs.cloud_provider + + name_guidelines = """ + The project name must adhere to the following requirements: + - Letters from A to Z (upper and lower case) and numbers + - Maximum accepted length of the name string is 16 characters + """ + if inputs.cloud_provider == schema.ProviderEnum.aws.value.lower(): + name_guidelines += "- Should NOT start with the string `aws`\n" + elif inputs.cloud_provider == schema.ProviderEnum.azure.value.lower(): + name_guidelines += "- Should NOT contain `-`\n" + + # PROJECT NAME + rich.print( + ( + f"\n 🪴 Next, give your Nebari instance a project name. This name is what your Kubernetes cluster will be referred to as.\n{name_guidelines}\n" + ) + ) + inputs.project_name = questionary.text( + "What project name would you like to use?", + qmark=qmark, + validate=lambda text: True if len(text) > 0 else "Please enter a value", + ).unsafe_ask() + + if not disable_checks: + check_project_name(ctx, inputs.project_name) + + # DOMAIN NAME + rich.print( + ( + "\n\n 🪴 Great! Now you need to provide a valid domain name (i.e. the URL) to access your Nebri instance. " + "This should be a domain that you own.\n\n" + ) + ) + inputs.domain_name = questionary.text( + "What domain name would you like to use?", + qmark=qmark, + validate=lambda text: True if len(text) > 0 else "Please enter a value", + ).unsafe_ask() + + # AUTH PROVIDER + rich.print( + ( + # TODO once docs are updated, add links for more details + "\n\n 🪴 Nebari comes with [green]Keycloak[/green], an open-source identity and access management tool. This is how users and permissions " + "are managed on the platform. To connect Keycloak with an identity provider, you can select one now.\n\n" + "\n\t❗️ [purple]password[/purple] is the default option and is not connected to any external identity provider.\n" + ) + ) + inputs.auth_provider = questionary.select( + "What authentication provider would you like?", + choices=enum_to_list(schema.AuthenticationEnum), + qmark=qmark, + ).unsafe_ask() + + if not disable_checks: + check_auth_provider_creds(ctx, auth_provider=inputs.auth_provider) + + if inputs.auth_provider.lower() == schema.AuthenticationEnum.auth0.value.lower(): + inputs.auth_auto_provision = questionary.confirm( + "Would you like us to auto provision the Auth0 Machine-to-Machine app?", + default=False, + qmark=qmark, + auto_enter=False, + ).unsafe_ask() + + elif inputs.auth_provider.lower() == schema.AuthenticationEnum.github.value.lower(): + rich.print( + ( + ":warning: If you haven't done so already, please ensure the following:\n" + f"The `Homepage URL` is set to: [green]https://{inputs.domain_name}[/green]\n" + f"The `Authorization callback URL` is set to: [green]https://{inputs.domain_name}/auth/realms/nebari/broker/github/endpoint[/green]\n\n" + ) + ) + + # GITOPS - REPOSITORY, CICD + rich.print( + ( + "\n\n 🪴 This next section is [italic]optional[/italic] but recommended. If you wish to adopt a GitOps approach to managing this platform, " + "we will walk you through a set of questions to get that setup. With this setup, Nebari will use GitHub Actions workflows (or GitLab equivalent) " + "to automatically handle the future deployments of your infrastructure.\n\n" + ) + ) + if questionary.confirm( + "Would you like to adopt a GitOps approach to managing Nebari?", + default=False, + qmark=qmark, + auto_enter=False, + ).unsafe_ask(): + repo_url = "http://{git_provider}/{org_name}/{repo_name}" + + git_provider = questionary.select( + "Which git provider would you like to use?", + choices=enum_to_list(GitRepoEnum), + qmark=qmark, + ).unsafe_ask() + + org_name = questionary.text( + f"Which user or organization will this repository live under? ({repo_url.format(git_provider=git_provider, org_name='', repo_name='')})", + qmark=qmark, + ).unsafe_ask() + + repo_name = questionary.text( + f"And what will the name of this repository be? ({repo_url.format(git_provider=git_provider, org_name=org_name, repo_name='')})", + qmark=qmark, + ).unsafe_ask() + + inputs.repository = repo_url.format( + git_provider=git_provider, org_name=org_name, repo_name=repo_name + ) + + if git_provider == GitRepoEnum.github.value.lower(): + inputs.repository_auto_provision = questionary.confirm( + f"Would you like nebari to create a remote repository on {git_provider}?", + default=False, + qmark=qmark, + auto_enter=False, + ).unsafe_ask() + + if not disable_checks and inputs.repository_auto_provision: + check_repository_creds(ctx, git_provider) + + if git_provider == GitRepoEnum.github.value.lower(): + inputs.ci_provider = CiEnum.github_actions.value.lower() + elif git_provider == GitRepoEnum.gitlab.value.lower(): + inputs.ci_provider = CiEnum.gitlab_ci.value.lower() + + # SSL CERTIFICATE + rich.print( + ( + "\n\n 🪴 This next section is [italic]optional[/italic] but recommended. If you want your Nebari domain to use a Let's Encrypt SSL certificate, " + "all we need is an email address from you.\n\n" + ) + ) + ssl_cert = questionary.confirm( + "Would you like to add a Let's Encrypt SSL certificate to your domain?", + default=False, + qmark=qmark, + auto_enter=False, + ).unsafe_ask() + + if ssl_cert: + inputs.ssl_cert_email = questionary.text( + "Which email address should Let's Encrypt associate the certificate with?", + qmark=qmark, + ).unsafe_ask() + + if not disable_checks: + check_ssl_cert_email(ctx, ssl_cert_email=inputs.ssl_cert_email) + + # ADVANCED FEATURES + rich.print( + ( + # TODO once docs are updated, add links for more info on these changes + "\n\n 🪴 This next section is [italic]optional[/italic] and includes advanced configuration changes to the " + "Terraform state, Kubernetes Namespace and Kubernetes version." + "\n ⚠️ caution is advised!\n\n" + ) + ) + if questionary.confirm( + "Would you like to make advanced configuration changes?", + default=False, + qmark=qmark, + auto_enter=False, + ).unsafe_ask(): + # TERRAFORM STATE + inputs.terraform_state = questionary.select( + "Where should the Terraform State be provisioned?", + choices=enum_to_list(TerraformStateEnum), + qmark=qmark, + ).unsafe_ask() + + # NAMESPACE + inputs.namespace = questionary.text( + "What would you like the main Kubernetes namespace to be called?", + default=inputs.namespace, + qmark=qmark, + ).unsafe_ask() + + # KUBERNETES VERSION + inputs.kubernetes_version = questionary.text( + "Which Kubernetes version would you like to use (if none provided; latest version will be installed)?", + qmark=qmark, + ).unsafe_ask() + + handle_init(inputs) + + rich.print( + ( + "\n\n\t:sparkles: [bold]Congratulations[/bold], you have generated the all important [purple]nebari-config.yaml[/purple] file :sparkles:\n\n" + "You can always make changes to your [purple]nebari-config.yaml[/purple] file by editing the file directly.\n" + "If you do make changes to it you can ensure it's still a valid configuration by running:\n\n" + "\t[green]nebari validate --config path/to/nebari-config.yaml[/green]\n\n" + ) + ) + + base_cmd = f"nebari init {inputs.cloud_provider}" + + def if_used(key, model=inputs, ignore_list=["cloud_provider"]): + if key not in ignore_list: + b = "--{key} {value}" + value = getattr(model, key) + if isinstance(value, str) and (value != "" or value is not None): + return b.format(key=key, value=value).replace("_", "-") + if isinstance(value, bool) and value: + return b.format(key=key, value=value).replace("_", "-") + + cmds = " ".join( + [_ for _ in [if_used(_) for _ in inputs.dict().keys()] if _ is not None] + ) + + rich.print( + ( + "For reference, if the previous Guided Init answers were converted into a direct [green]nebari init[/green] command, it would be:\n\n" + f"\t[green]{base_cmd} {cmds}[/green]\n\n" + ) + ) + + rich.print( + ( + "You can now deploy your Nebari instance with:\n\n" + "\t[green]nebari deploy -c nebari-config.yaml[/green]\n\n" + "For more information, run [green]nebari deploy --help[/green] or check out the documentation: " + "[green]https://www.nebari.dev/docs/how-tos/[/green]" + ) + ) + + except KeyboardInterrupt: + rich.print("\nUser quit the Guided Init.\n\n ") + raise typer.Exit() + + raise typer.Exit() diff --git a/src/nebari/__main__.py b/src/nebari/__main__.py index 29d39600d..b18eaf428 100644 --- a/src/nebari/__main__.py +++ b/src/nebari/__main__.py @@ -1,4 +1,10 @@ -from _nebari.__main__ import main +from _nebari.cli import create_cli + + +def main(): + cli = create_cli() + cli() + if __name__ == "__main__": main() diff --git a/src/nebari/schema.py b/src/nebari/schema.py index edace6932..a5f9d0745 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -429,11 +429,11 @@ def _validate_kubernetes_version(cls, value): class GoogleCloudPlatformProvider(Base): - project: str + project: str = Field(default_factory=lambda: os.environ['PROJECT_ID']) region: str = "us-central1" availability_zones: typing.Optional[typing.List[str]] = [] kubernetes_version: str = Field( - default_factory=lambda: google_cloud.kubernetes_versions()[-1] + default_factory=lambda: google_cloud.kubernetes_versions("us-central1")[-1] ) release_channel: typing.Optional[str] @@ -458,7 +458,7 @@ class GoogleCloudPlatformProvider(Base): @validator("kubernetes_version") def _validate_kubernetes_version(cls, value): - available_kubernetes_versions = google_cloud.kubernetes_versions() + available_kubernetes_versions = google_cloud.kubernetes_versions("us-central1") if value not in available_kubernetes_versions: raise ValueError( f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." @@ -1093,4 +1093,4 @@ def write_configuration(config_filename: pathlib.Path, config: Main, mode: str = yaml.default_flow_style = False with config_filename.open(mode) as f: - yaml.dump(config.dict(exclude_unset=True), f) + yaml.dump(config.dict(), f) From ced44d02ae7903d83eda4d7b3c00e7ba2686f5e7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Jun 2023 20:24:36 +0000 Subject: [PATCH 031/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/subcommands/init.py | 22 ++++++++++++---------- src/nebari/schema.py | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index c51098bf0..a6d3f1435 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -1,18 +1,14 @@ -import rich - -import re import os -import json import pathlib -from typing import Tuple +import re import questionary +import rich import typer -from _nebari.keycloak import do_keycloak, export_keycloak_users from _nebari.initialize import render_config -from nebari.hookspecs import hookimpl from nebari import schema +from nebari.hookspecs import hookimpl MISSING_CREDS_TEMPLATE = "Unable to locate your {provider} credentials, refer to this guide on how to generate them:\n\n[green]\t{link_to_docs}[/green]\n\n" LINKS_TO_DOCS_TEMPLATE = ( @@ -83,7 +79,7 @@ def handle_init(inputs: schema.InitInputs): ) try: - schema.write_configuration(pathlib.Path("nebari-config.yaml"), config, mode='x') + schema.write_configuration(pathlib.Path("nebari-config.yaml"), config, mode="x") except FileExistsError: raise ValueError( "A nebari-config.yaml file already exists. Please move or delete it and try again." @@ -481,7 +477,10 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): if not disable_checks: check_auth_provider_creds(ctx, auth_provider=inputs.auth_provider) - if inputs.auth_provider.lower() == schema.AuthenticationEnum.auth0.value.lower(): + if ( + inputs.auth_provider.lower() + == schema.AuthenticationEnum.auth0.value.lower() + ): inputs.auth_auto_provision = questionary.confirm( "Would you like us to auto provision the Auth0 Machine-to-Machine app?", default=False, @@ -489,7 +488,10 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): auto_enter=False, ).unsafe_ask() - elif inputs.auth_provider.lower() == schema.AuthenticationEnum.github.value.lower(): + elif ( + inputs.auth_provider.lower() + == schema.AuthenticationEnum.github.value.lower() + ): rich.print( ( ":warning: If you haven't done so already, please ensure the following:\n" diff --git a/src/nebari/schema.py b/src/nebari/schema.py index a5f9d0745..2503ef42a 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -429,7 +429,7 @@ def _validate_kubernetes_version(cls, value): class GoogleCloudPlatformProvider(Base): - project: str = Field(default_factory=lambda: os.environ['PROJECT_ID']) + project: str = Field(default_factory=lambda: os.environ["PROJECT_ID"]) region: str = "us-central1" availability_zones: typing.Optional[typing.List[str]] = [] kubernetes_version: str = Field( From 95cde1e427579a361a5362cb3620edc6076d0868 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 16 Jun 2023 16:28:49 -0400 Subject: [PATCH 032/147] Ensure that plugy extensions can be loaded via entrypoints --- src/nebari/plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index ce2b772ad..37e751774 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -33,7 +33,7 @@ if not hasattr(sys, "_called_from_test"): # Only load plugins if not running tests - pm.load_setuptools_entrypoints("datasette") + pm.load_setuptools_entrypoints("nebari") # Load default plugins for plugin in DEFAULT_PLUGINS: From ee0e4b8c69f6926b50b39beeff918df2ab76ea2b Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 16 Jun 2023 16:41:06 -0400 Subject: [PATCH 033/147] Adding import module subcommand option --- src/_nebari/cli.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/_nebari/cli.py b/src/_nebari/cli.py index 15df491e0..3b6829556 100644 --- a/src/_nebari/cli.py +++ b/src/_nebari/cli.py @@ -1,4 +1,5 @@ from typing import Optional +import importlib import typer from typer.core import TyperGroup @@ -13,6 +14,16 @@ def list_commands(self, ctx: typer.Context): return list(self.commands) +def version_callback(value: bool): + if value: + typer.echo(__version__) + raise typer.Exit() + + +def import_module(module: str): + importlib.__import__(module) + + def create_cli(): app = typer.Typer( cls=OrderCommands, @@ -22,20 +33,15 @@ def create_cli(): rich_markup_mode="rich", context_settings={"help_option_names": ["-h", "--help"]}, ) - pm.hook.nebari_subcommand(cli=app) @app.callback(invoke_without_command=True) - def version( - version: Optional[bool] = typer.Option( - None, - "-V", - "--version", - help="Nebari version number", - is_eager=True, - ), + def common( + ctx: typer.Context, + version: bool = typer.Option(None, "-V", "--version", help="Nebari version number", callback=version_callback), + import_module: str = typer.Option(None, "--import-module", help="Import nebari module", callback=import_module), ): - if version: - print(__version__) - raise typer.Exit() + pass + + pm.hook.nebari_subcommand(cli=app) return app From e231365c84b72d4c1e77987b1bde9f9d154a0085 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Jun 2023 20:41:23 +0000 Subject: [PATCH 034/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/cli.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/_nebari/cli.py b/src/_nebari/cli.py index 3b6829556..3be5560e0 100644 --- a/src/_nebari/cli.py +++ b/src/_nebari/cli.py @@ -1,4 +1,3 @@ -from typing import Optional import importlib import typer @@ -37,8 +36,16 @@ def create_cli(): @app.callback(invoke_without_command=True) def common( ctx: typer.Context, - version: bool = typer.Option(None, "-V", "--version", help="Nebari version number", callback=version_callback), - import_module: str = typer.Option(None, "--import-module", help="Import nebari module", callback=import_module), + version: bool = typer.Option( + None, + "-V", + "--version", + help="Nebari version number", + callback=version_callback, + ), + import_module: str = typer.Option( + None, "--import-module", help="Import nebari module", callback=import_module + ), ): pass From 468af6b76835b8d63e7d2cd5bdde9691b10c1c56 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Mon, 19 Jun 2023 09:23:37 -0400 Subject: [PATCH 035/147] More fixes around AWS deployment --- src/_nebari/cli.py | 6 ++- src/_nebari/stages/infrastructure/__init__.py | 37 +++++++++++++++++-- .../stages/kubernetes_ingress/__init__.py | 1 + src/nebari/hookspecs.py | 5 +-- src/nebari/schema.py | 34 +++++++++++------ 5 files changed, 63 insertions(+), 20 deletions(-) diff --git a/src/_nebari/cli.py b/src/_nebari/cli.py index 3be5560e0..d93a6988a 100644 --- a/src/_nebari/cli.py +++ b/src/_nebari/cli.py @@ -1,4 +1,5 @@ import importlib +import typing import typer from typer.core import TyperGroup @@ -20,7 +21,8 @@ def version_callback(value: bool): def import_module(module: str): - importlib.__import__(module) + if module is not None: + importlib.__import__(module) def create_cli(): @@ -43,7 +45,7 @@ def common( help="Nebari version number", callback=version_callback, ), - import_module: str = typer.Option( + import_module: typing.Optional[str] = typer.Option( None, "--import-module", help="Import nebari module", callback=import_module ), ): diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 1cecb4012..19c8eb7cf 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -52,8 +52,27 @@ class AzureInputVars(BaseCloudProviderInputVars, schema.AzureProvider): node_resource_group_name: str -class AWSInputVars(BaseCloudProviderInputVars, schema.AmazonWebServicesProvider): - pass +class AWSNodeGroupInputVars(schema.Base): + name: str + instance_type: str + gpu: bool = False + min_size: int + desired_size: int + max_size: int + single_subnet: bool + + +class AWSInputVars(schema.Base): + name: str + environment: str + existing_security_group_id: str = None + existing_subnet_ids: List[str] = None + region: str + kubernetes_version: str + node_groups: List[AWSNodeGroupInputVars] + availability_zones: List[str] + vpc_cidr_block: str + kubeconfig_filename: str = get_kubeconfig_filename() @contextlib.contextmanager @@ -183,10 +202,20 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): name=self.config.project_name, environment=self.config.namespace, existing_subnet_ids=self.config.amazon_web_services.existing_subnet_ids, - existing_security_group_ids=self.config.amazon_web_services.existing_security_group_ids, + existing_security_group_id=self.config.amazon_web_services.existing_security_group_ids, region=self.config.amazon_web_services.region, kubernetes_version=self.config.amazon_web_services.kubernetes_version, - node_groups=self.config.amazon_web_services.node_groups, + node_groups=[ + AWSNodeGroupInputVars( + name=name, + instance_type=node_group.instance, + gpu=node_group.gpu, + min_size=node_group.min_nodes, + desired_size=node_group.min_nodes, + max_size=node_group.max_nodes, + single_subnet=node_group.single_subnet, + ) for name, node_group in self.config.amazon_web_services.node_groups.items() + ], availability_zones=self.config.amazon_web_services.availability_zones, vpc_cidr_block=self.config.amazon_web_services.vpc_cidr_block, ).dict() diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index 40b4909b7..eface3561 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -1,3 +1,4 @@ +import time import socket import sys from typing import Any, Dict, List diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index 4f9e688f5..fac86ae87 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -1,7 +1,6 @@ import contextlib import pathlib -from collections.abc import Iterable -from typing import Any, Dict +from typing import Any, Dict, List import typer from pluggy import HookimplMarker, HookspecMarker @@ -41,7 +40,7 @@ def destroy( @hookspec -def nebari_stage() -> Iterable[NebariStage]: +def nebari_stage() -> List[NebariStage]: """Registers stages in nebari""" diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 2503ef42a..631c27dcb 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -349,8 +349,8 @@ class NodeGroup(Base): instance: str min_nodes: int max_nodes: int - gpu: typing.Optional[bool] = False - guest_accelerators: typing.Optional[typing.List[typing.Dict]] = [] + gpu: bool = False + guest_accelerators: typing.List[typing.Dict] = [] class Config: extra = "allow" @@ -379,10 +379,6 @@ def validate_guest_accelerators(cls, v): raise ValueError(assertion_error_message) -class AWSNodeGroup(NodeGroup): - single_subnet: typing.Optional[bool] = False - - class GCPIPAllocationPolicy(Base): cluster_secondary_range_name: str services_secondary_range_name: str @@ -491,6 +487,14 @@ def _validate_kubernetes_version(cls, value): return value +class AWSNodeGroup(Base): + instance: str + min_nodes: int = 0 + max_nodes: int + gpu: bool = False + single_subnet: bool = False + + class AmazonWebServicesProvider(Base): region: str = Field( default_factory=lambda: os.environ.get("AWS_DEFAULT_REGION", "us-west-2") @@ -501,16 +505,16 @@ class AmazonWebServicesProvider(Base): ) node_groups: typing.Dict[str, AWSNodeGroup] = { - "general": NodeGroup(instance="m5.2xlarge", min_nodes=1, max_nodes=1), - "user": NodeGroup( + "general": AWSNodeGroup(instance="m5.2xlarge", min_nodes=1, max_nodes=1), + "user": AWSNodeGroup( instance="m5.xlarge", min_nodes=1, max_nodes=5, single_subnet=False ), - "worker": NodeGroup( + "worker": AWSNodeGroup( instance="m5.xlarge", min_nodes=1, max_nodes=5, single_subnet=False ), } - existing_subnet_ids: typing.Optional[typing.List[str]] - existing_security_group_ids: typing.Optional[str] + existing_subnet_ids: typing.List[str] = None + existing_security_group_ids: str = None vpc_cidr_block: str = "10.10.0.0/16" @validator("kubernetes_version") @@ -522,6 +526,14 @@ def _validate_kubernetes_version(cls, value): ) return value + @root_validator + def _validate_provider(cls, values): + # populate availability zones if empty + if values.get('availability_zones') is None: + zones = amazon_web_services.zones(values['region']) + values['availability_zones'] = list(sorted(zones))[:2] + return values + class LocalProvider(Base): kube_context: typing.Optional[str] From f63cefd9be60582a0c89ea282dd887dded41dfb4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 13:24:03 +0000 Subject: [PATCH 036/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/stages/infrastructure/__init__.py | 3 ++- src/_nebari/stages/kubernetes_ingress/__init__.py | 2 +- src/nebari/schema.py | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 19c8eb7cf..367b492dc 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -214,7 +214,8 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): desired_size=node_group.min_nodes, max_size=node_group.max_nodes, single_subnet=node_group.single_subnet, - ) for name, node_group in self.config.amazon_web_services.node_groups.items() + ) + for name, node_group in self.config.amazon_web_services.node_groups.items() ], availability_zones=self.config.amazon_web_services.availability_zones, vpc_cidr_block=self.config.amazon_web_services.vpc_cidr_block, diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index eface3561..3c5708331 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -1,6 +1,6 @@ -import time import socket import sys +import time from typing import Any, Dict, List from _nebari import constants diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 631c27dcb..f1dfe0f8c 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -529,9 +529,9 @@ def _validate_kubernetes_version(cls, value): @root_validator def _validate_provider(cls, values): # populate availability zones if empty - if values.get('availability_zones') is None: - zones = amazon_web_services.zones(values['region']) - values['availability_zones'] = list(sorted(zones))[:2] + if values.get("availability_zones") is None: + zones = amazon_web_services.zones(values["region"]) + values["availability_zones"] = list(sorted(zones))[:2] return values From be97d69a25ce4ffb1475a1bac1257faa6ba6fabc Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Mon, 19 Jun 2023 20:51:13 -0400 Subject: [PATCH 037/147] Cleaner destroy when running stages. --- src/_nebari/cli.py | 1 + src/_nebari/destroy.py | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/_nebari/cli.py b/src/_nebari/cli.py index d93a6988a..608dd63c5 100644 --- a/src/_nebari/cli.py +++ b/src/_nebari/cli.py @@ -32,6 +32,7 @@ def create_cli(): add_completion=False, no_args_is_help=True, rich_markup_mode="rich", + pretty_exceptions_show_locals=False, context_settings={"help_option_names": ["-h", "--help"]}, ) diff --git a/src/_nebari/destroy.py b/src/_nebari/destroy.py index 1b5d5271a..9f4b604f2 100644 --- a/src/_nebari/destroy.py +++ b/src/_nebari/destroy.py @@ -14,8 +14,13 @@ def destroy_stages(config: schema.Main): status = {} with contextlib.ExitStack() as stack: for stage in get_available_stages(): - s = stage(output_directory=pathlib.Path("."), config=config) - stack.enter_context(s.destroy(stage_outputs, status)) + try: + s = stage(output_directory=pathlib.Path("."), config=config) + stack.enter_context(s.destroy(stage_outputs, status)) + except Exception as e: + stats[s.name] = False + print(f"ERROR: stage={s.name} failed due to {e}. Due to stages depending on each other we can only destroy stages that occur before this stage") + break return status From 9ce35b4d9b4d101097210be09bfd34fe68eb9eeb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 Jun 2023 00:51:34 +0000 Subject: [PATCH 038/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/destroy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/_nebari/destroy.py b/src/_nebari/destroy.py index 9f4b604f2..3e990626b 100644 --- a/src/_nebari/destroy.py +++ b/src/_nebari/destroy.py @@ -19,7 +19,9 @@ def destroy_stages(config: schema.Main): stack.enter_context(s.destroy(stage_outputs, status)) except Exception as e: stats[s.name] = False - print(f"ERROR: stage={s.name} failed due to {e}. Due to stages depending on each other we can only destroy stages that occur before this stage") + print( + f"ERROR: stage={s.name} failed due to {e}. Due to stages depending on each other we can only destroy stages that occur before this stage" + ) break return status From be9c034034ee2c7f8b1719ce8bb0da5e887ed2fb Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Mon, 19 Jun 2023 21:33:02 -0400 Subject: [PATCH 039/147] Missing schema import --- src/_nebari/subcommands/keycloak.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/_nebari/subcommands/keycloak.py b/src/_nebari/subcommands/keycloak.py index b64a6c2b5..34a3b3da9 100644 --- a/src/_nebari/subcommands/keycloak.py +++ b/src/_nebari/subcommands/keycloak.py @@ -6,6 +6,7 @@ from _nebari.keycloak import do_keycloak, export_keycloak_users from nebari.hookspecs import hookimpl +from nebari import schema @hookimpl From 7be54f1f6adbe33385e3940e1be7d9f75ce5a5f9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 Jun 2023 01:33:21 +0000 Subject: [PATCH 040/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/subcommands/keycloak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/subcommands/keycloak.py b/src/_nebari/subcommands/keycloak.py index 34a3b3da9..cab4f36f1 100644 --- a/src/_nebari/subcommands/keycloak.py +++ b/src/_nebari/subcommands/keycloak.py @@ -5,8 +5,8 @@ import typer from _nebari.keycloak import do_keycloak, export_keycloak_users -from nebari.hookspecs import hookimpl from nebari import schema +from nebari.hookspecs import hookimpl @hookimpl From 09c998d455d6ffcd9c6bfd07bbfacc7c45981986 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Mon, 19 Jun 2023 22:57:04 -0400 Subject: [PATCH 041/147] Working gcp deployment --- src/_nebari/stages/infrastructure/__init__.py | 51 +++++++++++++++++-- src/nebari/schema.py | 2 +- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 367b492dc..d3d2a2c1f 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -11,6 +11,7 @@ NebariGCPProvider, NebariTerraformState, ) +from _nebari import constants from _nebari.utils import modified_environ from nebari import schema from nebari.hookspecs import NebariStage, hookimpl @@ -43,8 +44,44 @@ class DigitalOceanInputVars(BaseCloudProviderInputVars, schema.DigitalOceanProvi pass -class GCPInputVars(BaseCloudProviderInputVars, schema.GoogleCloudPlatformProvider): - pass +class GCPGuestAccelerators(schema.Base): + type: str + count: int + + +class GCPNodeGroupInputVars(schema.Base): + name: str + instance_type: str + min_size: int + max_size: int + labels: Dict[str, str] = {} + preemptible: bool = False + guest_accelerators: List[GCPGuestAccelerators] + + +class GCPPrivateClusterConfig(schema.Base): + enable_private_nodes: bool + enable_private_endpoint: bool + master_ipv4_cidr_block: str + + +class GCPInputVars(schema.Base): + name: str + environment: str + region: str + project_id: str + availability_zones: List[str] + node_groups: List[GCPNodeGroupInputVars] + kubeconfig_filename: str = get_kubeconfig_filename() + tags: List[str] + kubernetes_version: str + release_channel: str + networking_mode: str + network: str + subnetwork: str = None + ip_allocation_policy: Dict[str, str] = None + master_authorized_networks_config: Dict[str, str] = None + private_cluster_config: GCPPrivateClusterConfig = None class AzureInputVars(BaseCloudProviderInputVars, schema.AzureProvider): @@ -174,7 +211,15 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): region=self.config.google_cloud_platform.region, project_id=self.config.google_cloud_platform.project, availability_zones=self.config.google_cloud_platform.availability_zones, - node_groups=self.config.google_cloud_platform.node_groups, + node_groups=[ + GCPNodeGroupInputVars( + name=name, + instance_type=node_group.instance, + min_size=node_group.min_nodes, + max_size=node_group.max_nodes, + guest_accelerators=node_group.guest_accelerators, + ) for name, node_group in self.config.google_cloud_platform.node_groups.items() + ], tags=self.config.google_cloud_platform.tags, kubernetes_version=self.config.google_cloud_platform.kubernetes_version, release_channel=self.config.google_cloud_platform.release_channel, diff --git a/src/nebari/schema.py b/src/nebari/schema.py index f1dfe0f8c..a2cc471a0 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -432,7 +432,7 @@ class GoogleCloudPlatformProvider(Base): default_factory=lambda: google_cloud.kubernetes_versions("us-central1")[-1] ) - release_channel: typing.Optional[str] + release_channel: str = constants.DEFAULT_GKE_RELEASE_CHANNEL node_groups: typing.Dict[str, NodeGroup] = { "general": NodeGroup(instance="n1-standard-8", min_nodes=1, max_nodes=1), "user": NodeGroup(instance="n1-standard-4", min_nodes=0, max_nodes=5), From a466e83e6bacdae73c03a2804f5e74bed4997e06 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 Jun 2023 02:57:30 +0000 Subject: [PATCH 042/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/stages/infrastructure/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index d3d2a2c1f..0f43ff228 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -11,7 +11,6 @@ NebariGCPProvider, NebariTerraformState, ) -from _nebari import constants from _nebari.utils import modified_environ from nebari import schema from nebari.hookspecs import NebariStage, hookimpl @@ -218,7 +217,8 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): min_size=node_group.min_nodes, max_size=node_group.max_nodes, guest_accelerators=node_group.guest_accelerators, - ) for name, node_group in self.config.google_cloud_platform.node_groups.items() + ) + for name, node_group in self.config.google_cloud_platform.node_groups.items() ], tags=self.config.google_cloud_platform.tags, kubernetes_version=self.config.google_cloud_platform.kubernetes_version, From fd8fc2298a639fe645c117286b801ae2707311fc Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Tue, 20 Jun 2023 11:54:18 -0400 Subject: [PATCH 043/147] Azure fixes --- src/_nebari/destroy.py | 2 +- src/_nebari/stages/infrastructure/__init__.py | 24 +++++++++++++++++-- src/nebari/schema.py | 2 +- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/_nebari/destroy.py b/src/_nebari/destroy.py index 3e990626b..06c133fa2 100644 --- a/src/_nebari/destroy.py +++ b/src/_nebari/destroy.py @@ -18,7 +18,7 @@ def destroy_stages(config: schema.Main): s = stage(output_directory=pathlib.Path("."), config=config) stack.enter_context(s.destroy(stage_outputs, status)) except Exception as e: - stats[s.name] = False + status[s.name] = False print( f"ERROR: stage={s.name} failed due to {e}. Due to stages depending on each other we can only destroy stages that occur before this stage" ) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 0f43ff228..c1b3a0d9c 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -83,9 +83,23 @@ class GCPInputVars(schema.Base): private_cluster_config: GCPPrivateClusterConfig = None -class AzureInputVars(BaseCloudProviderInputVars, schema.AzureProvider): +class AzureNodeGroupInputVars(schema.Base): + instance: str + min_nodes: int + max_nodes: int + + +class AzureInputVars(schema.Base): + name: str + environment: str + region: str + kubeconfig_filename: str = get_kubeconfig_filename() + kubernetes_version: str + node_groups: Dict[str, AzureNodeGroupInputVars] resource_group_name: str node_resource_group_name: str + vnet_subnet_id: str = None + private_cluster_enabled: bool class AWSNodeGroupInputVars(schema.Base): @@ -236,7 +250,13 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): environment=self.config.namespace, region=self.config.azure.region, kubernetes_version=self.config.azure.kubernetes_version, - node_groups=self.config.azure.node_groups, + node_groups={ + name: AzureNodeGroupInputVars( + instance=node_group.instance, + min_nodes=node_group.min_nodes, + max_nodes=node_group.max_nodes, + ) for name, node_group in self.config.azure.node_groups.items() + }, resource_group_name=f"{self.config.project_name}-{self.config.namespace}", node_resource_group_name=f"{self.config.project_name}-{self.config.namespace}-node-resource-group", vnet_subnet_id=self.config.azure.vnet_subnet_id, diff --git a/src/nebari/schema.py b/src/nebari/schema.py index a2cc471a0..906c4b521 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -473,7 +473,7 @@ class AzureProvider(Base): "user": NodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), "worker": NodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), } - storage_account_postfix: str = Field(default_factory=random_secure_string) + storage_account_postfix: str = Field(default_factory=lambda: random_secure_string(length=4)) vnet_subnet_id: typing.Optional[typing.Union[str, None]] = None private_cluster_enabled: bool = False From f3826e5e07d2a105c85db90faf9e6fa14038adee Mon Sep 17 00:00:00 2001 From: eskild <42120229+iameskild@users.noreply.github.com> Date: Tue, 20 Jun 2023 17:55:19 +0200 Subject: [PATCH 044/147] Add InputVar schema to kubernetes_services stage (#1837) --- .../stages/kubernetes_services/__init__.py | 261 +++++++++++++----- .../template/argo-workflows.tf | 4 - .../kubernetes_services/template/clearml.tf | 3 - .../template/conda-store.tf | 11 +- .../template/dask_gateway.tf | 1 - .../template/jupyterhub.tf | 21 +- .../template/monitoring.tf | 1 - .../kubernetes_services/template/variables.tf | 42 +-- src/nebari/schema.py | 10 +- 9 files changed, 214 insertions(+), 140 deletions(-) diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index eceba6678..ce32422bd 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -3,6 +3,8 @@ from typing import Any, Dict, List from urllib.parse import urlencode +from pydantic import Field + from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( NebariHelmProvider, @@ -17,6 +19,87 @@ TIMEOUT = 10 # seconds +# variables shared by multiple services +class KubernetesServicesInputVars(schema.Base): + name: str + environment: str + endpoint: str + realm_id: str + node_groups: Dict[str, Dict[str, str]] + conda_store_default_namespace: str + jupyterhub_logout_redirect_url: str = Field(alias="jupyterhub-logout-redirect-url") + + +class CondaStoreInputVars(schema.Base): + conda_store_environments: Dict[str, Dict[str, Any]] = Field( + alias="conda-store-environments" + ) + conda_store_filesystem_storage: Dict[str, Any] = Field( + alias="conda-store-filesystem-storage" + ) + conda_store_object_storage: Dict[str, Any] = Field( + alias="conda-store-object-storage" + ) + conda_store_extra_settings: Dict[str, Any] = Field( + alias="conda-store-extra-settings" + ) + conda_store_extra_config: Dict[str, Any] = Field(alias="conda-store-extra-config") + conda_store_image = Dict[str, Any] = Field(alias="conda-store-image") + conda_store_image_tag: str = Field(alias="conda-store-image-tag") + conda_store_service_token_scopes: Dict[str, Dict[str, Any]] = Field( + alias="conda-store-service-token-scopes" + ) + + +class JupyterhubInputVars(schema.Base): + cdsdashboards: Dict[str, Any] + jupyterhub_theme: Dict[str, Any] = Field(alias="jupyterhub-theme") + jupyterlab_image: Dict[str, Any] = Field(alias="jupyterlab-image") + jupyterhub_overrides: List[str] = Field(alias="jupyterhub-overrides") + jupyterhub_stared_storage: Dict[str, Any] = Field(alias="jupyterhub-shared-storage") + jupyterhub_shared_endpoint: str = Field(alias="jupyterhub-shared-endpoint") + jupyterhub_profiles = Dict[str, Any] = Field(alias="jupyterlab-profiles") + jupyterhub_image: Dict[str, Any] = Field(alias="jupyterhub-image") + jupyterhub_hub_extraEnv: str = Field(alias="jupyterhub-hub-extraEnv") + idle_culler_settings: Dict[str, Any] = Field(alias="idle-culler-settings") + + +class DaskGatewayInputVars(schema.Base): + dask_worker_image: Dict[str, Any] = Field(alias="dask-worker-image") + dask_gateway_profiles: Dict[str, Any] = Field(alias="dask-gateway-profiles") + + +class MonitoringInputVars(schema.Base): + monitoring_enabled: bool = Field(alias="monitoring-enabled") + + +class ArgoWorkflowsInputVars(schema.Base): + argo_workflows_enabled: bool = Field(alias="argo-workflows-enabled") + argo_workflows_overrides: List[str] = Field(alias="argo-workflows-overrides") + nebari_workflow_controller: bool = Field(alias="nebari-workflow-controller") + workflow_controller_image_tag: str = Field(alias="workflow-controller-image-tag") + keycloak_read_only_user_credentials: Dict[str, Any] = Field( + alias="keycloak-read-only-user-credentials" + ) + + +class KBatchInputVars(schema.Base): + kbatch_enabled: bool = Field(alias="kbatch-enabled") + + +class PrefectInputVars(schema.Base): + prefect_enabled: bool = Field(alias="prefect-enabled") + prefect_token: str = Field(alias="prefect-token") + prefect_image: str = Field(alias="prefect-image") + prefect_overrides: List[str] = Field(alias="prefect-overrides") + + +class ClearMLInputVars(schema.Base): + clearml_enabled: bool = Field(alias="clearml-enabled") + clearml_enable_forwardauth: bool = Field(alias="clearml-enable-forwardauth") + clearml_overrides: List[str] = Field(alias="clearml-overrides") + + def _split_docker_image_name(image_name): name, tag = image_name.split(":") return {"name": name, "tag": tag} @@ -63,6 +146,33 @@ def tf_objects(self) -> List[Dict]: def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): final_logout_uri = f"https://{self.config.domain}/hub/login" + realm_id = stage_outputs["stages/06-kubernetes-keycloak-configuration"][ + "realm_id" + ]["value"] + jupyterhub_shared_endpoint = ( + stage_outputs["stages/02-infrastructure"] + .get("nfs_endpoint", {}) + .get("value") + ) + keycloak_read_only_user_credentials = stage_outputs[ + "stages/06-kubernetes-keycloak-configuration" + ]["keycloak-read-only-user-credentials"]["value"] + + conda_store_token_scopes = { + "cdsdashboards": { + "primary_namespace": "cdsdashboards", + "role_bindings": { + "*/*": ["viewer"], + }, + }, + "dask-gateway": { + "primary_namespace": "", + "role_bindings": { + "*/*": ["viewer"], + }, + }, + } + # Compound any logout URLs from extensions so they are are logged out in succession # when Keycloak and JupyterHub are logged out for ext in self.config.tf_extensions: @@ -78,86 +188,93 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): ): jupyterhub_theme.update({"version": f"v{self.config.nebari_version}"}) - return { - "name": self.config.project_name, - "environment": self.config.namespace, - "endpoint": self.config.domain, - "realm_id": stage_outputs["stages/06-kubernetes-keycloak-configuration"][ - "realm_id" - ]["value"], - "node_groups": _calculate_node_groups(self.config), - # conda-store - "conda-store-environments": { + kubernetes_services_vars = KubernetesServicesInputVars( + name=self.config.project_name, + environment=self.config.namespace, + endpoint=self.config.domain, + realm_id=realm_id, + node_groups=_calculate_node_groups(self.config), + conda_store_default_namespace=self.config.conda_store.default_namespace, + ) + + conda_store_vars = CondaStoreInputVars( + conda_store_environments={ k: v.dict() for k, v in self.config.environments.items() }, - "conda-store-filesystem-storage": self.config.storage.conda_store, - "conda-store-service-token-scopes": { - "cdsdashboards": { - "primary_namespace": "cdsdashboards", - "role_bindings": { - "*/*": ["viewer"], - }, - }, - "dask-gateway": { - "primary_namespace": "", - "role_bindings": { - "*/*": ["viewer"], - }, - }, - }, - "conda-store-default-namespace": self.config.conda_store.default_namespace, - "conda-store-extra-settings": self.config.conda_store.extra_settings, - "conda-store-extra-config": self.config.conda_store.extra_config, - "conda-store-image-tag": self.config.conda_store.image_tag, - # jupyterhub - "cdsdashboards": self.config.cdsdashboards.dict(), - "jupyterhub-theme": self.config.theme.jupyterhub.dict(), - "jupyterhub-image": _split_docker_image_name( - self.config.default_images.jupyterhub - ), - "jupyterhub-shared-storage": self.config.storage.shared_filesystem, - "jupyterhub-shared-endpoint": stage_outputs["stages/02-infrastructure"] - .get("nfs_endpoint", {}) - .get("value"), - "jupyterlab-profiles": self.config.profiles.dict()["jupyterlab"], - "jupyterlab-image": _split_docker_image_name( + conda_store_filesystem_storage=self.config.storage.conda_store, + conda_store_object_storage=self.config.conda_store.object_storage, + conda_store_service_token_scopes=conda_store_token_scopes, + conda_store_extra_settings=self.config.conda_store.extra_settings, + conda_store_extra_config=self.config.conda_store.extra_config, + conda_store_image=self.config.conda_store.image, + conda_store_image_tag=self.config.conda_store.image_tag, + ) + + jupyterhub_vars = JupyterhubInputVars( + cdsdashboards=self.config.cdsdashboards.dict(), + jupyterhub_theme=jupyterhub_theme.dict(), + jupyterlab_image=_split_docker_image_name( self.config.default_images.jupyterlab ), - "jupyterhub-overrides": [json.dumps(self.config.jupyterhub.overrides)], - "jupyterhub-hub-extraEnv": json.dumps( + jupyterhub_stared_storage=self.config.storage.shared_filesystem, + jupyterhub_shared_endpoint=jupyterhub_shared_endpoint, + jupyterhub_profiles=self.config.profiles.dict()["jupyterlab"], + jupyterhub_image=_split_docker_image_name( + self.config.default_images.jupyterhub + ), + jupyterhub_overrides=[json.dumps(self.config.jupyterhub.overrides)], + jupyterhub_hub_extraEnv=json.dumps( self.config.jupyterhub.overrides.get("hub", {}).get("extraEnv", []) ), - # jupyterlab - "idle-culler-settings": self.config.jupyterlab.idle_culler.dict(), - # dask-gateway - "dask-worker-image": _split_docker_image_name( + idle_culler_settings=self.config.jupyterlab.idle_culler.dict(), + ) + + dask_gateway_vars = DaskGatewayInputVars( + dask_worker_image=_split_docker_image_name( self.config.default_images.dask_worker ), - "dask-gateway-profiles": self.config.profiles.dict()["dask_worker"], - # monitoring - "monitoring-enabled": self.config.monitoring.enabled, - # argo-worfklows - "argo-workflows-enabled": self.config.argo_workflows.enabled, - "argo-workflows-overrides": [ - json.dumps(self.config.argo_workflows.overrides) - ], - "nebari-workflow-controller": self.config.argo_workflows.nebari_workflow_controller.enabled, - "keycloak-read-only-user-credentials": stage_outputs[ - "stages/06-kubernetes-keycloak-configuration" - ]["keycloak-read-only-user-credentials"]["value"], - "workflow-controller-image-tag": self.config.argo_workflows.nebari_workflow_controller.image_tag, - # kbatch - "kbatch-enabled": self.config.kbatch.enabled, - # prefect - "prefect-enabled": self.config.prefect.enabled, - "prefect-token": self.config.prefect.token, - "prefect-image": self.config.prefect.image, - "prefect-overrides": self.config.prefect.overrides, - # clearml - "clearml-enabled": self.config.clearml.enabled, - "clearml-enable-forwardauth": self.config.clearml.enable_forward_auth, - "clearml-overrides": [json.dumps(self.config.clearml.overrides)], - "jupyterhub-logout-redirect-url": final_logout_uri, + dask_gateway_profiles=self.config.profiles.dict()["dask_worker"], + ) + + monitoring_vars = MonitoringInputVars( + enabled=self.config.monitoring.enabled, + ) + + argo_workflows_vars = ArgoWorkflowsInputVars( + argo_workflows_enabled=self.config.argo_workflows.enabled, + argo_workflows_overrides=[json.dumps(self.config.argo_workflows.overrides)], + nebari_workflow_controller=self.config.argo_workflows.nebari_workflow_controller.enabled, + workflow_controller_image_tag=self.config.argo_workflows.nebari_workflow_controller.image_tag, + keycloak_read_only_user_credentials=keycloak_read_only_user_credentials, + ) + + kbatch_vars = KBatchInputVars( + kbatch_enabled=self.config.kbatch.enabled, + ) + + prefect_vars = PrefectInputVars( + prefect_enabled=self.config.prefect.enabled, + prefect_token=self.config.prefect.token, + prefect_image=self.config.prefect.image, + prefect_overrides=[json.dumps(self.config.prefect.overrides)], + ) + + clearml_vars = ClearMLInputVars( + clearml_enabled=self.config.clearml.enabled, + clearml_enable_forwardauth=self.config.clearml.enable_forward_auth, + clearml_overrides=[json.dumps(self.config.clearml.overrides)], + ) + + return { + **kubernetes_services_vars.dict(), + **conda_store_vars.dict(), + **jupyterhub_vars.dict(), + **dask_gateway_vars.dict(), + **monitoring_vars.dict(), + **argo_workflows_vars.dict(), + **kbatch_vars.dict(), + **prefect_vars.dict(), + **clearml_vars.dict(), } def check(self, stage_outputs: Dict[str, Dict[str, Any]]): diff --git a/src/_nebari/stages/kubernetes_services/template/argo-workflows.tf b/src/_nebari/stages/kubernetes_services/template/argo-workflows.tf index 34927f2b5..ade8c2c3c 100644 --- a/src/_nebari/stages/kubernetes_services/template/argo-workflows.tf +++ b/src/_nebari/stages/kubernetes_services/template/argo-workflows.tf @@ -2,26 +2,22 @@ variable "argo-workflows-enabled" { description = "Argo Workflows enabled" type = bool - default = true } variable "argo-workflows-overrides" { description = "Argo Workflows helm chart overrides" type = list(string) - default = [] } variable "nebari-workflow-controller" { description = "Nebari Workflow Controller enabled" type = bool - default = true } variable "keycloak-read-only-user-credentials" { description = "Keycloak password for nebari-bot" type = map(string) - default = {} } variable "workflow-controller-image-tag" { diff --git a/src/_nebari/stages/kubernetes_services/template/clearml.tf b/src/_nebari/stages/kubernetes_services/template/clearml.tf index 59c5ceba4..6c619fc65 100644 --- a/src/_nebari/stages/kubernetes_services/template/clearml.tf +++ b/src/_nebari/stages/kubernetes_services/template/clearml.tf @@ -2,20 +2,17 @@ variable "clearml-enabled" { description = "Clearml enabled or disabled" type = bool - default = false } variable "clearml-enable-forwardauth" { description = "Clearml enabled or disabled forward authentication" type = bool - default = false } variable "clearml-overrides" { description = "Clearml helm chart overrides" type = list(string) - default = [] } diff --git a/src/_nebari/stages/kubernetes_services/template/conda-store.tf b/src/_nebari/stages/kubernetes_services/template/conda-store.tf index 330901ec1..904a17e8d 100644 --- a/src/_nebari/stages/kubernetes_services/template/conda-store.tf +++ b/src/_nebari/stages/kubernetes_services/template/conda-store.tf @@ -1,7 +1,6 @@ # ======================= VARIABLES ====================== variable "conda-store-environments" { description = "Conda-Store managed environments" - default = {} } variable "conda-store-filesystem-storage" { @@ -12,31 +11,31 @@ variable "conda-store-filesystem-storage" { variable "conda-store-object-storage" { description = "Conda-Store storage in GB for object storage. Conda-Store uses minio for object storage to be cloud agnostic. If empty default is var.conda-store-filesystem-storage value" type = string - default = null } variable "conda-store-extra-settings" { description = "Conda-Store extra traitlet settings to apply in `c.Class.key = value` form" type = map(any) - default = {} } variable "conda-store-extra-config" { description = "Additional traitlets configuration code to be ran" type = string - default = "" } variable "conda-store-image" { description = "Conda-Store image" type = string - default = "quansight/conda-store-server" } variable "conda-store-image-tag" { description = "Version of conda-store to use" type = string - default = "v0.4.14" +} + +variable "conda-store-service-token-scopes" { + description = "Map of services tokens and scopes for conda-store" + type = map(any) } # ====================== RESOURCES ======================= diff --git a/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf b/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf index 20bbb340a..765be2753 100644 --- a/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf +++ b/src/_nebari/stages/kubernetes_services/template/dask_gateway.tf @@ -9,7 +9,6 @@ variable "dask-worker-image" { variable "dask-gateway-profiles" { description = "Dask Gateway profiles to expose to user" - default = [] } diff --git a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf index ed3a478d7..abb76a3a7 100644 --- a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf +++ b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf @@ -5,17 +5,11 @@ variable "cdsdashboards" { cds_hide_user_named_servers = bool cds_hide_user_dashboard_servers = bool }) - default = { - enabled = true - cds_hide_user_named_servers = true - cds_hide_user_dashboard_servers = false - } } variable "jupyterhub-theme" { description = "JupyterHub theme" type = map(any) - default = {} } variable "jupyterhub-image" { @@ -40,7 +34,6 @@ variable "jupyterhub-shared-storage" { variable "jupyterhub-shared-endpoint" { description = "JupyterHub shared storage nfs endpoint" type = string - default = null } variable "jupyterlab-image" { @@ -53,7 +46,17 @@ variable "jupyterlab-image" { variable "jupyterlab-profiles" { description = "JupyterHub profiles to expose to user" - default = [] +} + +variable "jupyterhub-hub-extraEnv" { + description = "Extracted overrides to merge with jupyterhub.hub.extraEnv" + type = string + default = "[]" +} + +variable "idle-culler-settings" { + description = "Idle culler timeout settings (in minutes)" + type = any } @@ -136,6 +139,6 @@ module "jupyterhub" { jupyterhub-logout-redirect-url = var.jupyterhub-logout-redirect-url jupyterhub-hub-extraEnv = var.jupyterhub-hub-extraEnv - idle-culler-settings = local.idle-culler-settings + idle-culler-settings = var.idle-culler-settings } diff --git a/src/_nebari/stages/kubernetes_services/template/monitoring.tf b/src/_nebari/stages/kubernetes_services/template/monitoring.tf index 255b163d5..ec20a75ba 100644 --- a/src/_nebari/stages/kubernetes_services/template/monitoring.tf +++ b/src/_nebari/stages/kubernetes_services/template/monitoring.tf @@ -1,7 +1,6 @@ variable "monitoring-enabled" { description = "Prometheus and Grafana monitoring enabled" type = bool - default = true } module "monitoring" { diff --git a/src/_nebari/stages/kubernetes_services/template/variables.tf b/src/_nebari/stages/kubernetes_services/template/variables.tf index fd96f4fd0..63c6c5b4f 100644 --- a/src/_nebari/stages/kubernetes_services/template/variables.tf +++ b/src/_nebari/stages/kubernetes_services/template/variables.tf @@ -1,3 +1,5 @@ +# Variables that are shared between multiple kubernetes services + variable "name" { description = "Prefix name to assign to kubernetes resources" type = string @@ -32,47 +34,7 @@ variable "jupyterhub-logout-redirect-url" { default = "" } -variable "jupyterhub-hub-extraEnv" { - description = "Extracted overrides to merge with jupyterhub.hub.extraEnv" - type = string - default = "[]" -} - variable "conda-store-default-namespace" { description = "Default conda-store namespace name" type = string - default = "nebari-git" -} - -variable "conda-store-service-token-scopes" { - description = "Map of services tokens and scopes for conda-store" - type = map(any) - default = { - "cdsdashboards" = { - "primary_namespace" : "cdsdashboards", - "role_bindings" : { - "*/*" : ["viewer"], - } - } - } -} - - -variable "idle-culler-settings" { - description = "Idle culler timeout settings (in minutes)" - type = any -} - -# allows us to merge variables set in the nebari-config.yaml with the default values below -locals { - default-idle-culler-settings = { - kernel_cull_busy = false - kernel_cull_connected = true - kernel_cull_idle_timeout = 15 - kernel_cull_interval = 5 - server_shutdown_no_activity_timeout = 15 - terminal_cull_inactive_timeout = 15 - terminal_cull_interval = 5 - } - idle-culler-settings = merge(local.default-idle-culler-settings, var.idle-culler-settings) } diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 906c4b521..e53098052 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -12,10 +12,6 @@ from pydantic import Field, root_validator, validator from ruamel.yaml import YAML, yaml_object -yaml = YAML() -yaml.preserve_quotes = True -yaml.default_flow_style = False - from _nebari import constants from _nebari.provider.cloud import ( amazon_web_services, @@ -25,6 +21,10 @@ ) from _nebari.version import __version__, rounded_ver_parse +yaml = YAML() +yaml.preserve_quotes = True +yaml.default_flow_style = False + # Regex for suitable project names namestr_regex = r"^[A-Za-z][A-Za-z\-_]*[A-Za-z]$" @@ -207,8 +207,10 @@ class Prefect(Base): class CondaStore(Base): extra_settings: typing.Dict[str, typing.Any] = {} extra_config: str = "" + image: str = "quansight/conda-store-server" image_tag: str = constants.DEFAULT_CONDA_STORE_IMAGE_TAG default_namespace: str = "nebari-git" + object_storage: typing.Union[str, None] = None # ============= Terraform =============== From 46f4b6f854d4e3bf89f30b5ee3f3f43b344609fe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 Jun 2023 15:56:27 +0000 Subject: [PATCH 045/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/stages/infrastructure/__init__.py | 3 ++- src/nebari/schema.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index c1b3a0d9c..fe5875a68 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -255,7 +255,8 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): instance=node_group.instance, min_nodes=node_group.min_nodes, max_nodes=node_group.max_nodes, - ) for name, node_group in self.config.azure.node_groups.items() + ) + for name, node_group in self.config.azure.node_groups.items() }, resource_group_name=f"{self.config.project_name}-{self.config.namespace}", node_resource_group_name=f"{self.config.project_name}-{self.config.namespace}-node-resource-group", diff --git a/src/nebari/schema.py b/src/nebari/schema.py index e53098052..245042099 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -475,7 +475,9 @@ class AzureProvider(Base): "user": NodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), "worker": NodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), } - storage_account_postfix: str = Field(default_factory=lambda: random_secure_string(length=4)) + storage_account_postfix: str = Field( + default_factory=lambda: random_secure_string(length=4) + ) vnet_subnet_id: typing.Optional[typing.Union[str, None]] = None private_cluster_enabled: bool = False From c99334c3efa2a49721a43653ef2bb304a747aace Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Tue, 20 Jun 2023 17:10:54 -0400 Subject: [PATCH 046/147] Fixups for input_var schema --- .../stages/kubernetes_services/__init__.py | 72 ++++++++++--------- src/nebari/hookspecs.py | 1 + src/nebari/schema.py | 3 +- 3 files changed, 43 insertions(+), 33 deletions(-) diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index ce32422bd..6fbe9f509 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -26,25 +26,37 @@ class KubernetesServicesInputVars(schema.Base): endpoint: str realm_id: str node_groups: Dict[str, Dict[str, str]] - conda_store_default_namespace: str jupyterhub_logout_redirect_url: str = Field(alias="jupyterhub-logout-redirect-url") +def _split_docker_image_name(image_name): + name, tag = image_name.split(":") + return {"name": name, "tag": tag} + + +class ImageNameTag(schema.Base): + name: str + tag: str + + class CondaStoreInputVars(schema.Base): - conda_store_environments: Dict[str, Dict[str, Any]] = Field( + conda_store_environments: Dict[str, schema.CondaEnvironment] = Field( alias="conda-store-environments" ) - conda_store_filesystem_storage: Dict[str, Any] = Field( + conda_store_default_namespace: str = Field( + alias="conda-store-default-namespace" + ) + conda_store_filesystem_storage: str = Field( alias="conda-store-filesystem-storage" ) - conda_store_object_storage: Dict[str, Any] = Field( + conda_store_object_storage: str = Field( alias="conda-store-object-storage" ) conda_store_extra_settings: Dict[str, Any] = Field( alias="conda-store-extra-settings" ) - conda_store_extra_config: Dict[str, Any] = Field(alias="conda-store-extra-config") - conda_store_image = Dict[str, Any] = Field(alias="conda-store-image") + conda_store_extra_config: str = Field(alias="conda-store-extra-config") + conda_store_image: str = Field(alias="conda-store-image") conda_store_image_tag: str = Field(alias="conda-store-image-tag") conda_store_service_token_scopes: Dict[str, Dict[str, Any]] = Field( alias="conda-store-service-token-scopes" @@ -54,18 +66,18 @@ class CondaStoreInputVars(schema.Base): class JupyterhubInputVars(schema.Base): cdsdashboards: Dict[str, Any] jupyterhub_theme: Dict[str, Any] = Field(alias="jupyterhub-theme") - jupyterlab_image: Dict[str, Any] = Field(alias="jupyterlab-image") + jupyterlab_image: ImageNameTag = Field(alias="jupyterlab-image") jupyterhub_overrides: List[str] = Field(alias="jupyterhub-overrides") - jupyterhub_stared_storage: Dict[str, Any] = Field(alias="jupyterhub-shared-storage") - jupyterhub_shared_endpoint: str = Field(alias="jupyterhub-shared-endpoint") - jupyterhub_profiles = Dict[str, Any] = Field(alias="jupyterlab-profiles") - jupyterhub_image: Dict[str, Any] = Field(alias="jupyterhub-image") + jupyterhub_stared_storage: str = Field(alias="jupyterhub-shared-storage") + jupyterhub_shared_endpoint: str = Field(None, alias="jupyterhub-shared-endpoint") + jupyterhub_profiles: List[schema.JupyterLabProfile] = Field(alias="jupyterlab-profiles") + jupyterhub_image: ImageNameTag = Field(alias="jupyterhub-image") jupyterhub_hub_extraEnv: str = Field(alias="jupyterhub-hub-extraEnv") idle_culler_settings: Dict[str, Any] = Field(alias="idle-culler-settings") class DaskGatewayInputVars(schema.Base): - dask_worker_image: Dict[str, Any] = Field(alias="dask-worker-image") + dask_worker_image: ImageNameTag = Field(alias="dask-worker-image") dask_gateway_profiles: Dict[str, Any] = Field(alias="dask-gateway-profiles") @@ -89,9 +101,9 @@ class KBatchInputVars(schema.Base): class PrefectInputVars(schema.Base): prefect_enabled: bool = Field(alias="prefect-enabled") - prefect_token: str = Field(alias="prefect-token") - prefect_image: str = Field(alias="prefect-image") - prefect_overrides: List[str] = Field(alias="prefect-overrides") + prefect_token: str = Field(None, alias="prefect-token") + prefect_image: str = Field(None, alias="prefect-image") + prefect_overrides: Dict = Field(alias="prefect-overrides") class ClearMLInputVars(schema.Base): @@ -100,11 +112,6 @@ class ClearMLInputVars(schema.Base): clearml_overrides: List[str] = Field(alias="clearml-overrides") -def _split_docker_image_name(image_name): - name, tag = image_name.split(":") - return {"name": name, "tag": tag} - - def _calculate_node_groups(config: schema.Main): if config.provider == schema.ProviderEnum.aws: return { @@ -194,13 +201,14 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): endpoint=self.config.domain, realm_id=realm_id, node_groups=_calculate_node_groups(self.config), - conda_store_default_namespace=self.config.conda_store.default_namespace, + jupyterhub_logout_redirect_url=final_logout_uri, ) conda_store_vars = CondaStoreInputVars( conda_store_environments={ k: v.dict() for k, v in self.config.environments.items() }, + conda_store_default_namespace=self.config.conda_store.default_namespace, conda_store_filesystem_storage=self.config.storage.conda_store, conda_store_object_storage=self.config.conda_store.object_storage, conda_store_service_token_scopes=conda_store_token_scopes, @@ -237,7 +245,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): ) monitoring_vars = MonitoringInputVars( - enabled=self.config.monitoring.enabled, + monitoring_enabled=self.config.monitoring.enabled, ) argo_workflows_vars = ArgoWorkflowsInputVars( @@ -256,7 +264,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): prefect_enabled=self.config.prefect.enabled, prefect_token=self.config.prefect.token, prefect_image=self.config.prefect.image, - prefect_overrides=[json.dumps(self.config.prefect.overrides)], + prefect_overrides=self.config.prefect.overrides, ) clearml_vars = ClearMLInputVars( @@ -266,15 +274,15 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): ) return { - **kubernetes_services_vars.dict(), - **conda_store_vars.dict(), - **jupyterhub_vars.dict(), - **dask_gateway_vars.dict(), - **monitoring_vars.dict(), - **argo_workflows_vars.dict(), - **kbatch_vars.dict(), - **prefect_vars.dict(), - **clearml_vars.dict(), + **kubernetes_services_vars.dict(by_alias=True), + **conda_store_vars.dict(by_alias=True), + **jupyterhub_vars.dict(by_alias=True), + **dask_gateway_vars.dict(by_alias=True), + **monitoring_vars.dict(by_alias=True), + **argo_workflows_vars.dict(by_alias=True), + **kbatch_vars.dict(by_alias=True), + **prefect_vars.dict(by_alias=True), + **clearml_vars.dict(by_alias=True), } def check(self, stage_outputs: Dict[str, Dict[str, Any]]): diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index fac86ae87..cf276618d 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -1,3 +1,4 @@ + import contextlib import pathlib from typing import Any, Dict, List diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 245042099..0a7c58d38 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -132,6 +132,7 @@ class Base(pydantic.BaseModel): class Config: extra = "forbid" validate_assignment = True + allow_population_by_field_name = True # ============== CI/CD ============= @@ -210,7 +211,7 @@ class CondaStore(Base): image: str = "quansight/conda-store-server" image_tag: str = constants.DEFAULT_CONDA_STORE_IMAGE_TAG default_namespace: str = "nebari-git" - object_storage: typing.Union[str, None] = None + object_storage: str = "200Gi" # ============= Terraform =============== From 2a8999ec147a92b6d367e8cd4c36a10d87836826 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 Jun 2023 21:11:33 +0000 Subject: [PATCH 047/147] [pre-commit.ci] Apply automatic pre-commit fixes --- .../stages/kubernetes_services/__init__.py | 16 ++++++---------- src/nebari/hookspecs.py | 1 - 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index 6fbe9f509..099523e00 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -43,15 +43,9 @@ class CondaStoreInputVars(schema.Base): conda_store_environments: Dict[str, schema.CondaEnvironment] = Field( alias="conda-store-environments" ) - conda_store_default_namespace: str = Field( - alias="conda-store-default-namespace" - ) - conda_store_filesystem_storage: str = Field( - alias="conda-store-filesystem-storage" - ) - conda_store_object_storage: str = Field( - alias="conda-store-object-storage" - ) + conda_store_default_namespace: str = Field(alias="conda-store-default-namespace") + conda_store_filesystem_storage: str = Field(alias="conda-store-filesystem-storage") + conda_store_object_storage: str = Field(alias="conda-store-object-storage") conda_store_extra_settings: Dict[str, Any] = Field( alias="conda-store-extra-settings" ) @@ -70,7 +64,9 @@ class JupyterhubInputVars(schema.Base): jupyterhub_overrides: List[str] = Field(alias="jupyterhub-overrides") jupyterhub_stared_storage: str = Field(alias="jupyterhub-shared-storage") jupyterhub_shared_endpoint: str = Field(None, alias="jupyterhub-shared-endpoint") - jupyterhub_profiles: List[schema.JupyterLabProfile] = Field(alias="jupyterlab-profiles") + jupyterhub_profiles: List[schema.JupyterLabProfile] = Field( + alias="jupyterlab-profiles" + ) jupyterhub_image: ImageNameTag = Field(alias="jupyterhub-image") jupyterhub_hub_extraEnv: str = Field(alias="jupyterhub-hub-extraEnv") idle_culler_settings: Dict[str, Any] = Field(alias="idle-culler-settings") diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index cf276618d..fac86ae87 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -1,4 +1,3 @@ - import contextlib import pathlib from typing import Any, Dict, List From 3e81c67227710db9516a0a112a426964e5a32f12 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 10:22:43 -0400 Subject: [PATCH 048/147] Fixing digital ocean deployment --- src/_nebari/constants.py | 2 +- src/_nebari/provider/cloud/commons.py | 4 +++- src/nebari/schema.py | 18 +++++++++--------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py index dc1b2d884..aedaeee68 100644 --- a/src/_nebari/constants.py +++ b/src/_nebari/constants.py @@ -5,7 +5,7 @@ # 04-kubernetes-ingress DEFAULT_TRAEFIK_IMAGE_TAG = "2.9.1" -HIGHEST_SUPPORTED_K8S_VERSION = "1.25.12" +HIGHEST_SUPPORTED_K8S_VERSION = ("1", "24", "13") DEFAULT_GKE_RELEASE_CHANNEL = "UNSPECIFIED" DEFAULT_NEBARI_DASK_VERSION = CURRENT_RELEASE diff --git a/src/_nebari/provider/cloud/commons.py b/src/_nebari/provider/cloud/commons.py index ed1ed89b7..3dd5602f8 100644 --- a/src/_nebari/provider/cloud/commons.py +++ b/src/_nebari/provider/cloud/commons.py @@ -1,9 +1,11 @@ +import re from _nebari.constants import HIGHEST_SUPPORTED_K8S_VERSION def filter_by_highest_supported_k8s_version(k8s_versions_list): filtered_k8s_versions_list = [] for k8s_version in k8s_versions_list: - if k8s_version.split("-")[0] <= HIGHEST_SUPPORTED_K8S_VERSION: + version = re.search('(\d+)\.(\d+)\.(\d+)', k8s_version).groups() + if version <= HIGHEST_SUPPORTED_K8S_VERSION: filtered_k8s_versions_list.append(k8s_version) return filtered_k8s_versions_list diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 0a7c58d38..5725f31ef 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -406,9 +406,7 @@ class GCPPrivateClusterConfig(Base): class DigitalOceanProvider(Base): region: str = "nyc3" - kubernetes_version: str = Field( - default_factory=lambda: digital_ocean.kubernetes_versions()[-1] - ) + kubernetes_version: typing.Optional[str] # Digital Ocean image slugs are listed here https://slugs.do-api.dev/ node_groups: typing.Dict[str, NodeGroup] = { "general": NodeGroup(instance="g-8vcpu-32gb", min_nodes=1, max_nodes=1), @@ -417,14 +415,16 @@ class DigitalOceanProvider(Base): } tags: typing.Optional[typing.List[str]] = [] - @validator("kubernetes_version") - def _validate_kubernetes_version(cls, value): - available_kubernetes_versions = digital_ocean.kubernetes_versions() - if value not in available_kubernetes_versions: + @root_validator + def _validate_kubernetes_version(cls, values): + available_kubernetes_versions = digital_ocean.kubernetes_versions(values["region"]) + if values['kubernetes_version'] is not None and values['kubernetes_version'] not in available_kubernetes_versions: raise ValueError( - f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." + f"\nInvalid `kubernetes-version` provided: {values['kubernetes_version']}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." ) - return value + else: + values['kubernetes_version'] = available_kubernetes_versions[-1] + return values class GoogleCloudPlatformProvider(Base): From 719f5780fa8dcd87220495ca847e3cf0fced15c5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Jun 2023 14:25:20 +0000 Subject: [PATCH 049/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/provider/cloud/commons.py | 3 ++- src/nebari/schema.py | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/_nebari/provider/cloud/commons.py b/src/_nebari/provider/cloud/commons.py index 3dd5602f8..aef9a8c90 100644 --- a/src/_nebari/provider/cloud/commons.py +++ b/src/_nebari/provider/cloud/commons.py @@ -1,11 +1,12 @@ import re + from _nebari.constants import HIGHEST_SUPPORTED_K8S_VERSION def filter_by_highest_supported_k8s_version(k8s_versions_list): filtered_k8s_versions_list = [] for k8s_version in k8s_versions_list: - version = re.search('(\d+)\.(\d+)\.(\d+)', k8s_version).groups() + version = re.search("(\d+)\.(\d+)\.(\d+)", k8s_version).groups() if version <= HIGHEST_SUPPORTED_K8S_VERSION: filtered_k8s_versions_list.append(k8s_version) return filtered_k8s_versions_list diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 5725f31ef..4fe803c2e 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -417,13 +417,18 @@ class DigitalOceanProvider(Base): @root_validator def _validate_kubernetes_version(cls, values): - available_kubernetes_versions = digital_ocean.kubernetes_versions(values["region"]) - if values['kubernetes_version'] is not None and values['kubernetes_version'] not in available_kubernetes_versions: + available_kubernetes_versions = digital_ocean.kubernetes_versions( + values["region"] + ) + if ( + values["kubernetes_version"] is not None + and values["kubernetes_version"] not in available_kubernetes_versions + ): raise ValueError( f"\nInvalid `kubernetes-version` provided: {values['kubernetes_version']}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." ) else: - values['kubernetes_version'] = available_kubernetes_versions[-1] + values["kubernetes_version"] = available_kubernetes_versions[-1] return values From bba0161265347d681ed3227c0172baf3256e96ea Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 14:12:10 -0400 Subject: [PATCH 050/147] Further enhancements to the schema and making domain optional --- src/_nebari/deploy.py | 5 +- src/_nebari/stages/base.py | 13 +- src/_nebari/stages/infrastructure/__init__.py | 37 +++- .../stages/kubernetes_ingress/__init__.py | 45 ++--- .../stages/kubernetes_keycloak/__init__.py | 31 +--- .../stages/kubernetes_services/__init__.py | 36 +--- src/nebari/schema.py | 167 +++++++++--------- 7 files changed, 152 insertions(+), 182 deletions(-) diff --git a/src/_nebari/deploy.py b/src/_nebari/deploy.py index 18d2c38a5..a43a81645 100644 --- a/src/_nebari/deploy.py +++ b/src/_nebari/deploy.py @@ -74,7 +74,10 @@ def deploy_configuration( ) ) - logger.info(f"All nebari endpoints will be under https://{config.domain}") + if config.domain is None: + logger.info(f"All nebari endpoints will be under kubernetes load balancer address which cannot be known before deployment") + else: + logger.info(f"All nebari endpoints will be under https://{config.domain}") if disable_checks: logger.warning( diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index df16612d0..a09f5c61a 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -49,6 +49,13 @@ def render(self) -> Dict[str, str]: def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): return {} + def set_outputs(self, stage_outputs: Dict[str, Dict[str, Any]], outputs: Dict[str, Any]): + stage_key = "stages/" + self.name + if stage_key not in stage_outputs: + stage_outputs[stage_key] = {**outputs} + else: + stage_outputs[stage_key].update(outputs) + @contextlib.contextmanager def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): deploy_config = dict( @@ -60,7 +67,7 @@ def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): deploy_config["terraform_import"] = True deploy_config["state_imports"] = state_imports - stage_outputs["stages/" + self.name] = terraform.deploy(**deploy_config) + self.set_outputs(stage_outputs, terraform.deploy(**deploy_config)) yield def check(self, stage_outputs: Dict[str, Dict[str, Any]]): @@ -73,14 +80,14 @@ def destroy( status: Dict[str, bool], ignore_errors: bool = True, ): - stage_outputs["stages/" + self.name] = terraform.deploy( + self.set_outputs(stage_outputs, terraform.deploy( directory=str(self.output_directory / self.stage_prefix), input_vars=self.input_vars(stage_outputs), terraform_init=True, terraform_import=True, terraform_apply=False, terraform_destroy=False, - ) + )) yield try: terraform.deploy( diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index fe5875a68..edc217c72 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -53,8 +53,8 @@ class GCPNodeGroupInputVars(schema.Base): instance_type: str min_size: int max_size: int - labels: Dict[str, str] = {} - preemptible: bool = False + labels: Dict[str, str] + preemptible: bool guest_accelerators: List[GCPGuestAccelerators] @@ -125,6 +125,33 @@ class AWSInputVars(schema.Base): kubeconfig_filename: str = get_kubeconfig_filename() +def _calculate_node_groups(config: schema.Main): + if config.provider == schema.ProviderEnum.aws: + return { + group: {"key": "eks.amazonaws.com/nodegroup", "value": group} + for group in ["general", "user", "worker"] + } + elif config.provider == schema.ProviderEnum.gcp: + return { + group: {"key": "cloud.google.com/gke-nodepool", "value": group} + for group in ["general", "user", "worker"] + } + elif config.provider == schema.ProviderEnum.azure: + return { + group: {"key": "azure-node-pool", "value": group} + for group in ["general", "user", "worker"] + } + elif config.provider == schema.ProviderEnum.do: + return { + group: {"key": "doks.digitalocean.com/node-pool", "value": group} + for group in ["general", "user", "worker"] + } + elif config.provider == schema.ProviderEnum.existing: + return config.existing.node_selectors + else: + return config.local.dict()["node_selectors"] + + @contextlib.contextmanager def kubernetes_provider_context(kubernetes_credentials: Dict[str, str]): credential_mapping = { @@ -227,9 +254,11 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): node_groups=[ GCPNodeGroupInputVars( name=name, + labels=node_group.labels, instance_type=node_group.instance, min_size=node_group.min_nodes, max_size=node_group.max_nodes, + preemptible=node_group.preemptible, guest_accelerators=node_group.guest_accelerators, ) for name, node_group in self.config.google_cloud_platform.node_groups.items() @@ -316,6 +345,10 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]): print(f"After stage={self.name} kubernetes cluster successfully provisioned") + def set_outputs(self, stage_outputs: Dict[str, Dict[str, Any]], outputs: Dict[str, Any]): + outputs["node_selectors"] = _calculate_node_groups(self.config) + super().set_outputs(stage_outputs, outputs) + @contextlib.contextmanager def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): with super().deploy(stage_outputs): diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index 3c5708331..52de7c7ed 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -76,39 +76,12 @@ def provision_ingress_dns( checks.check_ingress_dns(stage_outputs, config, disable_prompt) -def _calculate_node_groups(config: schema.Main): - if config.provider == schema.ProviderEnum.aws: - return { - group: {"key": "eks.amazonaws.com/nodegroup", "value": group} - for group in ["general", "user", "worker"] - } - elif config.provider == schema.ProviderEnum.gcp: - return { - group: {"key": "cloud.google.com/gke-nodepool", "value": group} - for group in ["general", "user", "worker"] - } - elif config.provider == schema.ProviderEnum.azure: - return { - group: {"key": "azure-node-pool", "value": group} - for group in ["general", "user", "worker"] - } - elif config.provider == schema.ProviderEnum.do: - return { - group: {"key": "doks.digitalocean.com/node-pool", "value": group} - for group in ["general", "user", "worker"] - } - elif config.provider == schema.ProviderEnum.existing: - return config.existing.node_selectors - else: - return config.local.dict()["node_selectors"] - - def check_ingress_dns(stage_outputs, config, disable_prompt): directory = "stages/04-kubernetes-ingress" ip_or_name = stage_outputs[directory]["load_balancer_address"]["value"] ip = socket.gethostbyname(ip_or_name["hostname"] or ip_or_name["ip"]) - domain_name = config.domain + domain_name = stage_outputs[directory]["domain"] def _attempt_dns_lookup( domain_name, ip, num_attempts=NUM_ATTEMPTS, timeout=TIMEOUT @@ -183,12 +156,26 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): }, "name": self.config.project_name, "environment": self.config.namespace, - "node_groups": _calculate_node_groups(self.config), + "node_groups": stage_outputs["stages/02-infrastructure"]["node_selectors"], **self.config.ingress.terraform_overrides, }, **cert_details, } + def set_outputs(self, stage_outputs: Dict[str, Dict[str, Any]], outputs: Dict[str, Any]): + ip_or_name = outputs["load_balancer_address"][ + "value" + ] + host = ip_or_name["hostname"] or ip_or_name["ip"] + host = host.strip("\n") + + if self.config.domain is None: + outputs["domain"] = host + else: + outputs["domain"] = self.config.domain + + super().set_outputs(stage_outputs, outputs) + def check(self, stage_outputs: Dict[str, Dict[str, Any]]): def _attempt_tcp_connect( host, port, num_attempts=NUM_ATTEMPTS, timeout=TIMEOUT diff --git a/src/_nebari/stages/kubernetes_keycloak/__init__.py b/src/_nebari/stages/kubernetes_keycloak/__init__.py index ae7061cb0..2e8eedc33 100644 --- a/src/_nebari/stages/kubernetes_keycloak/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak/__init__.py @@ -42,33 +42,6 @@ def keycloak_provider_context(keycloak_credentials: Dict[str, str]): yield -def _calculate_node_groups(config: schema.Main): - if config.provider == schema.ProviderEnum.aws: - return { - group: {"key": "eks.amazonaws.com/nodegroup", "value": group} - for group in ["general", "user", "worker"] - } - elif config.provider == schema.ProviderEnum.gcp: - return { - group: {"key": "cloud.google.com/gke-nodepool", "value": group} - for group in ["general", "user", "worker"] - } - elif config.provider == schema.ProviderEnum.azure: - return { - group: {"key": "azure-node-pool", "value": group} - for group in ["general", "user", "worker"] - } - elif config.provider == schema.ProviderEnum.do: - return { - group: {"key": "doks.digitalocean.com/node-pool", "value": group} - for group in ["general", "user", "worker"] - } - elif config.provider == schema.ProviderEnum.existing: - return config.existing.node_selectors - else: - return config.local.dict()["node_selectors"] - - class KubernetesKeycloakStage(NebariTerraformStage): name = "05-kubernetes-keycloak" priority = 50 @@ -84,10 +57,10 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): return InputVars( name=self.config.project_name, environment=self.config.namespace, - endpoint=self.config.domain, + endpoint=stage_outputs["stages/04-kubernetes-ingress"]["domain"], initial_root_password=self.config.security.keycloak.initial_root_password, overrides=[json.dumps(self.config.security.keycloak.overrides)], - node_group=_calculate_node_groups(self.config)["general"], + node_group=stage_outputs["stages/02-infrastructure"]["node_selectors"]["general"], ).dict() def check(self, stage_outputs: Dict[str, Dict[str, Any]]): diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index 099523e00..d34c93dcc 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -108,33 +108,6 @@ class ClearMLInputVars(schema.Base): clearml_overrides: List[str] = Field(alias="clearml-overrides") -def _calculate_node_groups(config: schema.Main): - if config.provider == schema.ProviderEnum.aws: - return { - group: {"key": "eks.amazonaws.com/nodegroup", "value": group} - for group in ["general", "user", "worker"] - } - elif config.provider == schema.ProviderEnum.gcp: - return { - group: {"key": "cloud.google.com/gke-nodepool", "value": group} - for group in ["general", "user", "worker"] - } - elif config.provider == schema.ProviderEnum.azure: - return { - group: {"key": "azure-node-pool", "value": group} - for group in ["general", "user", "worker"] - } - elif config.provider == schema.ProviderEnum.do: - return { - group: {"key": "doks.digitalocean.com/node-pool", "value": group} - for group in ["general", "user", "worker"] - } - elif config.provider == schema.ProviderEnum.existing: - return config.existing.node_selectors - else: - return config.local.dict()["node_selectors"] - - class KubernetesServicesStage(NebariTerraformStage): name = "07-kubernetes-services" priority = 70 @@ -147,7 +120,8 @@ def tf_objects(self) -> List[Dict]: ] def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): - final_logout_uri = f"https://{self.config.domain}/hub/login" + domain = stage_outputs["stages/04-kubernetes-ingress"]["domain"] + final_logout_uri = f"https://{domain}/hub/login" realm_id = stage_outputs["stages/06-kubernetes-keycloak-configuration"][ "realm_id" @@ -181,7 +155,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): for ext in self.config.tf_extensions: if ext.logout != "": final_logout_uri = "{}?{}".format( - f"https://{self.config.domain}/{ext.urlslug}{ext.logout}", + f"https://{domain}/{ext.urlslug}{ext.logout}", urlencode({"redirect_uri": final_logout_uri}), ) @@ -194,9 +168,9 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): kubernetes_services_vars = KubernetesServicesInputVars( name=self.config.project_name, environment=self.config.namespace, - endpoint=self.config.domain, + endpoint=domain, realm_id=realm_id, - node_groups=_calculate_node_groups(self.config), + node_groups=stage_outputs["stages/02-infrastructure"]["node_selectors"], jupyterhub_logout_redirect_url=final_logout_uri, ) diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 4fe803c2e..7bcbd69a1 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -9,7 +9,7 @@ from abc import ABC import pydantic -from pydantic import Field, root_validator, validator +from pydantic import Field, root_validator, validator, conint from ruamel.yaml import YAML, yaml_object from _nebari import constants @@ -348,38 +348,38 @@ class NodeSelector(Base): worker: KeyValueDict -class NodeGroup(Base): +class DigitalOceanNodeGroup(Base): + """Representation of a node group with Digital Ocean + + - Kubernetes limits: https://docs.digitalocean.com/products/kubernetes/details/limits/ + - Available instance types: https://slugs.do-api.dev/ + """ instance: str - min_nodes: int - max_nodes: int - gpu: bool = False - guest_accelerators: typing.List[typing.Dict] = [] + min_nodes: conint(ge=1) = 1 + max_nodes: conint(ge=1) = 1 - class Config: - extra = "allow" - @validator("guest_accelerators") - def validate_guest_accelerators(cls, v): - if not v: - return v - if not isinstance(v, list): - raise ValueError("guest_accelerators must be a list") - for i in v: - assertion_error_message = """ - In order to successfully use guest accelerators, you must specify the following parameters: - - name (str): Machine type name of the GPU, available at https://cloud.google.com/compute/docs/gpus - count (int): Number of GPUs to attach to the instance - - See general information regarding GPU support at: - # TODO: replace with nebari.dev new URL - https://docs.nebari.dev/en/stable/source/admin_guide/gpu.html?#add-gpu-node-group - """ - try: - assert "name" in i and "count" in i - assert isinstance(i["name"], str) and isinstance(i["count"], int) - except AssertionError: - raise ValueError(assertion_error_message) +class DigitalOceanProvider(Base): + region: str = "nyc3" + kubernetes_version: typing.Optional[str] + # Digital Ocean image slugs are listed here https://slugs.do-api.dev/ + node_groups: typing.Dict[str, DigitalOceanNodeGroup] = { + "general": DigitalOceanNodeGroup(instance="g-8vcpu-32gb", min_nodes=1, max_nodes=1), + "user": DigitalOceanNodeGroup(instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5), + "worker": DigitalOceanNodeGroup(instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5), + } + tags: typing.Optional[typing.List[str]] = [] + + @root_validator + def _validate_kubernetes_version(cls, values): + available_kubernetes_versions = digital_ocean.kubernetes_versions(values["region"]) + if values['kubernetes_version'] is not None and values['kubernetes_version'] not in available_kubernetes_versions: + raise ValueError( + f"\nInvalid `kubernetes-version` provided: {values['kubernetes_version']}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." + ) + else: + values['kubernetes_version'] = available_kubernetes_versions[-1] + return values class GCPIPAllocationPolicy(Base): @@ -404,47 +404,35 @@ class GCPPrivateClusterConfig(Base): master_ipv4_cidr_block: str -class DigitalOceanProvider(Base): - region: str = "nyc3" - kubernetes_version: typing.Optional[str] - # Digital Ocean image slugs are listed here https://slugs.do-api.dev/ - node_groups: typing.Dict[str, NodeGroup] = { - "general": NodeGroup(instance="g-8vcpu-32gb", min_nodes=1, max_nodes=1), - "user": NodeGroup(instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5), - "worker": NodeGroup(instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5), - } - tags: typing.Optional[typing.List[str]] = [] +class GCPGuestAccelerator(Base): + """ + See general information regarding GPU support at: + # TODO: replace with nebari.dev new URL + https://docs.nebari.dev/en/stable/source/admin_guide/gpu.html?#add-gpu-node-group + """ + name: str + count: conint(ge=1) = 1 - @root_validator - def _validate_kubernetes_version(cls, values): - available_kubernetes_versions = digital_ocean.kubernetes_versions( - values["region"] - ) - if ( - values["kubernetes_version"] is not None - and values["kubernetes_version"] not in available_kubernetes_versions - ): - raise ValueError( - f"\nInvalid `kubernetes-version` provided: {values['kubernetes_version']}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." - ) - else: - values["kubernetes_version"] = available_kubernetes_versions[-1] - return values + +class GCPNodeGroup(Base): + instance: str + min_nodes: conint(ge=0) = 0 + max_nodes: conint(ge=1) = 1 + preemptible: bool = False + labels: typing.Dict[str, str] = {} + guest_accelerators: typing.List[GCPGuestAccelerator] = [] class GoogleCloudPlatformProvider(Base): project: str = Field(default_factory=lambda: os.environ["PROJECT_ID"]) region: str = "us-central1" availability_zones: typing.Optional[typing.List[str]] = [] - kubernetes_version: str = Field( - default_factory=lambda: google_cloud.kubernetes_versions("us-central1")[-1] - ) - + kubernetes_version: typing.Optional[str] release_channel: str = constants.DEFAULT_GKE_RELEASE_CHANNEL - node_groups: typing.Dict[str, NodeGroup] = { - "general": NodeGroup(instance="n1-standard-8", min_nodes=1, max_nodes=1), - "user": NodeGroup(instance="n1-standard-4", min_nodes=0, max_nodes=5), - "worker": NodeGroup(instance="n1-standard-4", min_nodes=0, max_nodes=5), + node_groups: typing.Dict[str, GCPNodeGroup] = { + "general": GCPNodeGroup(instance="n1-standard-8", min_nodes=1, max_nodes=1), + "user": GCPNodeGroup(instance="n1-standard-4", min_nodes=0, max_nodes=5), + "worker": GCPNodeGroup(instance="n1-standard-4", min_nodes=0, max_nodes=5), } tags: typing.Optional[typing.List[str]] = [] networking_mode: str = "ROUTE" @@ -460,26 +448,31 @@ class GoogleCloudPlatformProvider(Base): typing.Union[GCPPrivateClusterConfig, None] ] = None - @validator("kubernetes_version") - def _validate_kubernetes_version(cls, value): - available_kubernetes_versions = google_cloud.kubernetes_versions("us-central1") - if value not in available_kubernetes_versions: + @root_validator + def _validate_kubernetes_version(cls, values): + available_kubernetes_versions = google_cloud.kubernetes_versions(values["region"]) + if values['kubernetes_version'] is not None and values['kubernetes_version'] not in available_kubernetes_versions: raise ValueError( - f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." + f"\nInvalid `kubernetes-version` provided: {values['kubernetes_version']}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." ) - return value + else: + values['kubernetes_version'] = available_kubernetes_versions[-1] + return values + + +class AzureNodeGroup(Base): + instance: str + min_nodes: int + max_nodes: int class AzureProvider(Base): region: str = "Central US" - kubernetes_version: str = Field( - default_factory=lambda: azure_cloud.kubernetes_versions()[-1] - ) - - node_groups: typing.Dict[str, NodeGroup] = { - "general": NodeGroup(instance="Standard_D8_v3", min_nodes=1, max_nodes=1), - "user": NodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), - "worker": NodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), + kubernetes_version: typing.Optional[str] + node_groups: typing.Dict[str, AzureNodeGroup] = { + "general": AzureNodeGroup(instance="Standard_D8_v3", min_nodes=1, max_nodes=1), + "user": AzureNodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), + "worker": AzureNodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), } storage_account_postfix: str = Field( default_factory=lambda: random_secure_string(length=4) @@ -490,7 +483,9 @@ class AzureProvider(Base): @validator("kubernetes_version") def _validate_kubernetes_version(cls, value): available_kubernetes_versions = azure_cloud.kubernetes_versions() - if value not in available_kubernetes_versions: + if value is None: + value = available_kubernetes_versions[-1] + elif value not in available_kubernetes_versions: raise ValueError( f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." ) @@ -510,10 +505,7 @@ class AmazonWebServicesProvider(Base): default_factory=lambda: os.environ.get("AWS_DEFAULT_REGION", "us-west-2") ) availability_zones: typing.Optional[typing.List[str]] - kubernetes_version: str = Field( - default_factory=lambda: amazon_web_services.kubernetes_versions()[-1] - ) - + kubernetes_version: typing.Optional[str] node_groups: typing.Dict[str, AWSNodeGroup] = { "general": AWSNodeGroup(instance="m5.2xlarge", min_nodes=1, max_nodes=1), "user": AWSNodeGroup( @@ -530,16 +522,17 @@ class AmazonWebServicesProvider(Base): @validator("kubernetes_version") def _validate_kubernetes_version(cls, value): available_kubernetes_versions = amazon_web_services.kubernetes_versions() - if value not in available_kubernetes_versions: + if value is None: + value = available_kubernetes_versions[-1] + elif value not in available_kubernetes_versions: raise ValueError( f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." ) return value @root_validator - def _validate_provider(cls, values): - # populate availability zones if empty - if values.get("availability_zones") is None: + def _validate_availability_zones(cls, values): + if values["availability_zones"] is None: zones = amazon_web_services.zones(values["region"]) values["availability_zones"] = list(sorted(zones))[:2] return values @@ -854,7 +847,7 @@ class Main(Base): # In nebari_version only use major.minor.patch version - drop any pre/post/dev suffixes nebari_version: str = __version__ ci_cd: CICD = CICD() - domain: str + domain: typing.Optional[str] terraform_state: TerraformState = TerraformState() certificate: Certificate = Certificate() helm_extensions: typing.List[HelmExtension] = [] From 436529cfdc0cc267c0d050636ea3eabb32d8b76a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Jun 2023 18:13:01 +0000 Subject: [PATCH 051/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/deploy.py | 4 +- src/_nebari/stages/base.py | 23 ++++++---- src/_nebari/stages/infrastructure/__init__.py | 4 +- .../stages/kubernetes_ingress/__init__.py | 12 +++--- .../stages/kubernetes_keycloak/__init__.py | 4 +- src/nebari/schema.py | 42 +++++++++++++------ 6 files changed, 60 insertions(+), 29 deletions(-) diff --git a/src/_nebari/deploy.py b/src/_nebari/deploy.py index a43a81645..aff95693f 100644 --- a/src/_nebari/deploy.py +++ b/src/_nebari/deploy.py @@ -75,7 +75,9 @@ def deploy_configuration( ) if config.domain is None: - logger.info(f"All nebari endpoints will be under kubernetes load balancer address which cannot be known before deployment") + logger.info( + "All nebari endpoints will be under kubernetes load balancer address which cannot be known before deployment" + ) else: logger.info(f"All nebari endpoints will be under https://{config.domain}") diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index a09f5c61a..0d465bcd0 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -49,7 +49,9 @@ def render(self) -> Dict[str, str]: def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): return {} - def set_outputs(self, stage_outputs: Dict[str, Dict[str, Any]], outputs: Dict[str, Any]): + def set_outputs( + self, stage_outputs: Dict[str, Dict[str, Any]], outputs: Dict[str, Any] + ): stage_key = "stages/" + self.name if stage_key not in stage_outputs: stage_outputs[stage_key] = {**outputs} @@ -80,14 +82,17 @@ def destroy( status: Dict[str, bool], ignore_errors: bool = True, ): - self.set_outputs(stage_outputs, terraform.deploy( - directory=str(self.output_directory / self.stage_prefix), - input_vars=self.input_vars(stage_outputs), - terraform_init=True, - terraform_import=True, - terraform_apply=False, - terraform_destroy=False, - )) + self.set_outputs( + stage_outputs, + terraform.deploy( + directory=str(self.output_directory / self.stage_prefix), + input_vars=self.input_vars(stage_outputs), + terraform_init=True, + terraform_import=True, + terraform_apply=False, + terraform_destroy=False, + ), + ) yield try: terraform.deploy( diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index edc217c72..5ddcdcc74 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -345,7 +345,9 @@ def check(self, stage_outputs: Dict[str, Dict[str, Any]]): print(f"After stage={self.name} kubernetes cluster successfully provisioned") - def set_outputs(self, stage_outputs: Dict[str, Dict[str, Any]], outputs: Dict[str, Any]): + def set_outputs( + self, stage_outputs: Dict[str, Dict[str, Any]], outputs: Dict[str, Any] + ): outputs["node_selectors"] = _calculate_node_groups(self.config) super().set_outputs(stage_outputs, outputs) diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index 52de7c7ed..bda8d6795 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -156,16 +156,18 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): }, "name": self.config.project_name, "environment": self.config.namespace, - "node_groups": stage_outputs["stages/02-infrastructure"]["node_selectors"], + "node_groups": stage_outputs["stages/02-infrastructure"][ + "node_selectors" + ], **self.config.ingress.terraform_overrides, }, **cert_details, } - def set_outputs(self, stage_outputs: Dict[str, Dict[str, Any]], outputs: Dict[str, Any]): - ip_or_name = outputs["load_balancer_address"][ - "value" - ] + def set_outputs( + self, stage_outputs: Dict[str, Dict[str, Any]], outputs: Dict[str, Any] + ): + ip_or_name = outputs["load_balancer_address"]["value"] host = ip_or_name["hostname"] or ip_or_name["ip"] host = host.strip("\n") diff --git a/src/_nebari/stages/kubernetes_keycloak/__init__.py b/src/_nebari/stages/kubernetes_keycloak/__init__.py index 2e8eedc33..1eb9e0080 100644 --- a/src/_nebari/stages/kubernetes_keycloak/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak/__init__.py @@ -60,7 +60,9 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): endpoint=stage_outputs["stages/04-kubernetes-ingress"]["domain"], initial_root_password=self.config.security.keycloak.initial_root_password, overrides=[json.dumps(self.config.security.keycloak.overrides)], - node_group=stage_outputs["stages/02-infrastructure"]["node_selectors"]["general"], + node_group=stage_outputs["stages/02-infrastructure"]["node_selectors"][ + "general" + ], ).dict() def check(self, stage_outputs: Dict[str, Dict[str, Any]]): diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 7bcbd69a1..af07add67 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -9,7 +9,7 @@ from abc import ABC import pydantic -from pydantic import Field, root_validator, validator, conint +from pydantic import Field, conint, root_validator, validator from ruamel.yaml import YAML, yaml_object from _nebari import constants @@ -351,9 +351,10 @@ class NodeSelector(Base): class DigitalOceanNodeGroup(Base): """Representation of a node group with Digital Ocean - - Kubernetes limits: https://docs.digitalocean.com/products/kubernetes/details/limits/ - - Available instance types: https://slugs.do-api.dev/ + - Kubernetes limits: https://docs.digitalocean.com/products/kubernetes/details/limits/ + - Available instance types: https://slugs.do-api.dev/ """ + instance: str min_nodes: conint(ge=1) = 1 max_nodes: conint(ge=1) = 1 @@ -364,21 +365,32 @@ class DigitalOceanProvider(Base): kubernetes_version: typing.Optional[str] # Digital Ocean image slugs are listed here https://slugs.do-api.dev/ node_groups: typing.Dict[str, DigitalOceanNodeGroup] = { - "general": DigitalOceanNodeGroup(instance="g-8vcpu-32gb", min_nodes=1, max_nodes=1), - "user": DigitalOceanNodeGroup(instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5), - "worker": DigitalOceanNodeGroup(instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5), + "general": DigitalOceanNodeGroup( + instance="g-8vcpu-32gb", min_nodes=1, max_nodes=1 + ), + "user": DigitalOceanNodeGroup( + instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5 + ), + "worker": DigitalOceanNodeGroup( + instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5 + ), } tags: typing.Optional[typing.List[str]] = [] @root_validator def _validate_kubernetes_version(cls, values): - available_kubernetes_versions = digital_ocean.kubernetes_versions(values["region"]) - if values['kubernetes_version'] is not None and values['kubernetes_version'] not in available_kubernetes_versions: + available_kubernetes_versions = digital_ocean.kubernetes_versions( + values["region"] + ) + if ( + values["kubernetes_version"] is not None + and values["kubernetes_version"] not in available_kubernetes_versions + ): raise ValueError( f"\nInvalid `kubernetes-version` provided: {values['kubernetes_version']}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." ) else: - values['kubernetes_version'] = available_kubernetes_versions[-1] + values["kubernetes_version"] = available_kubernetes_versions[-1] return values @@ -410,6 +422,7 @@ class GCPGuestAccelerator(Base): # TODO: replace with nebari.dev new URL https://docs.nebari.dev/en/stable/source/admin_guide/gpu.html?#add-gpu-node-group """ + name: str count: conint(ge=1) = 1 @@ -450,13 +463,18 @@ class GoogleCloudPlatformProvider(Base): @root_validator def _validate_kubernetes_version(cls, values): - available_kubernetes_versions = google_cloud.kubernetes_versions(values["region"]) - if values['kubernetes_version'] is not None and values['kubernetes_version'] not in available_kubernetes_versions: + available_kubernetes_versions = google_cloud.kubernetes_versions( + values["region"] + ) + if ( + values["kubernetes_version"] is not None + and values["kubernetes_version"] not in available_kubernetes_versions + ): raise ValueError( f"\nInvalid `kubernetes-version` provided: {values['kubernetes_version']}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." ) else: - values['kubernetes_version'] = available_kubernetes_versions[-1] + values["kubernetes_version"] = available_kubernetes_versions[-1] return values From c3f1a6e54dd4259ea6ef8d3986d25415b56bdfcf Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 14:38:00 -0400 Subject: [PATCH 052/147] Prevent bucket from being deleted each time --- .../stages/terraform_state/template/do/modules/spaces/main.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/_nebari/stages/terraform_state/template/do/modules/spaces/main.tf b/src/_nebari/stages/terraform_state/template/do/modules/spaces/main.tf index e0e809c03..fc2d34c60 100644 --- a/src/_nebari/stages/terraform_state/template/do/modules/spaces/main.tf +++ b/src/_nebari/stages/terraform_state/template/do/modules/spaces/main.tf @@ -5,4 +5,8 @@ resource "digitalocean_spaces_bucket" "main" { force_destroy = var.force_destroy acl = (var.public ? "public-read" : "private") + + versioning { + enabled = false + } } From 035949995bb55ecb2ff9d23ec3cf1bbefc38566f Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 15:57:15 -0400 Subject: [PATCH 053/147] Significantly simplify the initialization --- src/_nebari/initialize.py | 94 ++++++++++++++------------- src/_nebari/provider/cloud/commons.py | 2 +- src/nebari/schema.py | 25 +++---- 3 files changed, 64 insertions(+), 57 deletions(-) diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index e864636b5..dcb9677db 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -33,82 +33,86 @@ def render_config( disable_prompt: bool = False, ssl_cert_email: str = None, ): + config = { + 'provider': cloud_provider.value, + 'namespace': namespace, + 'nebari_version': __version__, + } + if project_name is None and not disable_prompt: project_name = input("Provide project name: ") + config['project_name'] = project_name if nebari_domain is None and not disable_prompt: nebari_domain = input("Provide domain: ") + config['domain'] = nebari_domain - config = schema.Main( - project_name=project_name, - domain=nebari_domain, - provider=cloud_provider, - namespace=namespace, - nebari_version=__version__, - ) - config.ci_cd.type = ci_provider - config.terraform_state.type = terraform_state + config['ci_cd'] = {'type': ci_provider.value} + config['terraform_state'] = {'type': terraform_state.value} # Save default password to file - default_password_filename = Path(tempfile.gettempdir()) / "NEBARI_DEFAULT_PASSWORD" + default_password_filename = os.path.join( + tempfile.gettempdir(), "NEBARI_DEFAULT_PASSWORD" + ) + config['security'] = {'keycloak': {'initial_root_password': schema.random_secure_string(length=32)}} with open(default_password_filename, "w") as f: - f.write(config.security.keycloak.initial_root_password) + f.write(config['security']['keycloak']['initial_root_password']) os.chmod(default_password_filename, 0o700) - config.theme.jupyterhub.hub_title = f"Nebari - { project_name }" - config.theme.jupyterhub.welcome = """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""" + config['theme'] = {'jupyterhub': {'hub_title': f"Nebari - { project_name }"}} + config['theme']['jupyterhub']['welcome'] = """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""" - config.security.authentication.type = auth_provider - if config.security.authentication.type == schema.AuthenticationEnum.github: + config['security']['authentication'] = {'type': auth_provider.value} + if auth_provider == schema.AuthenticationEnum.github: if not disable_prompt: - config.security.authentication.config = schema.GithubConfig( - client_id=input("Github client_id: "), - client_secret=input("Github client_secret: "), - ) - elif config.security.authentication.type == schema.AuthenticationEnum.auth0: + config['security']['authentication']['config'] = { + 'client_id': input("Github client_id: "), + 'client_secret': input("Github client_secret: "), + } + elif auth_provider == schema.AuthenticationEnum.auth0: if auth_auto_provision: auth0_config = create_client(config.domain, config.project_name) - config.security.authentication.config = schema.Auth0Config(**auth0_config) + config['security']['authentication']['config'] = auth0_config else: - config.security.authentication.config = schema.Auth0Config( - client_id=input("Auth0 client_id: "), - client_secret=input("Auth0 client_secret: "), - auth0_subdomain=input("Auth0 subdomain: "), - ) - - if config.provider == schema.ProviderEnum.do: - config.theme.jupyterhub.hub_subtitle = f"{WELCOME_HEADER_TEXT} on Digital Ocean" - elif config.provider == schema.ProviderEnum.gcp: - config.theme.jupyterhub.hub_subtitle = ( + config['security']['authentication']['config'] = { + 'client_id': input("Auth0 client_id: "), + 'client_secret': input("Auth0 client_secret: "), + 'auth0_subdomain': input("Auth0 subdomain: "), + } + + if cloud_provider == schema.ProviderEnum.do: + config['theme']['jupyterhub']['hub_subtitle'] = f"{WELCOME_HEADER_TEXT} on Digital Ocean" + elif cloud_provider == schema.ProviderEnum.gcp: + config['theme']['jupyterhub']['hub_subtitle'] = ( f"{WELCOME_HEADER_TEXT} on Google Cloud Platform" ) if "PROJECT_ID" in os.environ: - config.google_cloud_platform.project = os.environ["PROJECT_ID"] + config['google_cloud_platform'] = {'project': os.environ["PROJECT_ID"]} elif not disable_prompt: - config.google_cloud_platform.project = input( + config['google_cloud_platform'] = {'project': input( "Enter Google Cloud Platform Project ID: " - ) - elif config.provider == schema.ProviderEnum.azure: - config.theme.jupyterhub.hub_subtitle = f"{WELCOME_HEADER_TEXT} on Azure" - elif config.provider == schema.ProviderEnum.aws: - config.theme.jupyterhub.hub_subtitle = ( + )} + elif cloud_provider == schema.ProviderEnum.azure: + config['theme']['jupyterhub']['hub_subtitle'] = f"{WELCOME_HEADER_TEXT} on Azure" + elif cloud_provider == schema.ProviderEnum.aws: + config['theme']['jupyterhub']['hub_subtitle'] = ( f"{WELCOME_HEADER_TEXT} on Amazon Web Services" ) - elif cloud_provider == "existing": - config.theme.jupyterhub.hub_subtitle = WELCOME_HEADER_TEXT - elif cloud_provider == "local": - config.theme.jupyterhub.hub_subtitle = WELCOME_HEADER_TEXT + elif cloud_provider == schema.ProviderEnum.existing: + config['theme']['jupyterhub']['hub_subtitle'] = WELCOME_HEADER_TEXT + elif cloud_provider == schema.ProviderEnum.local: + config['theme']['jupyterhub']['hub_subtitle'] = WELCOME_HEADER_TEXT if ssl_cert_email: - config.certificate.type = schema.CertificateEnum.letsencrypt - config.certificate.acme_email = ssl_cert_email + config['certificate'] = {'type': schema.CertificateEnum.letsencrypt.value} + config['certificate']['acme_email'] = ssl_cert_email if repository_auto_provision: GITHUB_REGEX = "(https://)?github.com/([^/]+)/([^/]+)/?" if re.search(GITHUB_REGEX, repository): match = re.search(GITHUB_REGEX, repository) git_repository = github_auto_provision( - config, match.group(2), match.group(3) + schema.Main.parse_obj(config), match.group(2), match.group(3) ) git_repository_initialize(git_repository) else: diff --git a/src/_nebari/provider/cloud/commons.py b/src/_nebari/provider/cloud/commons.py index aef9a8c90..09a967e53 100644 --- a/src/_nebari/provider/cloud/commons.py +++ b/src/_nebari/provider/cloud/commons.py @@ -6,7 +6,7 @@ def filter_by_highest_supported_k8s_version(k8s_versions_list): filtered_k8s_versions_list = [] for k8s_version in k8s_versions_list: - version = re.search("(\d+)\.(\d+)\.(\d+)", k8s_version).groups() + version = tuple(filter(None, re.search("(\d+)\.(\d+)(?:\.(\d+))?", k8s_version).groups())) if version <= HIGHEST_SUPPORTED_K8S_VERSION: filtered_k8s_versions_list.append(k8s_version) return filtered_k8s_versions_list diff --git a/src/nebari/schema.py b/src/nebari/schema.py index af07add67..79f36e197 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -152,7 +152,7 @@ class HelmExtension(Base): repository: str chart: str version: str - overrides: typing.Optional[typing.Dict] + overrides: typing.Dict = {} # ============== Argo-Workflows ========= @@ -537,16 +537,16 @@ class AmazonWebServicesProvider(Base): existing_security_group_ids: str = None vpc_cidr_block: str = "10.10.0.0/16" - @validator("kubernetes_version") - def _validate_kubernetes_version(cls, value): + @root_validator + def _validate_kubernetes_version(cls, values): available_kubernetes_versions = amazon_web_services.kubernetes_versions() - if value is None: - value = available_kubernetes_versions[-1] - elif value not in available_kubernetes_versions: + if values["kubernetes_version"] is None: + values["kubernetes_version"] = available_kubernetes_versions[-1] + elif values["kubernetes_version"] not in available_kubernetes_versions: raise ValueError( - f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." + f"\nInvalid `kubernetes-version` provided: {values['kubernetes_version']}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." ) - return value + return values @root_validator def _validate_availability_zones(cls, values): @@ -638,7 +638,7 @@ class JupyterLabProfile(Base): access: AccessEnum = AccessEnum.all display_name: str description: str - default: typing.Optional[bool] + default: bool = False users: typing.Optional[typing.List[str]] groups: typing.Optional[typing.List[str]] kubespawner_override: typing.Optional[KubeSpawner] @@ -1120,10 +1120,13 @@ def read_configuration(config_filename: pathlib.Path, read_environment: bool = T return config -def write_configuration(config_filename: pathlib.Path, config: Main, mode: str = "w"): +def write_configuration(config_filename: pathlib.Path, config: typing.Union[Main, typing.Dict], mode: str = "w"): yaml = YAML() yaml.preserve_quotes = True yaml.default_flow_style = False with config_filename.open(mode) as f: - yaml.dump(config.dict(), f) + if isinstance(config, Main): + yaml.dump(config.dict(), f) + else: + yaml.dump(config, f) From 87c85b5d0aed539a8f25d99a8873dcba55b90222 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Jun 2023 19:57:38 +0000 Subject: [PATCH 054/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/initialize.py | 80 +++++++++++++++------------ src/_nebari/provider/cloud/commons.py | 4 +- src/nebari/schema.py | 6 +- 3 files changed, 52 insertions(+), 38 deletions(-) diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index dcb9677db..4625de658 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -34,78 +34,86 @@ def render_config( ssl_cert_email: str = None, ): config = { - 'provider': cloud_provider.value, - 'namespace': namespace, - 'nebari_version': __version__, + "provider": cloud_provider.value, + "namespace": namespace, + "nebari_version": __version__, } if project_name is None and not disable_prompt: project_name = input("Provide project name: ") - config['project_name'] = project_name + config["project_name"] = project_name if nebari_domain is None and not disable_prompt: nebari_domain = input("Provide domain: ") - config['domain'] = nebari_domain + config["domain"] = nebari_domain - config['ci_cd'] = {'type': ci_provider.value} - config['terraform_state'] = {'type': terraform_state.value} + config["ci_cd"] = {"type": ci_provider.value} + config["terraform_state"] = {"type": terraform_state.value} # Save default password to file default_password_filename = os.path.join( tempfile.gettempdir(), "NEBARI_DEFAULT_PASSWORD" ) - config['security'] = {'keycloak': {'initial_root_password': schema.random_secure_string(length=32)}} + config["security"] = { + "keycloak": {"initial_root_password": schema.random_secure_string(length=32)} + } with open(default_password_filename, "w") as f: - f.write(config['security']['keycloak']['initial_root_password']) + f.write(config["security"]["keycloak"]["initial_root_password"]) os.chmod(default_password_filename, 0o700) - config['theme'] = {'jupyterhub': {'hub_title': f"Nebari - { project_name }"}} - config['theme']['jupyterhub']['welcome'] = """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""" + config["theme"] = {"jupyterhub": {"hub_title": f"Nebari - { project_name }"}} + config["theme"]["jupyterhub"][ + "welcome" + ] = """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""" - config['security']['authentication'] = {'type': auth_provider.value} + config["security"]["authentication"] = {"type": auth_provider.value} if auth_provider == schema.AuthenticationEnum.github: if not disable_prompt: - config['security']['authentication']['config'] = { - 'client_id': input("Github client_id: "), - 'client_secret': input("Github client_secret: "), + config["security"]["authentication"]["config"] = { + "client_id": input("Github client_id: "), + "client_secret": input("Github client_secret: "), } elif auth_provider == schema.AuthenticationEnum.auth0: if auth_auto_provision: auth0_config = create_client(config.domain, config.project_name) - config['security']['authentication']['config'] = auth0_config + config["security"]["authentication"]["config"] = auth0_config else: - config['security']['authentication']['config'] = { - 'client_id': input("Auth0 client_id: "), - 'client_secret': input("Auth0 client_secret: "), - 'auth0_subdomain': input("Auth0 subdomain: "), + config["security"]["authentication"]["config"] = { + "client_id": input("Auth0 client_id: "), + "client_secret": input("Auth0 client_secret: "), + "auth0_subdomain": input("Auth0 subdomain: "), } if cloud_provider == schema.ProviderEnum.do: - config['theme']['jupyterhub']['hub_subtitle'] = f"{WELCOME_HEADER_TEXT} on Digital Ocean" + config["theme"]["jupyterhub"][ + "hub_subtitle" + ] = f"{WELCOME_HEADER_TEXT} on Digital Ocean" elif cloud_provider == schema.ProviderEnum.gcp: - config['theme']['jupyterhub']['hub_subtitle'] = ( - f"{WELCOME_HEADER_TEXT} on Google Cloud Platform" - ) + config["theme"]["jupyterhub"][ + "hub_subtitle" + ] = f"{WELCOME_HEADER_TEXT} on Google Cloud Platform" if "PROJECT_ID" in os.environ: - config['google_cloud_platform'] = {'project': os.environ["PROJECT_ID"]} + config["google_cloud_platform"] = {"project": os.environ["PROJECT_ID"]} elif not disable_prompt: - config['google_cloud_platform'] = {'project': input( - "Enter Google Cloud Platform Project ID: " - )} + config["google_cloud_platform"] = { + "project": input("Enter Google Cloud Platform Project ID: ") + } elif cloud_provider == schema.ProviderEnum.azure: - config['theme']['jupyterhub']['hub_subtitle'] = f"{WELCOME_HEADER_TEXT} on Azure" + config["theme"]["jupyterhub"][ + "hub_subtitle" + ] = f"{WELCOME_HEADER_TEXT} on Azure" elif cloud_provider == schema.ProviderEnum.aws: - config['theme']['jupyterhub']['hub_subtitle'] = ( - f"{WELCOME_HEADER_TEXT} on Amazon Web Services" - ) + config["theme"]["jupyterhub"][ + "hub_subtitle" + ] = f"{WELCOME_HEADER_TEXT} on Amazon Web Services" elif cloud_provider == schema.ProviderEnum.existing: - config['theme']['jupyterhub']['hub_subtitle'] = WELCOME_HEADER_TEXT + config["theme"]["jupyterhub"]["hub_subtitle"] = WELCOME_HEADER_TEXT elif cloud_provider == schema.ProviderEnum.local: - config['theme']['jupyterhub']['hub_subtitle'] = WELCOME_HEADER_TEXT + config["theme"]["jupyterhub"]["hub_subtitle"] = WELCOME_HEADER_TEXT if ssl_cert_email: - config['certificate'] = {'type': schema.CertificateEnum.letsencrypt.value} - config['certificate']['acme_email'] = ssl_cert_email + config["certificate"] = {"type": schema.CertificateEnum.letsencrypt.value} + config["certificate"]["acme_email"] = ssl_cert_email if repository_auto_provision: GITHUB_REGEX = "(https://)?github.com/([^/]+)/([^/]+)/?" diff --git a/src/_nebari/provider/cloud/commons.py b/src/_nebari/provider/cloud/commons.py index 09a967e53..a12dbec8b 100644 --- a/src/_nebari/provider/cloud/commons.py +++ b/src/_nebari/provider/cloud/commons.py @@ -6,7 +6,9 @@ def filter_by_highest_supported_k8s_version(k8s_versions_list): filtered_k8s_versions_list = [] for k8s_version in k8s_versions_list: - version = tuple(filter(None, re.search("(\d+)\.(\d+)(?:\.(\d+))?", k8s_version).groups())) + version = tuple( + filter(None, re.search("(\d+)\.(\d+)(?:\.(\d+))?", k8s_version).groups()) + ) if version <= HIGHEST_SUPPORTED_K8S_VERSION: filtered_k8s_versions_list.append(k8s_version) return filtered_k8s_versions_list diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 79f36e197..2cb2eeca7 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -1120,7 +1120,11 @@ def read_configuration(config_filename: pathlib.Path, read_environment: bool = T return config -def write_configuration(config_filename: pathlib.Path, config: typing.Union[Main, typing.Dict], mode: str = "w"): +def write_configuration( + config_filename: pathlib.Path, + config: typing.Union[Main, typing.Dict], + mode: str = "w", +): yaml = YAML() yaml.preserve_quotes = True yaml.default_flow_style = False From 60bca92587a6360d5988b685e8464d624d11b116 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 16:21:03 -0400 Subject: [PATCH 055/147] Integrating PR https://github.com/nebari-dev/nebari/pull/1803 --- src/_nebari/initialize.py | 7 ++--- src/_nebari/subcommands/init.py | 54 ++++++++++++++++++--------------- src/nebari/schema.py | 2 +- 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index 4625de658..8e730004e 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -20,7 +20,7 @@ def render_config( project_name: str, - nebari_domain: str, + nebari_domain: str = None, cloud_provider: schema.ProviderEnum = schema.ProviderEnum.local, ci_provider: schema.CiEnum = schema.CiEnum.none, repository: str = None, @@ -43,9 +43,8 @@ def render_config( project_name = input("Provide project name: ") config["project_name"] = project_name - if nebari_domain is None and not disable_prompt: - nebari_domain = input("Provide domain: ") - config["domain"] = nebari_domain + if nebari_domain is not None: + config["domain"] = nebari_domain config["ci_cd"] = {"type": ci_provider.value} config["terraform_state"] = {"type": terraform_state.value} diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index a6d3f1435..3efbde8a8 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -1,6 +1,7 @@ import os import pathlib import re +import typing import questionary import rich @@ -288,7 +289,7 @@ def init( "-p", callback=check_project_name, ), - domain_name: str = typer.Option( + domain_name: typing.Optional[str] = typer.Option( ..., "--domain-name", "--domain", @@ -449,15 +450,17 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): # DOMAIN NAME rich.print( ( - "\n\n 🪴 Great! Now you need to provide a valid domain name (i.e. the URL) to access your Nebri instance. " - "This should be a domain that you own.\n\n" + "\n\n 🪴 Great! Now you can provide a valid domain name (i.e. the URL) to access your Nebri instance. " + "This should be a domain that you own. Default if unspecified is the IP of the load balancer.\n\n" ) ) - inputs.domain_name = questionary.text( - "What domain name would you like to use?", - qmark=qmark, - validate=lambda text: True if len(text) > 0 else "Please enter a value", - ).unsafe_ask() + inputs.domain_name = ( + questionary.text( + "What domain name would you like to use?", + qmark=qmark, + ).unsafe_ask() + or None + ) # AUTH PROVIDER rich.print( @@ -553,27 +556,28 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): inputs.ci_provider = CiEnum.gitlab_ci.value.lower() # SSL CERTIFICATE - rich.print( - ( - "\n\n 🪴 This next section is [italic]optional[/italic] but recommended. If you want your Nebari domain to use a Let's Encrypt SSL certificate, " - "all we need is an email address from you.\n\n" + if inputs.domain_name: + rich.print( + ( + "\n\n 🪴 This next section is [italic]optional[/italic] but recommended. If you want your Nebari domain to use a Let's Encrypt SSL certificate, " + "all we need is an email address from you.\n\n" + ) ) - ) - ssl_cert = questionary.confirm( - "Would you like to add a Let's Encrypt SSL certificate to your domain?", - default=False, - qmark=qmark, - auto_enter=False, - ).unsafe_ask() - - if ssl_cert: - inputs.ssl_cert_email = questionary.text( - "Which email address should Let's Encrypt associate the certificate with?", + ssl_cert = questionary.confirm( + "Would you like to add a Let's Encrypt SSL certificate to your domain?", + default=False, qmark=qmark, + auto_enter=False, ).unsafe_ask() - if not disable_checks: - check_ssl_cert_email(ctx, ssl_cert_email=inputs.ssl_cert_email) + if ssl_cert: + inputs.ssl_cert_email = questionary.text( + "Which email address should Let's Encrypt associate the certificate with?", + qmark=qmark, + ).unsafe_ask() + + if not disable_checks: + check_ssl_cert_email(ctx, ssl_cert_email=inputs.ssl_cert_email) # ADVANCED FEATURES rich.print( diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 2cb2eeca7..0c3d6538f 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -845,7 +845,7 @@ def project_name_convention(value: typing.Any, values): class InitInputs(Base): cloud_provider: ProviderEnum = ProviderEnum.local project_name: str = "" - domain_name: str = "" + domain_name: typing.Optional[str] = None namespace: typing.Optional[letter_dash_underscore_pydantic] = "dev" auth_provider: AuthenticationEnum = AuthenticationEnum.password auth_auto_provision: bool = False From b1213fff5686d925985db4afa604e52b78e0e603 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 16:28:36 -0400 Subject: [PATCH 056/147] Fixup for test-nebari-provider --- .github/workflows/test-provider.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-provider.yaml b/.github/workflows/test-provider.yaml index f56717abd..950ae4fed 100644 --- a/.github/workflows/test-provider.yaml +++ b/.github/workflows/test-provider.yaml @@ -123,7 +123,7 @@ jobs: - name: Nebari Initialize run: | - nebari init "${{ matrix.provider }}" --project "TestProvider" --domain "${{ matrix.provider }}.nebari.dev" --auth-provider github --disable-prompt --ci-provider ${{ matrix.cicd }} + nebari init "${{ matrix.provider }}" --project "TestProvider" --domain "${{ matrix.provider }}.nebari.dev" --auth-provider GitHub --disable-prompt --ci-provider ${{ matrix.cicd }} cat "nebari-config.yaml" - name: Nebari Render From 85548e0532535604c424f4443cc937f94e8dddc6 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 16:40:58 -0400 Subject: [PATCH 057/147] Fixup for nebari provider --- src/_nebari/subcommands/init.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index 3efbde8a8..e40aa8130 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -107,7 +107,7 @@ def check_ssl_cert_email(ctx: typer.Context, ssl_cert_email: str): def check_auth_provider_creds(ctx: typer.Context, auth_provider: str): """Validate the the necessary auth provider credentials have been set as environment variables.""" if ctx.params.get("disable_prompt"): - return auth_provider + return auth_provider.lower() auth_provider = auth_provider.lower() @@ -159,10 +159,11 @@ def check_auth_provider_creds(ctx: typer.Context, auth_provider: str): return auth_provider -def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: str): +def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: schema.ProviderEnum): """Validate that the necessary cloud credentials have been set as environment variables.""" + if ctx.params.get("disable_prompt"): - return cloud_provider + return cloud_provider.lower() cloud_provider = cloud_provider.lower() From f2d8e20e41200a46227f15f4210fd0314925dd33 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 16:47:28 -0400 Subject: [PATCH 058/147] Don't require the github client id/secret --- .github/workflows/test-provider.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-provider.yaml b/.github/workflows/test-provider.yaml index 950ae4fed..60649f79e 100644 --- a/.github/workflows/test-provider.yaml +++ b/.github/workflows/test-provider.yaml @@ -123,7 +123,7 @@ jobs: - name: Nebari Initialize run: | - nebari init "${{ matrix.provider }}" --project "TestProvider" --domain "${{ matrix.provider }}.nebari.dev" --auth-provider GitHub --disable-prompt --ci-provider ${{ matrix.cicd }} + nebari init "${{ matrix.provider }}" --project "TestProvider" --domain "${{ matrix.provider }}.nebari.dev" --auth-provider password --disable-prompt --ci-provider ${{ matrix.cicd }} cat "nebari-config.yaml" - name: Nebari Render From f77035556490044502e04f9e0acdf9ea56f38e1c Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 16:50:02 -0400 Subject: [PATCH 059/147] Missing schema import --- src/_nebari/provider/cicd/github.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/_nebari/provider/cicd/github.py b/src/_nebari/provider/cicd/github.py index 127df352c..51fba2643 100644 --- a/src/_nebari/provider/cicd/github.py +++ b/src/_nebari/provider/cicd/github.py @@ -8,6 +8,7 @@ from _nebari.constants import LATEST_SUPPORTED_PYTHON_VERSION from _nebari.provider.cicd.common import pip_install_nebari +from nebari import schema GITHUB_BASE_URL = "https://api.github.com/" From 94ddf58d40d5d839cd43a4b8d626980b850d180a Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 16:55:26 -0400 Subject: [PATCH 060/147] Fixing indexing to use pydantic schema --- src/_nebari/provider/cicd/github.py | 6 +++--- src/_nebari/provider/cicd/gitlab.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/_nebari/provider/cicd/github.py b/src/_nebari/provider/cicd/github.py index 51fba2643..4476d5e86 100644 --- a/src/_nebari/provider/cicd/github.py +++ b/src/_nebari/provider/cicd/github.py @@ -97,7 +97,7 @@ def create_repository(owner, repo, description, homepage, private=True): return f"git@github.com:{owner}/{repo}.git" -def gha_env_vars(config): +def gha_env_vars(config: schema.Main): env_vars = { "GITHUB_TOKEN": "${{ secrets.GITHUB_TOKEN }}", } @@ -241,7 +241,7 @@ def gen_nebari_ops(config): step3 = install_nebari_step(config.nebari_version) gha_steps = [step1, step2, step3] - for step in config["ci_cd"].get("before_script", []): + for step in config.ci_cd.before_script: gha_steps.append(GHA_job_step(**step)) step4 = GHA_job_step( @@ -268,7 +268,7 @@ def gen_nebari_ops(config): if config.ci_cd.commit_render: gha_steps.append(step5) - for step in config["ci_cd"].get("after_script", []): + for step in config.ci_cd.after_script: gha_steps.append(GHA_job_step(**step)) job1 = GHA_job_id( diff --git a/src/_nebari/provider/cicd/gitlab.py b/src/_nebari/provider/cicd/gitlab.py index f7f3b1201..c33013822 100644 --- a/src/_nebari/provider/cicd/gitlab.py +++ b/src/_nebari/provider/cicd/gitlab.py @@ -53,7 +53,7 @@ def gen_gitlab_ci(config): "git config user.name 'gitlab ci'", "git add .", "git diff --quiet && git diff --staged --quiet || (git commit -m '${COMMIT_MSG}'", - f"git push origin {branch})", + f"git push origin {config.ci_cd.branch})", ] if config.ci_cd.commit_render: From a0e833f75010c0f569ccab65ceffe7580b0da89e Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 16:59:17 -0400 Subject: [PATCH 061/147] Undefined variables --- src/_nebari/provider/cicd/github.py | 2 +- src/_nebari/provider/cicd/gitlab.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_nebari/provider/cicd/github.py b/src/_nebari/provider/cicd/github.py index 4476d5e86..b02c0bf32 100644 --- a/src/_nebari/provider/cicd/github.py +++ b/src/_nebari/provider/cicd/github.py @@ -257,7 +257,7 @@ def gen_nebari_ops(config): "git config user.name 'github action' ; " "git add ./.gitignore ./.github ./stages; " "git diff --quiet && git diff --staged --quiet || (git commit -m '${{ env.COMMIT_MSG }}') ; " - f"git push origin {branch}" + f"git push origin {config.ci_cd.branch}" ), env={ "COMMIT_MSG": GHA_job_steps_extras( diff --git a/src/_nebari/provider/cicd/gitlab.py b/src/_nebari/provider/cicd/gitlab.py index c33013822..e2d02b388 100644 --- a/src/_nebari/provider/cicd/gitlab.py +++ b/src/_nebari/provider/cicd/gitlab.py @@ -61,7 +61,7 @@ def gen_gitlab_ci(config): rules = [ GLCI_rules( - if_=f"$CI_COMMIT_BRANCH == '{branch}'", + if_=f"$CI_COMMIT_BRANCH == '{config.ci_cd.branch}'", changes=["nebari-config.yaml"], ) ] From 867ddfda0f97035472d0f6459cf1dc10def23de8 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 17:02:26 -0400 Subject: [PATCH 062/147] Yaml export --- src/_nebari/stages/bootstrap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index eb25a2536..226da9e60 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -67,7 +67,7 @@ def render(self) -> Dict[str, str]: exclude_unset=True, exclude_defaults=True, ) - workflow_yaml = yaml.dump( + workflow_yaml = schema.yaml.dump( json.loads(workflow_json), sort_keys=False, indent=2 ) contents.update({fn: workflow_yaml}) From 506f6f323e0b54e23f139ca16f895dbf8e86da04 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 17:05:48 -0400 Subject: [PATCH 063/147] Missing import --- src/_nebari/stages/bootstrap/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index 226da9e60..50a25caff 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -1,3 +1,4 @@ +import json from inspect import cleandoc from typing import Any, Dict, List From e71f3cea8da02e0c878fa341576146491456b03c Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 17:09:38 -0400 Subject: [PATCH 064/147] Expect pydantic schema --- src/_nebari/stages/bootstrap/__init__.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index 50a25caff..ce0f9c555 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -1,4 +1,3 @@ -import json from inspect import cleandoc from typing import Any, Dict, List @@ -62,14 +61,8 @@ def render(self) -> Dict[str, str]: contents = {} if self.config.ci_cd.type != schema.CiEnum.none: for fn, workflow in gen_cicd(self.config).items(): - workflow_json = workflow.json( - indent=2, - by_alias=True, - exclude_unset=True, - exclude_defaults=True, - ) workflow_yaml = schema.yaml.dump( - json.loads(workflow_json), sort_keys=False, indent=2 + workflow.dict(by_alias=True, exclude_unset=True, exclude_defaults=True), ) contents.update({fn: workflow_yaml}) From 9ab3342ca1a88bfb0cdbad0c48ed9613309b69fd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Jun 2023 21:10:04 +0000 Subject: [PATCH 065/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/stages/bootstrap/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index ce0f9c555..fb49d2adc 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -62,7 +62,9 @@ def render(self) -> Dict[str, str]: if self.config.ci_cd.type != schema.CiEnum.none: for fn, workflow in gen_cicd(self.config).items(): workflow_yaml = schema.yaml.dump( - workflow.dict(by_alias=True, exclude_unset=True, exclude_defaults=True), + workflow.dict( + by_alias=True, exclude_unset=True, exclude_defaults=True + ), ) contents.update({fn: workflow_yaml}) From 8e0c12d7be84651b67591fde5a5f7d3655e5b5f8 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 17:16:35 -0400 Subject: [PATCH 066/147] Dump to stringio --- src/_nebari/stages/bootstrap/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index fb49d2adc..b22896aa5 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -1,3 +1,4 @@ +import io from inspect import cleandoc from typing import Any, Dict, List @@ -61,12 +62,11 @@ def render(self) -> Dict[str, str]: contents = {} if self.config.ci_cd.type != schema.CiEnum.none: for fn, workflow in gen_cicd(self.config).items(): + stream = io.StringIO() workflow_yaml = schema.yaml.dump( - workflow.dict( - by_alias=True, exclude_unset=True, exclude_defaults=True - ), + workflow.dict(by_alias=True, exclude_unset=True, exclude_defaults=True), stream ) - contents.update({fn: workflow_yaml}) + contents.update({fn: stream.getvalue()}) contents.update(gen_gitignore()) return contents From 9151463f9074861a57fdd387ab7280caa30fbb4d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Jun 2023 21:17:25 +0000 Subject: [PATCH 067/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/stages/bootstrap/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index b22896aa5..a78dc326e 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -63,8 +63,11 @@ def render(self) -> Dict[str, str]: if self.config.ci_cd.type != schema.CiEnum.none: for fn, workflow in gen_cicd(self.config).items(): stream = io.StringIO() - workflow_yaml = schema.yaml.dump( - workflow.dict(by_alias=True, exclude_unset=True, exclude_defaults=True), stream + schema.yaml.dump( + workflow.dict( + by_alias=True, exclude_unset=True, exclude_defaults=True + ), + stream, ) contents.update({fn: stream.getvalue()}) From 4fd18a2adc2733b679bdb8accf61b419be05a242 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 17:31:18 -0400 Subject: [PATCH 068/147] Wrong path for render_template --- .github/actions/publish-from-template/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/publish-from-template/action.yml b/.github/actions/publish-from-template/action.yml index 1e5117c47..9c209e3e8 100644 --- a/.github/actions/publish-from-template/action.yml +++ b/.github/actions/publish-from-template/action.yml @@ -16,7 +16,7 @@ runs: shell: bash env: ${{ env }} run: - python .github/actions/publish-from-template/render_template.py ${{ + python ${{ github.action_path }}/render_template.py ${{ inputs.filename }} - uses: JasonEtco/create-an-issue@v2 From bbd8b3a90f6e7974bf4a4df595c74ee00623eb1a Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 17:35:28 -0400 Subject: [PATCH 069/147] Fixing broken path in action --- .github/actions/publish-from-template/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/publish-from-template/action.yml b/.github/actions/publish-from-template/action.yml index 9c209e3e8..cde326890 100644 --- a/.github/actions/publish-from-template/action.yml +++ b/.github/actions/publish-from-template/action.yml @@ -16,8 +16,7 @@ runs: shell: bash env: ${{ env }} run: - python ${{ github.action_path }}/render_template.py ${{ - inputs.filename }} + python ${{ github.action_path }}/publish-from-template.py ${{inputs.filename }} - uses: JasonEtco/create-an-issue@v2 # Only render template and create an issue in case the workflow is a scheduled one From 1307a59486b0672ae977662e9e08090c7ac354b4 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 21 Jun 2023 17:48:53 -0400 Subject: [PATCH 070/147] Fixing the cli tests --- tests/tests_unit/test_cli.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/tests/tests_unit/test_cli.py b/tests/tests_unit/test_cli.py index a45072d70..c3b4372f8 100644 --- a/tests/tests_unit/test_cli.py +++ b/tests/tests_unit/test_cli.py @@ -2,7 +2,7 @@ import pytest -from _nebari.schema import InitInputs +from nebari import schema from _nebari.utils import load_yaml PROJECT_NAME = "clitest" @@ -27,7 +27,7 @@ def test_nebari_init(tmp_path, namespace, auth_provider, ci_provider, ssl_cert_e "--disable-prompt", ] - default_values = InitInputs() + default_values = schema.InitInputs() if namespace: command.append(f"--namespace={namespace}") @@ -48,23 +48,12 @@ def test_nebari_init(tmp_path, namespace, auth_provider, ci_provider, ssl_cert_e subprocess.run(command, cwd=tmp_path, check=True) - config = load_yaml(tmp_path / "nebari-config.yaml") + config = schema.read_configuration(tmp_path / "nebari-config.yaml") - assert config.get("namespace") == namespace - assert ( - config.get("security", {}).get("authentication", {}).get("type").lower() - == auth_provider - ) - ci_cd = config.get("ci_cd", None) - if ci_cd: - assert ci_cd.get("type", {}) == ci_provider - else: - assert ci_cd == ci_provider - acme_email = config.get("certificate", None) - if acme_email: - assert acme_email.get("acme_email") == ssl_cert_email - else: - assert acme_email == ssl_cert_email + assert config.namespace == namespace + assert config.security.authentication.type.lower() == auth_provider + assert config.ci_cd.type == ci_provider + assert config.certificate.acme_email == ssl_cert_email def test_python_invocation(): From 556467488918f699d043db296dbffb553d937936 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Jun 2023 21:49:07 +0000 Subject: [PATCH 071/147] [pre-commit.ci] Apply automatic pre-commit fixes --- tests/tests_unit/test_cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/tests_unit/test_cli.py b/tests/tests_unit/test_cli.py index c3b4372f8..ecd38167a 100644 --- a/tests/tests_unit/test_cli.py +++ b/tests/tests_unit/test_cli.py @@ -3,7 +3,6 @@ import pytest from nebari import schema -from _nebari.utils import load_yaml PROJECT_NAME = "clitest" DOMAIN_NAME = "clitest.dev" From a88e6f1d9591a435c0cf4fd2b447ca9893bb4a33 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 22 Jun 2023 14:57:47 -0400 Subject: [PATCH 072/147] Adding controls around stages exclusion --- src/_nebari/cli.py | 54 +++++++++++++++--- src/_nebari/deploy.py | 88 +++++++++++------------------- src/_nebari/destroy.py | 38 ++++++------- src/_nebari/render.py | 7 +-- src/_nebari/stages/base.py | 32 ++++++++++- src/_nebari/subcommands/deploy.py | 4 +- src/_nebari/subcommands/destroy.py | 5 +- src/_nebari/subcommands/info.py | 42 ++++++++++++++ src/_nebari/subcommands/render.py | 3 +- src/nebari/plugins.py | 36 +++++++----- src/nebari/schema.py | 4 ++ 11 files changed, 205 insertions(+), 108 deletions(-) create mode 100644 src/_nebari/subcommands/info.py diff --git a/src/_nebari/cli.py b/src/_nebari/cli.py index 608dd63c5..bfb9a1d5f 100644 --- a/src/_nebari/cli.py +++ b/src/_nebari/cli.py @@ -1,11 +1,15 @@ import importlib import typing +import pathlib +import sys +import os import typer from typer.core import TyperGroup from _nebari.version import __version__ -from nebari.plugins import pm +from nebari import schema +from nebari.plugins import pm, load_plugins class OrderCommands(TyperGroup): @@ -20,9 +24,25 @@ def version_callback(value: bool): raise typer.Exit() -def import_module(module: str): - if module is not None: - importlib.__import__(module) +def exclude_stages(ctx: typer.Context, stages: typing.List[str]): + ctx.ensure_object(schema.CLIContext) + ctx.obj.excluded_stages = stages + return stages + + +def exclude_default_stages(ctx: typer.Context, exclude_default_stages: bool): + ctx.ensure_object(schema.CLIContext) + ctx.obj.exclude_default_stages = exclude_default_stages + return exclude_default_stages + + +def import_plugin(plugins: typing.List[str]): + try: + load_plugins(plugins) + except ModuleNotFoundError as e: + typer.echo("ERROR: Python module {e.name} not found. Make sure that the module is in your python path {sys.path}") + typer.Exit() + return plugins def create_cli(): @@ -36,7 +56,7 @@ def create_cli(): context_settings={"help_option_names": ["-h", "--help"]}, ) - @app.callback(invoke_without_command=True) + @app.callback() def common( ctx: typer.Context, version: bool = typer.Option( @@ -46,11 +66,29 @@ def common( help="Nebari version number", callback=version_callback, ), - import_module: typing.Optional[str] = typer.Option( - None, "--import-module", help="Import nebari module", callback=import_module + plugins: typing.List[str] = typer.Option( + [], "--import-plugin", help="Import nebari plugin", ), + excluded_stages: typing.List[str] = typer.Option( + [], "--exclude-stage", help="Exclude nebari stage(s) by name or regex", + ), + exclude_default_stages: bool = typer.Option( + False, '--exclude-default-stages', help="Exclude default nebari included stages", + ) ): - pass + try: + load_plugins(plugins) + except ModuleNotFoundError as e: + typer.echo("ERROR: Python module {e.name} not found. Make sure that the module is in your python path {sys.path}") + typer.Exit() + + from _nebari.stages.base import get_available_stages + ctx.ensure_object(schema.CLIContext) + ctx.obj.stages = get_available_stages( + exclude_default_stages=exclude_default_stages, + exclude_stages=excluded_stages + ) + pm.hook.nebari_subcommand(cli=app) diff --git a/src/_nebari/deploy.py b/src/_nebari/deploy.py index aff95693f..0c4f52308 100644 --- a/src/_nebari/deploy.py +++ b/src/_nebari/deploy.py @@ -3,59 +3,23 @@ import pathlib import subprocess import textwrap +from typing import List from _nebari.stages.base import get_available_stages from _nebari.utils import timer -from nebari import schema +from nebari import schema, hookspecs logger = logging.getLogger(__name__) -def guided_install( - config: schema.Main, - dns_provider, - dns_auto_provision, - disable_prompt=False, - disable_checks=False, - skip_remote_state_provision=False, -): - stage_outputs = {} - with contextlib.ExitStack() as stack: - for stage in get_available_stages(): - s = stage(output_directory=pathlib.Path("."), config=config) - stack.enter_context(s.deploy(stage_outputs)) - - if not disable_checks: - s.check(stage_outputs) - print("Nebari deployed successfully") - - print("Services:") - for service_name, service in stage_outputs["stages/07-kubernetes-services"][ - "service_urls" - ]["value"].items(): - print(f" - {service_name} -> {service['url']}") - - print( - f"Kubernetes kubeconfig located at file://{stage_outputs['stages/02-infrastructure']['kubeconfig_filename']['value']}" - ) - username = "root" - password = config.security.keycloak.initial_root_password - if password: - print(f"Kubecloak master realm username={username} password={password}") - - print( - "Additional administration docs can be found at https://docs.nebari.dev/en/stable/source/admin_guide/" - ) - return stage_outputs - - def deploy_configuration( config: schema.Main, + stages: List[hookspecs.NebariStage], dns_provider, dns_auto_provision, - disable_prompt, - disable_checks, - skip_remote_state_provision, + disable_prompt: bool = False, + disable_checks: bool = False, + skip_remote_state_provision: bool = False, ): if config.prevent_deploy: raise ValueError( @@ -87,16 +51,30 @@ def deploy_configuration( ) with timer(logger, "deploying Nebari"): - try: - return guided_install( - config, - dns_provider, - dns_auto_provision, - disable_prompt, - disable_checks, - skip_remote_state_provision, - ) - except subprocess.CalledProcessError as e: - logger.error("subprocess command failed") - logger.error(e.output) - raise e + stage_outputs = {} + with contextlib.ExitStack() as stack: + for stage in stages: + s = stage(output_directory=pathlib.Path("."), config=config) + stack.enter_context(s.deploy(stage_outputs)) + + if not disable_checks: + s.check(stage_outputs) + print("Nebari deployed successfully") + + print("Services:") + for service_name, service in stage_outputs["stages/07-kubernetes-services"][ + "service_urls" + ]["value"].items(): + print(f" - {service_name} -> {service['url']}") + + print( + f"Kubernetes kubeconfig located at file://{stage_outputs['stages/02-infrastructure']['kubeconfig_filename']['value']}" + ) + username = "root" + password = config.security.keycloak.initial_root_password + if password: + print(f"Kubecloak master realm username={username} password={password}") + + print( + "Additional administration docs can be found at https://docs.nebari.dev/en/stable/source/admin_guide/" + ) diff --git a/src/_nebari/destroy.py b/src/_nebari/destroy.py index 06c133fa2..6768a5b4f 100644 --- a/src/_nebari/destroy.py +++ b/src/_nebari/destroy.py @@ -1,38 +1,36 @@ import contextlib import logging import pathlib +from typing import List from _nebari.stages.base import get_available_stages from _nebari.utils import timer -from nebari import schema +from nebari import schema, hookspecs logger = logging.getLogger(__name__) -def destroy_stages(config: schema.Main): - stage_outputs = {} - status = {} - with contextlib.ExitStack() as stack: - for stage in get_available_stages(): - try: - s = stage(output_directory=pathlib.Path("."), config=config) - stack.enter_context(s.destroy(stage_outputs, status)) - except Exception as e: - status[s.name] = False - print( - f"ERROR: stage={s.name} failed due to {e}. Due to stages depending on each other we can only destroy stages that occur before this stage" - ) - break - return status - - -def destroy_configuration(config: schema.Main): +def destroy_configuration(config: schema.Main, stages: List[hookspecs.NebariStage]): logger.info( """Removing all infrastructure, your local files will still remain, you can use 'nebari deploy' to re-install infrastructure using same config file\n""" ) + + stage_outputs = {} + status = {} + with timer(logger, "destroying Nebari"): - status = destroy_stages(config) + with contextlib.ExitStack() as stack: + for stage in stages: + try: + s = stage(output_directory=pathlib.Path("."), config=config) + stack.enter_context(s.destroy(stage_outputs, status)) + except Exception as e: + status[s.name] = False + print( + f"ERROR: stage={s.name} failed due to {e}. Due to stages depending on each other we can only destroy stages that occur before this stage" + ) + break for stage_name, success in status.items(): if not success: diff --git a/src/_nebari/render.py b/src/_nebari/render.py index 22782133a..5419f949d 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -9,11 +9,10 @@ from rich.table import Table from _nebari.deprecate import DEPRECATED_FILE_PATHS -from _nebari.stages.base import get_available_stages -from nebari import schema +from nebari import schema, hookspecs -def render_template(output_directory: pathlib.Path, config: schema.Main, dry_run=False): +def render_template(output_directory: pathlib.Path, config: schema.Main, stages: List[hookspecs.NebariStage], dry_run=False): output_directory = pathlib.Path(output_directory).resolve() if output_directory == pathlib.Path.home(): print("ERROR: Deploying Nebari in home directory is not advised!") @@ -24,7 +23,7 @@ def render_template(output_directory: pathlib.Path, config: schema.Main, dry_run output_directory.mkdir(exist_ok=True, parents=True) contents = {} - for stage in get_available_stages(): + for stage in stages: contents.update( stage(output_directory=output_directory, config=config).render() ) diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index 0d465bcd0..ae2e9477b 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -1,7 +1,9 @@ import contextlib import inspect import itertools +import importlib import os +import re import pathlib from typing import Any, Dict, List, Tuple @@ -110,8 +112,23 @@ def destroy( status["stages/" + self.name] = False -def get_available_stages(): - from nebari.plugins import pm +def get_available_stages(exclude_default_stages: bool = False, exclude_stages: List[str] = []): + from nebari.plugins import pm, load_plugins + + DEFAULT_STAGES = [ + "_nebari.stages.bootstrap", + "_nebari.stages.terraform_state", + "_nebari.stages.infrastructure", + "_nebari.stages.kubernetes_initialize", + "_nebari.stages.kubernetes_ingress", + "_nebari.stages.kubernetes_keycloak", + "_nebari.stages.kubernetes_keycloak_configuration", + "_nebari.stages.kubernetes_services", + "_nebari.stages.nebari_tf_extensions", + ] + + if not exclude_default_stages: + load_plugins(DEFAULT_STAGES) stages = itertools.chain.from_iterable(pm.hook.nebari_stage()) @@ -127,4 +144,13 @@ def get_available_stages(): filtered_stages.insert(0, stage) visited_stage_names.add(stage.name) - return filtered_stages + # filter out stages which match excluded stages + included_stages = [] + for stage in filtered_stages: + for exclude_stage in exclude_stages: + if re.fullmatch(exclude_stage, stage.name) is not None: + break + else: + included_stages.append(stage) + + return included_stages diff --git a/src/_nebari/subcommands/deploy.py b/src/_nebari/subcommands/deploy.py index 18214a28d..dd686ff73 100644 --- a/src/_nebari/subcommands/deploy.py +++ b/src/_nebari/subcommands/deploy.py @@ -12,6 +12,7 @@ def nebari_subcommand(cli: typer.Typer): @cli.command() def deploy( + ctx: typer.Context, config_filename: pathlib.Path = typer.Option( ..., "--config", @@ -61,10 +62,11 @@ def deploy( config = schema.read_configuration(config_filename) if not disable_render: - render_template(output_directory, config) + render_template(output_directory, ctx.obj.config, stages) deploy_configuration( config, + ctx.obj.stages, dns_provider=dns_provider, dns_auto_provision=dns_auto_provision, disable_prompt=disable_prompt, diff --git a/src/_nebari/subcommands/destroy.py b/src/_nebari/subcommands/destroy.py index d38fb5736..356988171 100644 --- a/src/_nebari/subcommands/destroy.py +++ b/src/_nebari/subcommands/destroy.py @@ -12,6 +12,7 @@ def nebari_subcommand(cli: typer.Typer): @cli.command() def destroy( + ctx: typer.Context, config_filename: pathlib.Path = typer.Option( ..., "-c", "--config", help="nebari configuration file path" ), @@ -42,9 +43,9 @@ def _run_destroy( config = schema.read_configuration(config_filename) if not disable_render: - render_template(output_directory, config) + render_template(output_directory, config, ctx.obj.stages) - destroy_configuration(config) + destroy_configuration(config, ctx.obj.stages) if disable_prompt: _run_destroy() diff --git a/src/_nebari/subcommands/info.py b/src/_nebari/subcommands/info.py new file mode 100644 index 000000000..9c68ab302 --- /dev/null +++ b/src/_nebari/subcommands/info.py @@ -0,0 +1,42 @@ +import collections + +import typer +import rich +from rich.table import Table +import typer + +from nebari.hookspecs import hookimpl +from nebari.plugins import pm +from _nebari.stages.base import get_available_stages +from _nebari.version import __version__ + + +@hookimpl +def nebari_subcommand(cli: typer.Typer): + @cli.command() + def info(ctx: typer.Context): + rich.print(f'Nebari version: {__version__}') + + hooks = collections.defaultdict(list) + for plugin in pm.get_plugins(): + for hook in pm.get_hookcallers(plugin): + hooks[hook.name].append(plugin.__name__) + + table = Table(title='Hooks') + table.add_column("hook", justify="left", no_wrap=True) + table.add_column("module", justify="left", no_wrap=True) + + for hook_name, modules in hooks.items(): + for module in modules: + table.add_row(hook_name, module) + + rich.print(table) + + table = Table(title="Runtime Stage Ordering") + table.add_column("name") + table.add_column("priority") + table.add_column("module") + for stage in ctx.obj.stages: + table.add_row(stage.name, str(stage.priority), f'{stage.__module__}.{stage.__name__}') + + rich.print(table) diff --git a/src/_nebari/subcommands/render.py b/src/_nebari/subcommands/render.py index 837ad22e6..a4ba34fac 100644 --- a/src/_nebari/subcommands/render.py +++ b/src/_nebari/subcommands/render.py @@ -11,6 +11,7 @@ def nebari_subcommand(cli: typer.Typer): @cli.command(rich_help_panel="Additional Commands") def render( + ctx: typer.Context, output_directory: pathlib.Path = typer.Option( "./", "-o", @@ -33,4 +34,4 @@ def render( Dynamically render the Terraform scripts and other files from your [purple]nebari-config.yaml[/purple] file. """ config = schema.read_configuration(config_filename) - render_template(output_directory, config, dry_run=dry_run) + render_template(output_directory, config, ctx.obj.stages, dry_run=dry_run) diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index 37e751774..f3a126d56 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -1,21 +1,13 @@ import importlib import sys +import typing +import os import pluggy from nebari import hookspecs -DEFAULT_PLUGINS = [ - # stages - "_nebari.stages.bootstrap", - "_nebari.stages.terraform_state", - "_nebari.stages.infrastructure", - "_nebari.stages.kubernetes_initialize", - "_nebari.stages.kubernetes_ingress", - "_nebari.stages.kubernetes_keycloak", - "_nebari.stages.kubernetes_keycloak_configuration", - "_nebari.stages.kubernetes_services", - "_nebari.stages.nebari_tf_extensions", +DEFAULT_SUBCOMMAND_PLUGINS = [ # subcommands "_nebari.subcommands.init", "_nebari.subcommands.dev", @@ -26,6 +18,7 @@ "_nebari.subcommands.support", "_nebari.subcommands.upgrade", "_nebari.subcommands.validate", + "_nebari.subcommands.info", ] pm = pluggy.PluginManager("nebari") @@ -36,6 +29,21 @@ pm.load_setuptools_entrypoints("nebari") # Load default plugins -for plugin in DEFAULT_PLUGINS: - mod = importlib.import_module(plugin) - pm.register(mod, plugin) +def load_plugins(plugins: typing.List[str]): + def _import_module_from_filename(filename: str): + module_name = f"_nebari.stages._files.{plugin.replace(os.sep, '.')}" + spec = importlib.util.spec_from_file_location(module_name, plugin) + mod = importlib.util.module_from_spec(spec) + sys.modules[module_name] = mod + spec.loader.exec_module(mod) + return mod + + for plugin in plugins: + if plugin.endswith('.py'): + mod = _import_module_from_filename(plugin) + else: + mod = importlib.import_module(plugin) + + pm.register(mod, plugin) + +load_plugins(DEFAULT_SUBCOMMAND_PLUGINS) diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 0c3d6538f..62972c065 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -858,6 +858,10 @@ class InitInputs(Base): disable_prompt: bool = False +class CLIContext(Base): + stages: typing.List = [] + + class Main(Base): provider: ProviderEnum = ProviderEnum.local project_name: str From d20b7a878746cef586d654b0b3dc695de588a820 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 22 Jun 2023 18:58:56 +0000 Subject: [PATCH 073/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/cli.py | 36 +++++++++++++++++++-------------- src/_nebari/deploy.py | 4 +--- src/_nebari/destroy.py | 3 +-- src/_nebari/render.py | 9 +++++++-- src/_nebari/stages/base.py | 9 +++++---- src/_nebari/subcommands/info.py | 14 ++++++------- src/nebari/plugins.py | 6 ++++-- 7 files changed, 46 insertions(+), 35 deletions(-) diff --git a/src/_nebari/cli.py b/src/_nebari/cli.py index bfb9a1d5f..0ebef8f1d 100644 --- a/src/_nebari/cli.py +++ b/src/_nebari/cli.py @@ -1,15 +1,11 @@ -import importlib import typing -import pathlib -import sys -import os import typer from typer.core import TyperGroup from _nebari.version import __version__ from nebari import schema -from nebari.plugins import pm, load_plugins +from nebari.plugins import load_plugins, pm class OrderCommands(TyperGroup): @@ -39,8 +35,10 @@ def exclude_default_stages(ctx: typer.Context, exclude_default_stages: bool): def import_plugin(plugins: typing.List[str]): try: load_plugins(plugins) - except ModuleNotFoundError as e: - typer.echo("ERROR: Python module {e.name} not found. Make sure that the module is in your python path {sys.path}") + except ModuleNotFoundError: + typer.echo( + "ERROR: Python module {e.name} not found. Make sure that the module is in your python path {sys.path}" + ) typer.Exit() return plugins @@ -67,29 +65,37 @@ def common( callback=version_callback, ), plugins: typing.List[str] = typer.Option( - [], "--import-plugin", help="Import nebari plugin", + [], + "--import-plugin", + help="Import nebari plugin", ), excluded_stages: typing.List[str] = typer.Option( - [], "--exclude-stage", help="Exclude nebari stage(s) by name or regex", + [], + "--exclude-stage", + help="Exclude nebari stage(s) by name or regex", ), exclude_default_stages: bool = typer.Option( - False, '--exclude-default-stages', help="Exclude default nebari included stages", - ) + False, + "--exclude-default-stages", + help="Exclude default nebari included stages", + ), ): try: load_plugins(plugins) - except ModuleNotFoundError as e: - typer.echo("ERROR: Python module {e.name} not found. Make sure that the module is in your python path {sys.path}") + except ModuleNotFoundError: + typer.echo( + "ERROR: Python module {e.name} not found. Make sure that the module is in your python path {sys.path}" + ) typer.Exit() from _nebari.stages.base import get_available_stages + ctx.ensure_object(schema.CLIContext) ctx.obj.stages = get_available_stages( exclude_default_stages=exclude_default_stages, - exclude_stages=excluded_stages + exclude_stages=excluded_stages, ) - pm.hook.nebari_subcommand(cli=app) return app diff --git a/src/_nebari/deploy.py b/src/_nebari/deploy.py index 0c4f52308..0b669931e 100644 --- a/src/_nebari/deploy.py +++ b/src/_nebari/deploy.py @@ -1,13 +1,11 @@ import contextlib import logging import pathlib -import subprocess import textwrap from typing import List -from _nebari.stages.base import get_available_stages from _nebari.utils import timer -from nebari import schema, hookspecs +from nebari import hookspecs, schema logger = logging.getLogger(__name__) diff --git a/src/_nebari/destroy.py b/src/_nebari/destroy.py index 6768a5b4f..ecea5874a 100644 --- a/src/_nebari/destroy.py +++ b/src/_nebari/destroy.py @@ -3,9 +3,8 @@ import pathlib from typing import List -from _nebari.stages.base import get_available_stages from _nebari.utils import timer -from nebari import schema, hookspecs +from nebari import hookspecs, schema logger = logging.getLogger(__name__) diff --git a/src/_nebari/render.py b/src/_nebari/render.py index 5419f949d..ba8138633 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -9,10 +9,15 @@ from rich.table import Table from _nebari.deprecate import DEPRECATED_FILE_PATHS -from nebari import schema, hookspecs +from nebari import hookspecs, schema -def render_template(output_directory: pathlib.Path, config: schema.Main, stages: List[hookspecs.NebariStage], dry_run=False): +def render_template( + output_directory: pathlib.Path, + config: schema.Main, + stages: List[hookspecs.NebariStage], + dry_run=False, +): output_directory = pathlib.Path(output_directory).resolve() if output_directory == pathlib.Path.home(): print("ERROR: Deploying Nebari in home directory is not advised!") diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index ae2e9477b..4bdb7169f 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -1,10 +1,9 @@ import contextlib import inspect import itertools -import importlib import os -import re import pathlib +import re from typing import Any, Dict, List, Tuple from _nebari.provider import terraform @@ -112,8 +111,10 @@ def destroy( status["stages/" + self.name] = False -def get_available_stages(exclude_default_stages: bool = False, exclude_stages: List[str] = []): - from nebari.plugins import pm, load_plugins +def get_available_stages( + exclude_default_stages: bool = False, exclude_stages: List[str] = [] +): + from nebari.plugins import load_plugins, pm DEFAULT_STAGES = [ "_nebari.stages.bootstrap", diff --git a/src/_nebari/subcommands/info.py b/src/_nebari/subcommands/info.py index 9c68ab302..f49f3da90 100644 --- a/src/_nebari/subcommands/info.py +++ b/src/_nebari/subcommands/info.py @@ -1,28 +1,26 @@ import collections -import typer import rich -from rich.table import Table import typer +from rich.table import Table +from _nebari.version import __version__ from nebari.hookspecs import hookimpl from nebari.plugins import pm -from _nebari.stages.base import get_available_stages -from _nebari.version import __version__ @hookimpl def nebari_subcommand(cli: typer.Typer): @cli.command() def info(ctx: typer.Context): - rich.print(f'Nebari version: {__version__}') + rich.print(f"Nebari version: {__version__}") hooks = collections.defaultdict(list) for plugin in pm.get_plugins(): for hook in pm.get_hookcallers(plugin): hooks[hook.name].append(plugin.__name__) - table = Table(title='Hooks') + table = Table(title="Hooks") table.add_column("hook", justify="left", no_wrap=True) table.add_column("module", justify="left", no_wrap=True) @@ -37,6 +35,8 @@ def info(ctx: typer.Context): table.add_column("priority") table.add_column("module") for stage in ctx.obj.stages: - table.add_row(stage.name, str(stage.priority), f'{stage.__module__}.{stage.__name__}') + table.add_row( + stage.name, str(stage.priority), f"{stage.__module__}.{stage.__name__}" + ) rich.print(table) diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index f3a126d56..40afdf5b2 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -1,7 +1,7 @@ import importlib +import os import sys import typing -import os import pluggy @@ -28,6 +28,7 @@ # Only load plugins if not running tests pm.load_setuptools_entrypoints("nebari") + # Load default plugins def load_plugins(plugins: typing.List[str]): def _import_module_from_filename(filename: str): @@ -39,11 +40,12 @@ def _import_module_from_filename(filename: str): return mod for plugin in plugins: - if plugin.endswith('.py'): + if plugin.endswith(".py"): mod = _import_module_from_filename(plugin) else: mod = importlib.import_module(plugin) pm.register(mod, plugin) + load_plugins(DEFAULT_SUBCOMMAND_PLUGINS) From e8e0defabce50c24594ca40f4178a09f9dce004b Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 22 Jun 2023 15:00:58 -0400 Subject: [PATCH 074/147] Typo in deploy subcommand --- src/_nebari/subcommands/deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/subcommands/deploy.py b/src/_nebari/subcommands/deploy.py index dd686ff73..35318fa11 100644 --- a/src/_nebari/subcommands/deploy.py +++ b/src/_nebari/subcommands/deploy.py @@ -62,7 +62,7 @@ def deploy( config = schema.read_configuration(config_filename) if not disable_render: - render_template(output_directory, ctx.obj.config, stages) + render_template(output_directory, config, ctx.obj.stages) deploy_configuration( config, From 5c680a6c7b65a49083644f75a797b3816cccb50f Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 22 Jun 2023 15:23:04 -0400 Subject: [PATCH 075/147] Remove image from kubespawner to preserve default behavior --- src/nebari/schema.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 62972c065..79dd7649d 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -628,7 +628,6 @@ class KubeSpawner(Base): cpu_guarantee: int mem_limit: str mem_guarantee: str - image: typing.Optional[str] class Config: extra = "allow" From 1683dec984c369d961eee91fc5365c087563efb4 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 22 Jun 2023 17:01:07 -0400 Subject: [PATCH 076/147] More fixing of the tests --- src/_nebari/initialize.py | 16 ++++++++--- src/nebari/schema.py | 6 ++-- tests/tests_unit/test_cli.py | 24 ++++++++-------- tests/tests_unit/test_init.py | 2 +- tests/tests_unit/test_render.py | 44 ++++------------------------- tests/tests_unit/test_schema.py | 48 ++++++++++++++++++++++++++++++-- tests/tests_unit/test_upgrade.py | 37 +++++++----------------- 7 files changed, 87 insertions(+), 90 deletions(-) diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index 8e730004e..b92cfd5f9 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -87,24 +87,32 @@ def render_config( config["theme"]["jupyterhub"][ "hub_subtitle" ] = f"{WELCOME_HEADER_TEXT} on Digital Ocean" + if kubernetes_version is not None: + config["digital_ocean"] = {"kubernetes_version": kubernetes_version} elif cloud_provider == schema.ProviderEnum.gcp: config["theme"]["jupyterhub"][ "hub_subtitle" ] = f"{WELCOME_HEADER_TEXT} on Google Cloud Platform" + config["google_cloud_platform"] = {} if "PROJECT_ID" in os.environ: - config["google_cloud_platform"] = {"project": os.environ["PROJECT_ID"]} + config["google_cloud_platform"]["project"] = os.environ["PROJECT_ID"] elif not disable_prompt: - config["google_cloud_platform"] = { - "project": input("Enter Google Cloud Platform Project ID: ") - } + config["google_cloud_platform"]["project"] = input("Enter Google Cloud Platform Project ID: ") + + if kubernetes_version is not None: + config["google_cloud_platform"]["kubernetes_version"] = kubernetes_version elif cloud_provider == schema.ProviderEnum.azure: config["theme"]["jupyterhub"][ "hub_subtitle" ] = f"{WELCOME_HEADER_TEXT} on Azure" + if kubernetes_version is not None: + config["azure"] = {"kubernetes_version": kubernetes_version} elif cloud_provider == schema.ProviderEnum.aws: config["theme"]["jupyterhub"][ "hub_subtitle" ] = f"{WELCOME_HEADER_TEXT} on Amazon Web Services" + if kubernetes_version is not None: + config["amazon_web_services"] = {"kubernetes_version": kubernetes_version} elif cloud_provider == schema.ProviderEnum.existing: config["theme"]["jupyterhub"]["hub_subtitle"] = WELCOME_HEADER_TEXT elif cloud_provider == schema.ProviderEnum.local: diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 79dd7649d..a40b86d9e 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -30,7 +30,7 @@ def random_secure_string( - length: int = 32, chars: str = string.ascii_lowercase + string.digits + length: int = 16, chars: str = string.ascii_lowercase + string.digits ): return "".join(secrets.choice(chars) for i in range(length)) @@ -1053,7 +1053,7 @@ def set_nested_attribute(data: typing.Any, attrs: typing.List[str], value: typin def _get_attr(d: typing.Any, attr: str): if hasattr(d, "__getitem__"): - if re.fullmatch("\d+", attr): + if re.fullmatch(r"\d+", attr): try: return d[int(attr)] except Exception: @@ -1065,7 +1065,7 @@ def _get_attr(d: typing.Any, attr: str): def _set_attr(d: typing.Any, attr: str, value: typing.Any): if hasattr(d, "__getitem__"): - if re.fullmatch("\d+", attr): + if re.fullmatch(r"\d+", attr): try: d[int(attr)] = value except Exception: diff --git a/tests/tests_unit/test_cli.py b/tests/tests_unit/test_cli.py index ecd38167a..9061d6be8 100644 --- a/tests/tests_unit/test_cli.py +++ b/tests/tests_unit/test_cli.py @@ -12,7 +12,7 @@ "namespace, auth_provider, ci_provider, ssl_cert_email", ( [None, None, None, None], - ["prod", "github", "github-actions", "it@acme.org"], + ["prod", "password", "github-actions", "it@acme.org"], ), ) def test_nebari_init(tmp_path, namespace, auth_provider, ci_provider, ssl_cert_email): @@ -55,15 +55,13 @@ def test_nebari_init(tmp_path, namespace, auth_provider, ci_provider, ssl_cert_e assert config.certificate.acme_email == ssl_cert_email -def test_python_invocation(): - def run(command): - return subprocess.run( - command, check=True, capture_output=True, text=True - ).stdout.strip() - - command = ["nebari", "--version"] - - actual = run(["python", "-m", *command]) - expected = run(command) - - assert actual == expected +@pytest.mark.parametrize('command', + ( + ["nebari", "--version"], + ["nebari", "info"], + ) +) +def test_nebari_commands_no_args(command): + subprocess.run( + command, check=True, capture_output=True, text=True + ).stdout.strip() diff --git a/tests/tests_unit/test_init.py b/tests/tests_unit/test_init.py index a64d511fc..897839cbd 100644 --- a/tests/tests_unit/test_init.py +++ b/tests/tests_unit/test_init.py @@ -6,7 +6,7 @@ @pytest.mark.parametrize( "k8s_version, expected", [(None, True), ("1.19", True), (1000, ValueError)] ) -def test_init(setup_fixture, k8s_version, expected): +def test_init(setup_fixture, k8s_version, expected, render_config_partial): (nebari_config_loc, render_config_inputs) = setup_fixture ( project, diff --git a/tests/tests_unit/test_render.py b/tests/tests_unit/test_render.py index 2ec7f407a..1ee009d6a 100644 --- a/tests/tests_unit/test_render.py +++ b/tests/tests_unit/test_render.py @@ -4,13 +4,14 @@ import pytest from ruamel.yaml import YAML -from _nebari.render import render_template, set_env_vars_in_config +from _nebari.render import render_template +from _nebari.stages.base import get_available_stages from .utils import PRESERVED_DIR, render_config_partial @pytest.fixture -def write_nebari_config_to_file(setup_fixture): +def write_nebari_config_to_file(setup_fixture, render_config_partial): nebari_config_loc, render_config_inputs = setup_fixture ( project, @@ -31,47 +32,12 @@ def write_nebari_config_to_file(setup_fixture): kubernetes_version=None, ) - # write to nebari_config.yaml - yaml = YAML(typ="unsafe", pure=True) - yaml.dump(config, nebari_config_loc) - - render_template(nebari_config_loc.parent, nebari_config_loc) + stages = get_available_stages() + render_template(str(nebari_config_loc.parent), config, stages) yield setup_fixture -def test_get_secret_config_entries(monkeypatch): - sec1 = "secret1" - sec2 = "nestedsecret1" - config_orig = { - "key1": "value1", - "key2": "NEBARI_SECRET_secret_val", - "key3": { - "nested_key1": "nested_value1", - "nested_key2": "NEBARI_SECRET_nested_secret_val", - }, - } - expected = { - "key1": "value1", - "key2": sec1, - "key3": { - "nested_key1": "nested_value1", - "nested_key2": sec2, - }, - } - - # should raise error if implied env var is not set - with pytest.raises(EnvironmentError): - config = config_orig.copy() - set_env_vars_in_config(config) - - monkeypatch.setenv("secret_val", sec1, prepend=False) - monkeypatch.setenv("nested_secret_val", sec2, prepend=False) - config = config_orig.copy() - set_env_vars_in_config(config) - assert config == expected - - def test_render_template(write_nebari_config_to_file): nebari_config_loc, render_config_inputs = write_nebari_config_to_file ( diff --git a/tests/tests_unit/test_schema.py b/tests/tests_unit/test_schema.py index d4d8cf878..3199ff16e 100644 --- a/tests/tests_unit/test_schema.py +++ b/tests/tests_unit/test_schema.py @@ -3,7 +3,49 @@ from .utils import render_config_partial -def test_schema(setup_fixture): +def test_minimal_schema(): + config = nebari.schema.Main(project_name="test") + assert config.project_name == "test" + assert config.storage.conda_store == '200Gi' + + +def test_minimal_schema_from_file(tmp_path): + filename = tmp_path / "nebari-config.yaml" + with filename.open("w") as f: + f.write("project_name: test\n") + + config = nebari.schema.read_configuration(filename) + assert config.project_name == "test" + assert config.storage.conda_store == '200Gi' + + +def test_minimal_schema_from_file_with_env(tmp_path, monkeypatch): + filename = tmp_path / "nebari-config.yaml" + with filename.open("w") as f: + f.write("project_name: test\n") + + monkeypatch.setenv('NEBARI_SECRET__project_name', 'env') + monkeypatch.setenv('NEBARI_SECRET__storage__conda_store', '1000Gi') + + config = nebari.schema.read_configuration(filename) + assert config.project_name == "env" + assert config.storage.conda_store == '1000Gi' + + +def test_minimal_schema_from_file_with_env(tmp_path, monkeypatch): + filename = tmp_path / "nebari-config.yaml" + with filename.open("w") as f: + f.write("project_name: test\n") + + monkeypatch.setenv('NEBARI_SECRET__project_name', 'env') + monkeypatch.setenv('NEBARI_SECRET__storage__conda_store', '1000Gi') + + config = nebari.schema.read_configuration(filename, read_environment=False) + assert config.project_name == "test" + assert config.storage.conda_store == '200Gi' + + +def test_render_schema(setup_fixture, render_config_partial): (nebari_config_loc, render_config_inputs) = setup_fixture ( project, @@ -23,5 +65,5 @@ def test_schema(setup_fixture): auth_provider=auth_provider, kubernetes_version=None, ) - - _nebari.schema.verify(config) + assert config.project_name == project + assert config.namespace == namespace diff --git a/tests/tests_unit/test_upgrade.py b/tests/tests_unit/test_upgrade.py index f53d980f4..718c4925a 100644 --- a/tests/tests_unit/test_upgrade.py +++ b/tests/tests_unit/test_upgrade.py @@ -2,9 +2,9 @@ import pytest -from _nebari.upgrade import do_upgrade, load_yaml, verify +from _nebari.upgrade import do_upgrade from _nebari.version import __version__, rounded_ver_parse - +from nebari import schema @pytest.fixture def qhub_users_import_json(): @@ -69,38 +69,21 @@ def test_upgrade_4_0( return # Check the resulting YAML - config = load_yaml(tmp_qhub_config) - - verify( - config - ) # Would raise an error if invalid by current Nebari version's standards + config = schema.read_configuration(tmp_qhub_config) - assert len(config["security"]["keycloak"]["initial_root_password"]) == 16 - - assert "users" not in config["security"] - assert "groups" not in config["security"] + assert len(config.security.keycloak.initial_root_password) == 16 + assert not hasattr(config.security, "users") + assert not hasattr(config.security, "groups") __rounded_version__ = ".".join([str(c) for c in rounded_ver_parse(__version__)]) # Check image versions have been bumped up - assert ( - config["default_images"]["jupyterhub"] - == f"quansight/nebari-jupyterhub:v{__rounded_version__}" - ) - assert ( - config["profiles"]["jupyterlab"][0]["kubespawner_override"]["image"] - == f"quansight/nebari-jupyterlab:v{__rounded_version__}" - ) - - assert ( - config.get("security", {}).get("authentication", {}).get("type", "") != "custom" - ) + assert config.default_images.jupyterhub == f"quansight/nebari-jupyterhub:v{__rounded_version__}" + assert config.profiles.jupyterlab[0].kubespawner_override.image == f"quansight/nebari-jupyterlab:v{__rounded_version__}" + assert config.security.authentication.type != "custom" # Keycloak import users json - assert ( - Path(tmp_path, "nebari-users-import.json").read_text().rstrip() - == qhub_users_import_json - ) + assert Path(tmp_path, "nebari-users-import.json").read_text().rstrip() == qhub_users_import_json # Check backup tmp_qhub_config_backup = Path(tmp_path, f"{old_qhub_config_path.name}.old.backup") From 9964ef02ef0fdea2d30125146b2ba47d5e3eb0ab Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 22 Jun 2023 17:02:49 -0400 Subject: [PATCH 077/147] Exclude yaml checks to new stages directory --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4efaf1895..2fd8e1409 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: check-json - id: check-yaml # jinja2 templates for helm charts - exclude: "src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/(clearml/chart/templates/.*|prefect/chart/templates/.*)" + exclude: "src/_nebari/stages/kubernetes-services/template/modules/kubernetes/services/(clearml/chart/templates/.*|prefect/chart/templates/.*)" args: [--allow-multiple-documents] - id: check-toml # Lint: Checks that non-binary executables have a proper shebang. From cdfe3e6c975c7da395ea6ab51aff5cc55807374f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 22 Jun 2023 21:03:36 +0000 Subject: [PATCH 078/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/initialize.py | 4 +++- tests/tests_unit/test_cli.py | 9 ++++----- tests/tests_unit/test_schema.py | 16 ++++++++-------- tests/tests_unit/test_upgrade.py | 16 +++++++++++++--- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index b92cfd5f9..d9f952975 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -97,7 +97,9 @@ def render_config( if "PROJECT_ID" in os.environ: config["google_cloud_platform"]["project"] = os.environ["PROJECT_ID"] elif not disable_prompt: - config["google_cloud_platform"]["project"] = input("Enter Google Cloud Platform Project ID: ") + config["google_cloud_platform"]["project"] = input( + "Enter Google Cloud Platform Project ID: " + ) if kubernetes_version is not None: config["google_cloud_platform"]["kubernetes_version"] = kubernetes_version diff --git a/tests/tests_unit/test_cli.py b/tests/tests_unit/test_cli.py index 9061d6be8..0de08b706 100644 --- a/tests/tests_unit/test_cli.py +++ b/tests/tests_unit/test_cli.py @@ -55,13 +55,12 @@ def test_nebari_init(tmp_path, namespace, auth_provider, ci_provider, ssl_cert_e assert config.certificate.acme_email == ssl_cert_email -@pytest.mark.parametrize('command', +@pytest.mark.parametrize( + "command", ( ["nebari", "--version"], ["nebari", "info"], - ) + ), ) def test_nebari_commands_no_args(command): - subprocess.run( - command, check=True, capture_output=True, text=True - ).stdout.strip() + subprocess.run(command, check=True, capture_output=True, text=True).stdout.strip() diff --git a/tests/tests_unit/test_schema.py b/tests/tests_unit/test_schema.py index 3199ff16e..0010f49bb 100644 --- a/tests/tests_unit/test_schema.py +++ b/tests/tests_unit/test_schema.py @@ -6,7 +6,7 @@ def test_minimal_schema(): config = nebari.schema.Main(project_name="test") assert config.project_name == "test" - assert config.storage.conda_store == '200Gi' + assert config.storage.conda_store == "200Gi" def test_minimal_schema_from_file(tmp_path): @@ -16,7 +16,7 @@ def test_minimal_schema_from_file(tmp_path): config = nebari.schema.read_configuration(filename) assert config.project_name == "test" - assert config.storage.conda_store == '200Gi' + assert config.storage.conda_store == "200Gi" def test_minimal_schema_from_file_with_env(tmp_path, monkeypatch): @@ -24,12 +24,12 @@ def test_minimal_schema_from_file_with_env(tmp_path, monkeypatch): with filename.open("w") as f: f.write("project_name: test\n") - monkeypatch.setenv('NEBARI_SECRET__project_name', 'env') - monkeypatch.setenv('NEBARI_SECRET__storage__conda_store', '1000Gi') + monkeypatch.setenv("NEBARI_SECRET__project_name", "env") + monkeypatch.setenv("NEBARI_SECRET__storage__conda_store", "1000Gi") config = nebari.schema.read_configuration(filename) assert config.project_name == "env" - assert config.storage.conda_store == '1000Gi' + assert config.storage.conda_store == "1000Gi" def test_minimal_schema_from_file_with_env(tmp_path, monkeypatch): @@ -37,12 +37,12 @@ def test_minimal_schema_from_file_with_env(tmp_path, monkeypatch): with filename.open("w") as f: f.write("project_name: test\n") - monkeypatch.setenv('NEBARI_SECRET__project_name', 'env') - monkeypatch.setenv('NEBARI_SECRET__storage__conda_store', '1000Gi') + monkeypatch.setenv("NEBARI_SECRET__project_name", "env") + monkeypatch.setenv("NEBARI_SECRET__storage__conda_store", "1000Gi") config = nebari.schema.read_configuration(filename, read_environment=False) assert config.project_name == "test" - assert config.storage.conda_store == '200Gi' + assert config.storage.conda_store == "200Gi" def test_render_schema(setup_fixture, render_config_partial): diff --git a/tests/tests_unit/test_upgrade.py b/tests/tests_unit/test_upgrade.py index 718c4925a..8c86e89c6 100644 --- a/tests/tests_unit/test_upgrade.py +++ b/tests/tests_unit/test_upgrade.py @@ -6,6 +6,7 @@ from _nebari.version import __version__, rounded_ver_parse from nebari import schema + @pytest.fixture def qhub_users_import_json(): return ( @@ -78,12 +79,21 @@ def test_upgrade_4_0( __rounded_version__ = ".".join([str(c) for c in rounded_ver_parse(__version__)]) # Check image versions have been bumped up - assert config.default_images.jupyterhub == f"quansight/nebari-jupyterhub:v{__rounded_version__}" - assert config.profiles.jupyterlab[0].kubespawner_override.image == f"quansight/nebari-jupyterlab:v{__rounded_version__}" + assert ( + config.default_images.jupyterhub + == f"quansight/nebari-jupyterhub:v{__rounded_version__}" + ) + assert ( + config.profiles.jupyterlab[0].kubespawner_override.image + == f"quansight/nebari-jupyterlab:v{__rounded_version__}" + ) assert config.security.authentication.type != "custom" # Keycloak import users json - assert Path(tmp_path, "nebari-users-import.json").read_text().rstrip() == qhub_users_import_json + assert ( + Path(tmp_path, "nebari-users-import.json").read_text().rstrip() + == qhub_users_import_json + ) # Check backup tmp_qhub_config_backup = Path(tmp_path, f"{old_qhub_config_path.name}.old.backup") From 79820f5cebd690e88548e6d5032508062e488790 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 22 Jun 2023 17:04:56 -0400 Subject: [PATCH 079/147] Typo in yaml exclusion path --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2fd8e1409..fd650b0d4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: check-json - id: check-yaml # jinja2 templates for helm charts - exclude: "src/_nebari/stages/kubernetes-services/template/modules/kubernetes/services/(clearml/chart/templates/.*|prefect/chart/templates/.*)" + exclude: "src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/(clearml/chart/templates/.*|prefect/chart/templates/.*)" args: [--allow-multiple-documents] - id: check-toml # Lint: Checks that non-binary executables have a proper shebang. From 6507ed1ab99d403f01364140bc5026850b85cb0e Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 22 Jun 2023 17:06:16 -0400 Subject: [PATCH 080/147] Fixing ruff checks --- src/nebari/hookspecs.py | 5 +++-- tests/tests_unit/test_schema.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index fac86ae87..3b16b867a 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -5,11 +5,12 @@ import typer from pluggy import HookimplMarker, HookspecMarker +from nebari import schema + + hookspec = HookspecMarker("nebari") hookimpl = HookimplMarker("nebari") -from nebari import schema - class NebariStage: name = None diff --git a/tests/tests_unit/test_schema.py b/tests/tests_unit/test_schema.py index 0010f49bb..50dc341ac 100644 --- a/tests/tests_unit/test_schema.py +++ b/tests/tests_unit/test_schema.py @@ -32,7 +32,7 @@ def test_minimal_schema_from_file_with_env(tmp_path, monkeypatch): assert config.storage.conda_store == "1000Gi" -def test_minimal_schema_from_file_with_env(tmp_path, monkeypatch): +def test_minimal_schema_from_file_without_env(tmp_path, monkeypatch): filename = tmp_path / "nebari-config.yaml" with filename.open("w") as f: f.write("project_name: test\n") From 8cb8d5571d4d4a8729538572e1a935bdf768203c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 22 Jun 2023 21:06:31 +0000 Subject: [PATCH 081/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/nebari/hookspecs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index 3b16b867a..8f041d66f 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -7,7 +7,6 @@ from nebari import schema - hookspec = HookspecMarker("nebari") hookimpl = HookimplMarker("nebari") From dfc92b76414d94828188d116be3f7a4f8a6676fd Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 23 Jun 2023 10:54:23 -0400 Subject: [PATCH 082/147] Reworking stages for more complete testing and passing --- src/_nebari/stages/base.py | 20 ++-- src/nebari/plugins.py | 6 +- tests/tests_unit/test_render.py | 173 ++++++++++++++++++-------------- tests/tests_unit/test_schema.py | 34 ++----- 4 files changed, 121 insertions(+), 112 deletions(-) diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index 4bdb7169f..bd4f5199b 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -34,17 +34,15 @@ def render(self) -> Dict[str, str]: } for root, dirs, filenames in os.walk(self.template_directory): for filename in filenames: - contents[ - os.path.join( - self.stage_prefix, - os.path.relpath( - os.path.join(root, filename), self.template_directory - ), - ) - ] = open( - os.path.join(root, filename), - "rb", - ).read() + with open(os.path.join(root, filename), "rb") as f: + contents[ + os.path.join( + self.stage_prefix, + os.path.relpath( + os.path.join(root, filename), self.template_directory + ), + ) + ] = f.read() return contents def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index 40afdf5b2..37fdc7bae 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -45,7 +45,11 @@ def _import_module_from_filename(filename: str): else: mod = importlib.import_module(plugin) - pm.register(mod, plugin) + try: + pm.register(mod, plugin) + except ValueError: + # Pluin already registered + pass load_plugins(DEFAULT_SUBCOMMAND_PLUGINS) diff --git a/tests/tests_unit/test_render.py b/tests/tests_unit/test_render.py index 1ee009d6a..10e137c76 100644 --- a/tests/tests_unit/test_render.py +++ b/tests/tests_unit/test_render.py @@ -1,5 +1,4 @@ import os -from pathlib import Path import pytest from ruamel.yaml import YAML @@ -10,76 +9,102 @@ from .utils import PRESERVED_DIR, render_config_partial -@pytest.fixture -def write_nebari_config_to_file(setup_fixture, render_config_partial): - nebari_config_loc, render_config_inputs = setup_fixture - ( - project, - namespace, - domain, - cloud_provider, - ci_provider, - auth_provider, - ) = render_config_inputs - - config = render_config_partial( - project_name=project, - namespace=namespace, - nebari_domain=domain, - cloud_provider=cloud_provider, - ci_provider=ci_provider, - auth_provider=auth_provider, - kubernetes_version=None, - ) - - stages = get_available_stages() - render_template(str(nebari_config_loc.parent), config, stages) - - yield setup_fixture - - -def test_render_template(write_nebari_config_to_file): - nebari_config_loc, render_config_inputs = write_nebari_config_to_file - ( - project, - namespace, - domain, - cloud_provider, - ci_provider, - auth_provider, - ) = render_config_inputs - - yaml = YAML() - nebari_config_json = yaml.load(nebari_config_loc.read_text()) - - assert nebari_config_json["project_name"] == project - assert nebari_config_json["namespace"] == namespace - assert nebari_config_json["domain"] == domain - assert nebari_config_json["provider"] == cloud_provider - - -def test_exists_after_render(write_nebari_config_to_file): - items_to_check = [ - ".gitignore", - "stages", - "nebari-config.yaml", - PRESERVED_DIR, - ] - - nebari_config_loc, _ = write_nebari_config_to_file - - yaml = YAML() - nebari_config_json = yaml.load(nebari_config_loc.read_text()) - - # list of files/dirs available after `nebari render` command - ls = os.listdir(Path(nebari_config_loc).parent.resolve()) - - cicd = nebari_config_json.get("ci_cd", {}).get("type", None) - - if cicd == "github-actions": - items_to_check.append(".github") - elif cicd == "gitlab-ci": - items_to_check.append(".gitlab-ci.yml") - - for i in items_to_check: - assert i in ls +def test_render_config(nebari_render): + output_directory, config_filename = nebari_render + config = schema.read_configuration(config_filename) + assert {'nebari-config.yaml', 'stages', '.gitignore'} <= set(os.listdir(output_directory)) + assert {'07-kubernetes-services', '02-infrastructure', '01-terraform-state', '05-kubernetes-keycloak', '08-nebari-tf-extensions', '06-kubernetes-keycloak-configuration', '04-kubernetes-ingress', '03-kubernetes-initialize'} == set(os.listdir(output_directory / "stages")) + + if config.provider == schema.ProviderEnum.do: + assert (output_directory / "stages" / "01-terraform-state/do").is_dir() + assert (output_directory / "stages" / "02-infrastructure/do").is_dir() + elif config.provider == schema.ProviderEnum.aws: + assert (output_directory / "stages" / "01-terraform-state/aws").is_dir() + assert (output_directory / "stages" / "02-infrastructure/aws").is_dir() + elif config.provider == schema.ProviderEnum.gcp: + assert (output_directory / "stages" / "01-terraform-state/gcp").is_dir() + assert (output_directory / "stages" / "02-infrastructure/gcp").is_dir() + elif config.provider == schema.ProviderEnum.azure: + assert (output_directory / "stages" / "01-terraform-state/azure").is_dir() + assert (output_directory / "stages" / "02-infrastructure/azure").is_dir() + + if config.ci_cd.type == schema.CiEnum.github_actions: + assert (output_directory / ".github/workflows/").is_dir() + elif config.ci_cd.type == schema.CiEnum.gitlab_ci: + assert (output_directory / ".gitlab-ci.yml").is_file() + + + +# @pytest.fixture +# def write_nebari_config_to_file(setup_fixture, render_config_partial): +# nebari_config_loc, render_config_inputs = setup_fixture +# ( +# project, +# namespace, +# domain, +# cloud_provider, +# ci_provider, +# auth_provider, +# ) = render_config_inputs + +# config = render_config_partial( +# project_name=project, +# namespace=namespace, +# nebari_domain=domain, +# cloud_provider=cloud_provider, +# ci_provider=ci_provider, +# auth_provider=auth_provider, +# kubernetes_version=None, +# ) + +# stages = get_available_stages() +# render_template(str(nebari_config_loc.parent), config, stages) + +# yield setup_fixture + + +# def test_render_template(write_nebari_config_to_file): +# nebari_config_loc, render_config_inputs = write_nebari_config_to_file +# ( +# project, +# namespace, +# domain, +# cloud_provider, +# ci_provider, +# auth_provider, +# ) = render_config_inputs + +# yaml = YAML() +# nebari_config_json = yaml.load(nebari_config_loc.read_text()) + +# assert nebari_config_json["project_name"] == project +# assert nebari_config_json["namespace"] == namespace +# assert nebari_config_json["domain"] == domain +# assert nebari_config_json["provider"] == cloud_provider + + +# def test_exists_after_render(write_nebari_config_to_file): +# items_to_check = [ +# ".gitignore", +# "stages", +# "nebari-config.yaml", +# PRESERVED_DIR, +# ] + +# nebari_config_loc, _ = write_nebari_config_to_file + +# yaml = YAML() +# nebari_config_json = yaml.load(nebari_config_loc.read_text()) + +# # list of files/dirs available after `nebari render` command +# ls = os.listdir(Path(nebari_config_loc).parent.resolve()) + +# cicd = nebari_config_json.get("ci_cd", {}).get("type", None) + +# if cicd == "github-actions": +# items_to_check.append(".github") +# elif cicd == "gitlab-ci": +# items_to_check.append(".gitlab-ci.yml") + +# for i in items_to_check: +# assert i in ls diff --git a/tests/tests_unit/test_schema.py b/tests/tests_unit/test_schema.py index 50dc341ac..44d0b6d2f 100644 --- a/tests/tests_unit/test_schema.py +++ b/tests/tests_unit/test_schema.py @@ -4,7 +4,7 @@ def test_minimal_schema(): - config = nebari.schema.Main(project_name="test") + config = schema.Main(project_name="test") assert config.project_name == "test" assert config.storage.conda_store == "200Gi" @@ -14,7 +14,7 @@ def test_minimal_schema_from_file(tmp_path): with filename.open("w") as f: f.write("project_name: test\n") - config = nebari.schema.read_configuration(filename) + config = schema.read_configuration(filename) assert config.project_name == "test" assert config.storage.conda_store == "200Gi" @@ -27,7 +27,7 @@ def test_minimal_schema_from_file_with_env(tmp_path, monkeypatch): monkeypatch.setenv("NEBARI_SECRET__project_name", "env") monkeypatch.setenv("NEBARI_SECRET__storage__conda_store", "1000Gi") - config = nebari.schema.read_configuration(filename) + config = schema.read_configuration(filename) assert config.project_name == "env" assert config.storage.conda_store == "1000Gi" @@ -40,30 +40,12 @@ def test_minimal_schema_from_file_without_env(tmp_path, monkeypatch): monkeypatch.setenv("NEBARI_SECRET__project_name", "env") monkeypatch.setenv("NEBARI_SECRET__storage__conda_store", "1000Gi") - config = nebari.schema.read_configuration(filename, read_environment=False) + config = schema.read_configuration(filename, read_environment=False) assert config.project_name == "test" assert config.storage.conda_store == "200Gi" -def test_render_schema(setup_fixture, render_config_partial): - (nebari_config_loc, render_config_inputs) = setup_fixture - ( - project, - namespace, - domain, - cloud_provider, - ci_provider, - auth_provider, - ) = render_config_inputs - - config = render_config_partial( - project_name=project, - namespace=namespace, - nebari_domain=domain, - cloud_provider=cloud_provider, - ci_provider=ci_provider, - auth_provider=auth_provider, - kubernetes_version=None, - ) - assert config.project_name == project - assert config.namespace == namespace +def test_render_schema(nebari_config): + assert isinstance(nebari_config, schema.Main) + assert nebari_config.project_name == f"pytest{nebari_config.provider.value}" + assert nebari_config.namespace == "dev" From fe48c243369f583b882703bed57309f6276edffe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 23 Jun 2023 15:04:09 +0000 Subject: [PATCH 083/147] [pre-commit.ci] Apply automatic pre-commit fixes --- tests/tests_unit/test_render.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/tests_unit/test_render.py b/tests/tests_unit/test_render.py index 10e137c76..edf48b305 100644 --- a/tests/tests_unit/test_render.py +++ b/tests/tests_unit/test_render.py @@ -12,8 +12,19 @@ def test_render_config(nebari_render): output_directory, config_filename = nebari_render config = schema.read_configuration(config_filename) - assert {'nebari-config.yaml', 'stages', '.gitignore'} <= set(os.listdir(output_directory)) - assert {'07-kubernetes-services', '02-infrastructure', '01-terraform-state', '05-kubernetes-keycloak', '08-nebari-tf-extensions', '06-kubernetes-keycloak-configuration', '04-kubernetes-ingress', '03-kubernetes-initialize'} == set(os.listdir(output_directory / "stages")) + assert {"nebari-config.yaml", "stages", ".gitignore"} <= set( + os.listdir(output_directory) + ) + assert { + "07-kubernetes-services", + "02-infrastructure", + "01-terraform-state", + "05-kubernetes-keycloak", + "08-nebari-tf-extensions", + "06-kubernetes-keycloak-configuration", + "04-kubernetes-ingress", + "03-kubernetes-initialize", + } == set(os.listdir(output_directory / "stages")) if config.provider == schema.ProviderEnum.do: assert (output_directory / "stages" / "01-terraform-state/do").is_dir() @@ -34,7 +45,6 @@ def test_render_config(nebari_render): assert (output_directory / ".gitlab-ci.yml").is_file() - # @pytest.fixture # def write_nebari_config_to_file(setup_fixture, render_config_partial): # nebari_config_loc, render_config_inputs = setup_fixture From 2dd5dcb96e0bb2ca24160ef1e821ea40f27114ae Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 23 Jun 2023 11:10:47 -0400 Subject: [PATCH 084/147] Fixing render template path --- .github/actions/publish-from-template/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/publish-from-template/action.yml b/.github/actions/publish-from-template/action.yml index cde326890..8c45e3727 100644 --- a/.github/actions/publish-from-template/action.yml +++ b/.github/actions/publish-from-template/action.yml @@ -16,7 +16,7 @@ runs: shell: bash env: ${{ env }} run: - python ${{ github.action_path }}/publish-from-template.py ${{inputs.filename }} + python ${{ github.action_path }}/render_template.py ${{inputs.filename }} - uses: JasonEtco/create-an-issue@v2 # Only render template and create an issue in case the workflow is a scheduled one From 644995ff12ad4b5740d73372d9113f7e21d25a6c Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 23 Jun 2023 12:00:52 -0400 Subject: [PATCH 085/147] Adding a timeout to notebook tests --- .github/workflows/kubernetes_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/kubernetes_test.yaml b/.github/workflows/kubernetes_test.yaml index d987066ec..ce34caa6c 100644 --- a/.github/workflows/kubernetes_test.yaml +++ b/.github/workflows/kubernetes_test.yaml @@ -177,6 +177,7 @@ jobs: pytest tests/tests_deployment/ -v -s - name: JupyterHub Notebook Tests + timeout-minutes: 2 # run jhub-client after pytest since jhubctl can cleanup # the running server run: | From de247300f094bba0e04e40ef783c7a7b31af99c5 Mon Sep 17 00:00:00 2001 From: eskild <42120229+iameskild@users.noreply.github.com> Date: Tue, 27 Jun 2023 20:49:24 +0200 Subject: [PATCH 086/147] Add NebariPluginManager, allow schema to be dynamically extended (#1844) --- pytest.ini | 2 + src/_nebari/__main__.py | 10 -- src/_nebari/cli.py | 101 ----------- src/_nebari/stages/base.py | 48 ----- src/_nebari/subcommands/deploy.py | 11 +- src/_nebari/subcommands/destroy.py | 12 +- src/_nebari/subcommands/dev.py | 7 +- src/_nebari/subcommands/info.py | 42 ----- src/_nebari/subcommands/init.py | 67 ++++--- src/_nebari/subcommands/keycloak.py | 6 +- src/_nebari/subcommands/render.py | 9 +- src/_nebari/subcommands/validate.py | 6 +- src/nebari/__main__.py | 5 +- src/nebari/hookspecs.py | 6 +- src/nebari/plugins.py | 269 +++++++++++++++++++++++++--- src/nebari/schema.py | 12 +- 16 files changed, 346 insertions(+), 267 deletions(-) delete mode 100644 src/_nebari/__main__.py delete mode 100644 src/_nebari/cli.py delete mode 100644 src/_nebari/subcommands/info.py diff --git a/pytest.ini b/pytest.ini index 89f5ec586..ee4e7f5cb 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,6 +6,8 @@ addopts = --tb=native # turn warnings into errors -Werror + # ignore deprecation warnings (TODO: filter further) + -W ignore::DeprecationWarning markers = conda: conda required to run this test (deselect with '-m \"not conda\"') aws: deploy on aws diff --git a/src/_nebari/__main__.py b/src/_nebari/__main__.py deleted file mode 100644 index b18eaf428..000000000 --- a/src/_nebari/__main__.py +++ /dev/null @@ -1,10 +0,0 @@ -from _nebari.cli import create_cli - - -def main(): - cli = create_cli() - cli() - - -if __name__ == "__main__": - main() diff --git a/src/_nebari/cli.py b/src/_nebari/cli.py deleted file mode 100644 index 0ebef8f1d..000000000 --- a/src/_nebari/cli.py +++ /dev/null @@ -1,101 +0,0 @@ -import typing - -import typer -from typer.core import TyperGroup - -from _nebari.version import __version__ -from nebari import schema -from nebari.plugins import load_plugins, pm - - -class OrderCommands(TyperGroup): - def list_commands(self, ctx: typer.Context): - """Return list of commands in the order appear.""" - return list(self.commands) - - -def version_callback(value: bool): - if value: - typer.echo(__version__) - raise typer.Exit() - - -def exclude_stages(ctx: typer.Context, stages: typing.List[str]): - ctx.ensure_object(schema.CLIContext) - ctx.obj.excluded_stages = stages - return stages - - -def exclude_default_stages(ctx: typer.Context, exclude_default_stages: bool): - ctx.ensure_object(schema.CLIContext) - ctx.obj.exclude_default_stages = exclude_default_stages - return exclude_default_stages - - -def import_plugin(plugins: typing.List[str]): - try: - load_plugins(plugins) - except ModuleNotFoundError: - typer.echo( - "ERROR: Python module {e.name} not found. Make sure that the module is in your python path {sys.path}" - ) - typer.Exit() - return plugins - - -def create_cli(): - app = typer.Typer( - cls=OrderCommands, - help="Nebari CLI 🪴", - add_completion=False, - no_args_is_help=True, - rich_markup_mode="rich", - pretty_exceptions_show_locals=False, - context_settings={"help_option_names": ["-h", "--help"]}, - ) - - @app.callback() - def common( - ctx: typer.Context, - version: bool = typer.Option( - None, - "-V", - "--version", - help="Nebari version number", - callback=version_callback, - ), - plugins: typing.List[str] = typer.Option( - [], - "--import-plugin", - help="Import nebari plugin", - ), - excluded_stages: typing.List[str] = typer.Option( - [], - "--exclude-stage", - help="Exclude nebari stage(s) by name or regex", - ), - exclude_default_stages: bool = typer.Option( - False, - "--exclude-default-stages", - help="Exclude default nebari included stages", - ), - ): - try: - load_plugins(plugins) - except ModuleNotFoundError: - typer.echo( - "ERROR: Python module {e.name} not found. Make sure that the module is in your python path {sys.path}" - ) - typer.Exit() - - from _nebari.stages.base import get_available_stages - - ctx.ensure_object(schema.CLIContext) - ctx.obj.stages = get_available_stages( - exclude_default_stages=exclude_default_stages, - exclude_stages=excluded_stages, - ) - - pm.hook.nebari_subcommand(cli=app) - - return app diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index bd4f5199b..8f603250d 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -1,9 +1,7 @@ import contextlib import inspect -import itertools import os import pathlib -import re from typing import Any, Dict, List, Tuple from _nebari.provider import terraform @@ -107,49 +105,3 @@ def destroy( if not ignore_errors: raise e status["stages/" + self.name] = False - - -def get_available_stages( - exclude_default_stages: bool = False, exclude_stages: List[str] = [] -): - from nebari.plugins import load_plugins, pm - - DEFAULT_STAGES = [ - "_nebari.stages.bootstrap", - "_nebari.stages.terraform_state", - "_nebari.stages.infrastructure", - "_nebari.stages.kubernetes_initialize", - "_nebari.stages.kubernetes_ingress", - "_nebari.stages.kubernetes_keycloak", - "_nebari.stages.kubernetes_keycloak_configuration", - "_nebari.stages.kubernetes_services", - "_nebari.stages.nebari_tf_extensions", - ] - - if not exclude_default_stages: - load_plugins(DEFAULT_STAGES) - - stages = itertools.chain.from_iterable(pm.hook.nebari_stage()) - - # order stages by priority - sorted_stages = sorted(stages, key=lambda s: s.priority) - - # filter out duplicate stages with same name (keep highest priority) - visited_stage_names = set() - filtered_stages = [] - for stage in reversed(sorted_stages): - if stage.name in visited_stage_names: - continue - filtered_stages.insert(0, stage) - visited_stage_names.add(stage.name) - - # filter out stages which match excluded stages - included_stages = [] - for stage in filtered_stages: - for exclude_stage in exclude_stages: - if re.fullmatch(exclude_stage, stage.name) is not None: - break - else: - included_stages.append(stage) - - return included_stages diff --git a/src/_nebari/subcommands/deploy.py b/src/_nebari/subcommands/deploy.py index 35318fa11..412238f75 100644 --- a/src/_nebari/subcommands/deploy.py +++ b/src/_nebari/subcommands/deploy.py @@ -59,14 +59,19 @@ def deploy( """ Deploy the Nebari cluster from your [purple]nebari-config.yaml[/purple] file. """ - config = schema.read_configuration(config_filename) + from nebari.plugins import nebari_plugin_manager + + stages = nebari_plugin_manager.ordered_stages + config_schema = nebari_plugin_manager.config_schema + + config = schema.read_configuration(config_filename, config_schema=config_schema) if not disable_render: - render_template(output_directory, config, ctx.obj.stages) + render_template(output_directory, config, stages) deploy_configuration( config, - ctx.obj.stages, + stages, dns_provider=dns_provider, dns_auto_provision=dns_auto_provision, disable_prompt=disable_prompt, diff --git a/src/_nebari/subcommands/destroy.py b/src/_nebari/subcommands/destroy.py index 356988171..5b1e56656 100644 --- a/src/_nebari/subcommands/destroy.py +++ b/src/_nebari/subcommands/destroy.py @@ -36,16 +36,22 @@ def destroy( """ Destroy the Nebari cluster from your [purple]nebari-config.yaml[/purple] file. """ + from nebari.plugins import nebari_plugin_manager + + stages = nebari_plugin_manager.ordered_stages + config_schema = nebari_plugin_manager.config_schema def _run_destroy( config_filename=config_filename, disable_render=disable_render ): - config = schema.read_configuration(config_filename) + config = schema.read_configuration( + config_filename, config_schema=config_schema + ) if not disable_render: - render_template(output_directory, config, ctx.obj.stages) + render_template(output_directory, config, stages) - destroy_configuration(config, ctx.obj.stages) + destroy_configuration(config, stages) if disable_prompt: _run_destroy() diff --git a/src/_nebari/subcommands/dev.py b/src/_nebari/subcommands/dev.py index 296d3d219..54ee0f6b9 100644 --- a/src/_nebari/subcommands/dev.py +++ b/src/_nebari/subcommands/dev.py @@ -4,6 +4,7 @@ import typer from _nebari.keycloak import keycloak_rest_api_call +from nebari import schema from nebari.hookspecs import hookimpl @@ -45,6 +46,10 @@ def keycloak_api( Please use this at your own risk. """ - schema.read_configuration(config_filename) + from nebari.plugins import nebari_plugin_manager + + config_schema = nebari_plugin_manager.config_schema + + schema.read_configuration(config_filename, config_schema=config_schema) r = keycloak_rest_api_call(config_filename, request=request) print(json.dumps(r, indent=4)) diff --git a/src/_nebari/subcommands/info.py b/src/_nebari/subcommands/info.py deleted file mode 100644 index f49f3da90..000000000 --- a/src/_nebari/subcommands/info.py +++ /dev/null @@ -1,42 +0,0 @@ -import collections - -import rich -import typer -from rich.table import Table - -from _nebari.version import __version__ -from nebari.hookspecs import hookimpl -from nebari.plugins import pm - - -@hookimpl -def nebari_subcommand(cli: typer.Typer): - @cli.command() - def info(ctx: typer.Context): - rich.print(f"Nebari version: {__version__}") - - hooks = collections.defaultdict(list) - for plugin in pm.get_plugins(): - for hook in pm.get_hookcallers(plugin): - hooks[hook.name].append(plugin.__name__) - - table = Table(title="Hooks") - table.add_column("hook", justify="left", no_wrap=True) - table.add_column("module", justify="left", no_wrap=True) - - for hook_name, modules in hooks.items(): - for module in modules: - table.add_row(hook_name, module) - - rich.print(table) - - table = Table(title="Runtime Stage Ordering") - table.add_column("name") - table.add_column("priority") - table.add_column("module") - for stage in ctx.obj.stages: - table.add_row( - stage.name, str(stage.priority), f"{stage.__module__}.{stage.__name__}" - ) - - rich.print(table) diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index e40aa8130..cff725001 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -6,6 +6,7 @@ import questionary import rich import typer +from pydantic import BaseModel from _nebari.initialize import render_config from nebari import schema @@ -45,19 +46,10 @@ def enum_to_list(enum_cls): return [e.value for e in enum_cls] -def handle_init(inputs: schema.InitInputs): +def handle_init(inputs: schema.InitInputs, config_schema: BaseModel): """ Take the inputs from the `nebari init` command, render the config and write it to a local yaml file. """ - # if NEBARI_IMAGE_TAG: - # print( - # f"Modifying the image tags for the `default_images`, setting tags to: {NEBARI_IMAGE_TAG}" - # ) - - # if NEBARI_DASK_VERSION: - # print( - # f"Modifying the version of the `nebari_dask` package, setting version to: {NEBARI_DASK_VERSION}" - # ) # this will force the `set_kubernetes_version` to grab the latest version if inputs.kubernetes_version == "latest": @@ -80,7 +72,9 @@ def handle_init(inputs: schema.InitInputs): ) try: - schema.write_configuration(pathlib.Path("nebari-config.yaml"), config, mode="x") + schema.write_configuration( + inputs.output, config, mode="x", config_schema=config_schema + ) except FileExistsError: raise ValueError( "A nebari-config.yaml file already exists. Please move or delete it and try again." @@ -104,6 +98,24 @@ def check_ssl_cert_email(ctx: typer.Context, ssl_cert_email: str): return ssl_cert_email +def check_repository_creds(ctx: typer.Context, git_provider: str): + """Validate the necessary Git provider (GitHub) credentials are set.""" + + if ( + git_provider == schema.GitRepoEnum.github.value.lower() + and not os.environ.get("GITHUB_USERNAME") + or not os.environ.get("GITHUB_TOKEN") + ): + os.environ["GITHUB_USERNAME"] = typer.prompt( + "Paste your GITHUB_USERNAME", + hide_input=True, + ) + os.environ["GITHUB_TOKEN"] = typer.prompt( + "Paste your GITHUB_TOKEN", + hide_input=True, + ) + + def check_auth_provider_creds(ctx: typer.Context, auth_provider: str): """Validate the the necessary auth provider credentials have been set as environment variables.""" if ctx.params.get("disable_prompt"): @@ -333,6 +345,12 @@ def init( False, is_eager=True, ), + output: str = typer.Option( + pathlib.Path("nebari-config.yaml"), + "--output", + "-o", + help="Output file path for the rendered config file.", + ), ): """ Create and initialize your [purple]nebari-config.yaml[/purple] file. @@ -363,8 +381,13 @@ def init( inputs.kubernetes_version = kubernetes_version inputs.ssl_cert_email = ssl_cert_email inputs.disable_prompt = disable_prompt + inputs.output = output + + from nebari.plugins import nebari_plugin_manager - handle_init(inputs) + handle_init(inputs, config_schema=nebari_plugin_manager.config_schema) + + nebari_plugin_manager.load_config(output) def guided_init_wizard(ctx: typer.Context, guided_init: str): @@ -522,7 +545,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): git_provider = questionary.select( "Which git provider would you like to use?", - choices=enum_to_list(GitRepoEnum), + choices=enum_to_list(schema.GitRepoEnum), qmark=qmark, ).unsafe_ask() @@ -540,7 +563,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): git_provider=git_provider, org_name=org_name, repo_name=repo_name ) - if git_provider == GitRepoEnum.github.value.lower(): + if git_provider == schema.GitRepoEnum.github.value.lower(): inputs.repository_auto_provision = questionary.confirm( f"Would you like nebari to create a remote repository on {git_provider}?", default=False, @@ -551,10 +574,10 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): if not disable_checks and inputs.repository_auto_provision: check_repository_creds(ctx, git_provider) - if git_provider == GitRepoEnum.github.value.lower(): - inputs.ci_provider = CiEnum.github_actions.value.lower() - elif git_provider == GitRepoEnum.gitlab.value.lower(): - inputs.ci_provider = CiEnum.gitlab_ci.value.lower() + if git_provider == schema.GitRepoEnum.github.value.lower(): + inputs.ci_provider = schema.CiEnum.github_actions.value.lower() + elif git_provider == schema.GitRepoEnum.gitlab.value.lower(): + inputs.ci_provider = schema.CiEnum.gitlab_ci.value.lower() # SSL CERTIFICATE if inputs.domain_name: @@ -598,7 +621,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): # TERRAFORM STATE inputs.terraform_state = questionary.select( "Where should the Terraform State be provisioned?", - choices=enum_to_list(TerraformStateEnum), + choices=enum_to_list(schema.TerraformStateEnum), qmark=qmark, ).unsafe_ask() @@ -615,7 +638,11 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): qmark=qmark, ).unsafe_ask() - handle_init(inputs) + from nebari.plugins import nebari_plugin_manager + + config_schema = nebari_plugin_manager.config_schema + + handle_init(inputs, config_schema=config_schema) rich.print( ( diff --git a/src/_nebari/subcommands/keycloak.py b/src/_nebari/subcommands/keycloak.py index cab4f36f1..ddf144ced 100644 --- a/src/_nebari/subcommands/keycloak.py +++ b/src/_nebari/subcommands/keycloak.py @@ -71,6 +71,10 @@ def export_users( ), ): """Export the users in Keycloak.""" - config = schema.read_configuration(config_filename) + from nebari.plugins import nebari_plugin_manager + + config_schema = nebari_plugin_manager.config_schema + + config = schema.read_configuration(config_filename, config_schema=config_schema) r = export_keycloak_users(config, realm=realm) print(json.dumps(r, indent=4)) diff --git a/src/_nebari/subcommands/render.py b/src/_nebari/subcommands/render.py index a4ba34fac..a8f1f0448 100644 --- a/src/_nebari/subcommands/render.py +++ b/src/_nebari/subcommands/render.py @@ -33,5 +33,10 @@ def render( """ Dynamically render the Terraform scripts and other files from your [purple]nebari-config.yaml[/purple] file. """ - config = schema.read_configuration(config_filename) - render_template(output_directory, config, ctx.obj.stages, dry_run=dry_run) + from nebari.plugins import nebari_plugin_manager + + stages = nebari_plugin_manager.ordered_stages + config_schema = nebari_plugin_manager.config_schema + + config = schema.read_configuration(config_filename, config_schema=config_schema) + render_template(output_directory, config, stages, dry_run=dry_run) diff --git a/src/_nebari/subcommands/validate.py b/src/_nebari/subcommands/validate.py index 062cec8a3..5dfa81308 100644 --- a/src/_nebari/subcommands/validate.py +++ b/src/_nebari/subcommands/validate.py @@ -29,5 +29,9 @@ def validate( # comment_on_pr(config) pass else: - schema.read_configuration(config_filename) + from nebari.plugins import nebari_plugin_manager + + config_schema = nebari_plugin_manager.config_schema + + schema.read_configuration(config_filename, config_schema=config_schema) print("[bold purple]Successfully validated configuration.[/bold purple]") diff --git a/src/nebari/__main__.py b/src/nebari/__main__.py index b18eaf428..4a884d3c9 100644 --- a/src/nebari/__main__.py +++ b/src/nebari/__main__.py @@ -1,9 +1,8 @@ -from _nebari.cli import create_cli +from nebari.plugins import nebari_plugin_manager def main(): - cli = create_cli() - cli() + nebari_plugin_manager.create_cli() if __name__ == "__main__": diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index 8f041d66f..7e4c47009 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -4,6 +4,7 @@ import typer from pluggy import HookimplMarker, HookspecMarker +from pydantic import BaseModel from nebari import schema @@ -12,8 +13,9 @@ class NebariStage: - name = None - priority = None + name: str = None + priority: int = None + stage_schema: BaseModel = None def __init__(self, output_directory: pathlib.Path, config: schema.Main): self.output_directory = output_directory diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index 37fdc7bae..465ceb27e 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -1,11 +1,21 @@ +import collections import importlib +import itertools import os +import re import sys import typing +from pathlib import Path import pluggy +import rich +import typer +from pydantic import BaseModel, create_model +from rich.table import Table +from typer.core import TyperGroup -from nebari import hookspecs +from _nebari.version import __version__ +from nebari import hookspecs, schema DEFAULT_SUBCOMMAND_PLUGINS = [ # subcommands @@ -18,38 +28,243 @@ "_nebari.subcommands.support", "_nebari.subcommands.upgrade", "_nebari.subcommands.validate", - "_nebari.subcommands.info", ] -pm = pluggy.PluginManager("nebari") -pm.add_hookspecs(hookspecs) +DEFAULT_STAGES_PLUGINS = [ + # stages + "_nebari.stages.bootstrap", + "_nebari.stages.terraform_state", + "_nebari.stages.infrastructure", + "_nebari.stages.kubernetes_initialize", + "_nebari.stages.kubernetes_ingress", + "_nebari.stages.kubernetes_keycloak", + "_nebari.stages.kubernetes_keycloak_configuration", + "_nebari.stages.kubernetes_services", + "_nebari.stages.nebari_tf_extensions", +] + + +class NebariPluginManager: + plugin_manager = pluggy.PluginManager("nebari") + + ordered_stages: typing.List[hookspecs.NebariStage] = [] + exclude_default_stages: bool = False + exclude_stages: typing.List[str] = [] + + cli: typer.Typer = None + + schema_name: str = "NebariConfig" + config_schema: typing.Union[BaseModel, None] = None + config_path: typing.Union[Path, None] = None + config: typing.Union[BaseModel, None] = None + + def __init__(self) -> None: + self.plugin_manager.add_hookspecs(hookspecs) + + if not hasattr(sys, "_called_from_test"): + # Only load plugins if not running tests + self.plugin_manager.load_setuptools_entrypoints("nebari") + + self.load_subcommands(DEFAULT_SUBCOMMAND_PLUGINS) + self.ordered_stages = self.get_available_stages() + self.config_schema = self.extend_schema() + + def load_subcommands(self, subcommand: typing.List[str]): + self._load_plugins(subcommand) + + def load_stages(self, stages: typing.List[str]): + self._load_plugins(stages) + + def _load_plugins(self, plugins: typing.List[str]): + def _import_module_from_filename(plugin: str): + module_name = f"_nebari.stages._files.{plugin.replace(os.sep, '.')}" + spec = importlib.util.spec_from_file_location(module_name, plugin) + mod = importlib.util.module_from_spec(spec) + sys.modules[module_name] = mod + spec.loader.exec_module(mod) + return mod + + for plugin in plugins: + if plugin.endswith(".py"): + mod = _import_module_from_filename(plugin) + else: + mod = importlib.import_module(plugin) + + try: + self.plugin_manager.register(mod, plugin) + except ValueError: + # Pluin already registered + pass + + def get_available_stages(self): + if not self.exclude_default_stages: + self.load_stages(DEFAULT_STAGES_PLUGINS) + + stages = itertools.chain.from_iterable(self.plugin_manager.hook.nebari_stage()) + + # order stages by priority + sorted_stages = sorted(stages, key=lambda s: s.priority) + + # filter out duplicate stages with same name (keep highest priority) + visited_stage_names = set() + filtered_stages = [] + for stage in reversed(sorted_stages): + if stage.name in visited_stage_names: + continue + filtered_stages.insert(0, stage) + visited_stage_names.add(stage.name) + + # filter out stages which match excluded stages + included_stages = [] + for stage in filtered_stages: + for exclude_stage in self.exclude_stages: + if re.fullmatch(exclude_stage, stage.name) is not None: + break + else: + included_stages.append(stage) + + return included_stages + + def load_config(self, config_path: typing.Union[str, Path]): + if isinstance(config_path, str): + config_path = Path(config_path) + + if not config_path.exists(): + raise FileNotFoundError(f"Config file {config_path} not found") + + self.config_path = config_path + self.config = schema.read_configuration(config_path) + + def _create_dynamic_schema( + self, base: BaseModel, stage: BaseModel, stage_name: str + ) -> BaseModel: + stage_fields = { + n: (f.type_, f.default if f.default is not None else ...) + for n, f in stage.__fields__.items() + } + # ensure top-level key for `stage` is set to `stage_name` + stage_model = create_model(stage_name, __base__=schema.Base, **stage_fields) + extra_fields = {stage_name: (stage_model, None)} + return create_model(self.schema_name, __base__=base, **extra_fields) + + def extend_schema(self, base_schema: BaseModel = schema.Main) -> BaseModel: + config_schema = base_schema + for stages in self.ordered_stages: + if stages.stage_schema: + config_schema = self._create_dynamic_schema( + config_schema, + stages.stage_schema, + stages.name, + ) + return config_schema + + def _version_callback(self, value: bool): + if value: + typer.echo(__version__) + raise typer.Exit() + + def create_cli(self) -> typer.Typer: + class OrderCommands(TyperGroup): + def list_commands(self, ctx: typer.Context): + """Return list of commands in the order appear.""" + return list(self.commands) + + cli = typer.Typer( + cls=OrderCommands, + help="Nebari CLI 🪴", + add_completion=False, + no_args_is_help=True, + rich_markup_mode="rich", + pretty_exceptions_show_locals=False, + context_settings={"help_option_names": ["-h", "--help"]}, + ) + + @cli.callback() + def common( + ctx: typer.Context, + version: bool = typer.Option( + None, + "-V", + "--version", + help="Nebari version number", + callback=self._version_callback, + ), + extra_stages: typing.List[str] = typer.Option( + [], + "--import-plugin", + help="Import nebari plugin", + ), + extra_subcommands: typing.List[str] = typer.Option( + [], + "--import-subcommand", + help="Import nebari subcommand", + ), + excluded_stages: typing.List[str] = typer.Option( + [], + "--exclude-stage", + help="Exclude nebari stage(s) by name or regex", + ), + exclude_default_stages: bool = typer.Option( + False, + "--exclude-default-stages", + help="Exclude default nebari included stages", + ), + ): + try: + self.load_stages(extra_stages) + self.load_subcommands(extra_subcommands) + except ModuleNotFoundError: + typer.echo( + "ERROR: Python module {e.name} not found. Make sure that the module is in your python path {sys.path}" + ) + typer.Exit() + + self.exclude_default_stages = exclude_default_stages + self.exclude_stages = excluded_stages + self.ordered_stages = self.get_available_stages() + self.config_schema = self.extend_schema() + + @cli.command() + def info(ctx: typer.Context): + """ + Display the version and available hooks for Nebari. + """ + rich.print(f"Nebari version: {__version__}") + + hooks = collections.defaultdict(list) + for plugin in self.plugin_manager.get_plugins(): + for hook in self.plugin_manager.get_hookcallers(plugin): + hooks[hook.name].append(plugin.__name__) + + table = Table(title="Hooks") + table.add_column("hook", justify="left", no_wrap=True) + table.add_column("module", justify="left", no_wrap=True) + + for hook_name, modules in hooks.items(): + for module in modules: + table.add_row(hook_name, module) + + rich.print(table) -if not hasattr(sys, "_called_from_test"): - # Only load plugins if not running tests - pm.load_setuptools_entrypoints("nebari") + table = Table(title="Runtime Stage Ordering") + table.add_column("name") + table.add_column("priority") + table.add_column("module") + for stage in self.ordered_stages: + table.add_row( + stage.name, + str(stage.priority), + f"{stage.__module__}.{stage.__name__}", + ) + rich.print(table) -# Load default plugins -def load_plugins(plugins: typing.List[str]): - def _import_module_from_filename(filename: str): - module_name = f"_nebari.stages._files.{plugin.replace(os.sep, '.')}" - spec = importlib.util.spec_from_file_location(module_name, plugin) - mod = importlib.util.module_from_spec(spec) - sys.modules[module_name] = mod - spec.loader.exec_module(mod) - return mod + self.plugin_manager.hook.nebari_subcommand(cli=cli) - for plugin in plugins: - if plugin.endswith(".py"): - mod = _import_module_from_filename(plugin) - else: - mod = importlib.import_module(plugin) + self.cli = cli + self.cli() - try: - pm.register(mod, plugin) - except ValueError: - # Pluin already registered - pass + return cli -load_plugins(DEFAULT_SUBCOMMAND_PLUGINS) +nebari_plugin_manager = NebariPluginManager() diff --git a/src/nebari/schema.py b/src/nebari/schema.py index a40b86d9e..da8a62acc 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -855,6 +855,7 @@ class InitInputs(Base): kubernetes_version: typing.Union[str, None] = None ssl_cert_email: typing.Union[str, None] = None disable_prompt: bool = False + output: pathlib.Path = pathlib.Path("nebari-config.yaml") class CLIContext(Base): @@ -1101,7 +1102,11 @@ def set_config_from_environment_variables( return config -def read_configuration(config_filename: pathlib.Path, read_environment: bool = True): +def read_configuration( + config_filename: pathlib.Path, + read_environment: bool = True, + config_schema: pydantic.BaseModel = Main, +): """Read configuration from multiple sources and apply validation""" filename = pathlib.Path(config_filename) @@ -1115,7 +1120,7 @@ def read_configuration(config_filename: pathlib.Path, read_environment: bool = T ) with filename.open() as f: - config = Main(**yaml.load(f.read())) + config = config_schema(**yaml.load(f.read())) if read_environment: config = set_config_from_environment_variables(config) @@ -1127,13 +1132,14 @@ def write_configuration( config_filename: pathlib.Path, config: typing.Union[Main, typing.Dict], mode: str = "w", + config_schema: pydantic.BaseModel = Main, ): yaml = YAML() yaml.preserve_quotes = True yaml.default_flow_style = False with config_filename.open(mode) as f: - if isinstance(config, Main): + if isinstance(config, config_schema): yaml.dump(config.dict(), f) else: yaml.dump(config, f) From be03543657e3723ccb50272d6bf149efd5217404 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Tue, 27 Jun 2023 22:43:26 -0400 Subject: [PATCH 087/147] Moving schema to stages --- src/_nebari/cli.py | 85 ++ src/_nebari/initialize.py | 11 +- src/_nebari/stages/bootstrap/__init__.py | 32 + src/_nebari/stages/infrastructure/__init__.py | 344 ++++++- .../stages/kubernetes_ingress/__init__.py | 46 + .../stages/kubernetes_initialize/__init__.py | 43 +- .../stages/kubernetes_keycloak/__init__.py | 118 +++ .../stages/kubernetes_services/__init__.py | 318 ++++++- .../stages/nebari_tf_extensions/__init__.py | 40 + .../stages/terraform_state/__init__.py | 46 +- src/_nebari/subcommands/info.py | 43 + src/_nebari/subcommands/validate.py | 5 +- src/nebari/__main__.py | 5 +- src/nebari/hookspecs.py | 6 +- src/nebari/plugins.py | 168 +--- src/nebari/schema.py | 855 +----------------- 16 files changed, 1130 insertions(+), 1035 deletions(-) create mode 100644 src/_nebari/cli.py create mode 100644 src/_nebari/subcommands/info.py diff --git a/src/_nebari/cli.py b/src/_nebari/cli.py new file mode 100644 index 000000000..272ab21d4 --- /dev/null +++ b/src/_nebari/cli.py @@ -0,0 +1,85 @@ +import typing + +import typer +from typer.core import TyperGroup + +from _nebari.version import __version__ +from nebari import schema +from nebari.plugins import nebari_plugin_manager + + +class OrderCommands(TyperGroup): + def list_commands(self, ctx: typer.Context): + """Return list of commands in the order appear.""" + return list(self.commands) + + +def version_callback(value: bool): + if value: + typer.echo(__version__) + raise typer.Exit() + + +def exclude_stages(ctx: typer.Context, stages: typing.List[str]): + nebari_plugin_manager.excluded_stages = stages + return stages + + +def exclude_default_stages(ctx: typer.Context, exclude_default_stages: bool): + nebari_plugin_manager.exclude_default_stages = exclude_default_stages + return exclude_default_stages + + +def import_plugin(plugins: typing.List[str]): + try: + nebari_plugin_manager.load_plugins(plugins) + except ModuleNotFoundError: + typer.echo( + "ERROR: Python module {e.name} not found. Make sure that the module is in your python path {sys.path}" + ) + typer.Exit() + return plugins + + +def create_cli(): + app = typer.Typer( + cls=OrderCommands, + help="Nebari CLI 🪴", + add_completion=False, + no_args_is_help=True, + rich_markup_mode="rich", + pretty_exceptions_show_locals=False, + context_settings={"help_option_names": ["-h", "--help"]}, + ) + + @app.callback() + def common( + ctx: typer.Context, + version: bool = typer.Option( + None, + "-V", + "--version", + help="Nebari version number", + callback=version_callback, + ), + plugins: typing.List[str] = typer.Option( + [], + "--import-plugin", + help="Import nebari plugin", + ), + excluded_stages: typing.List[str] = typer.Option( + [], + "--exclude-stage", + help="Exclude nebari stage(s) by name or regex", + ), + exclude_default_stages: bool = typer.Option( + False, + "--exclude-default-stages", + help="Exclude default nebari included stages", + ), + ): + pass + + nebari_plugin_manager.plugin_manager.hook.nebari_subcommand(cli=app) + + return app diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index d9f952975..78ebaeac9 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -2,7 +2,8 @@ import os import re import tempfile -from pathlib import Path +import string +import secrets import requests @@ -18,6 +19,12 @@ WELCOME_HEADER_TEXT = "Your open source data science platform, hosted" +def random_secure_string( + length: int = 16, chars: str = string.ascii_lowercase + string.digits +): + return "".join(secrets.choice(chars) for i in range(length)) + + def render_config( project_name: str, nebari_domain: str = None, @@ -54,7 +61,7 @@ def render_config( tempfile.gettempdir(), "NEBARI_DEFAULT_PASSWORD" ) config["security"] = { - "keycloak": {"initial_root_password": schema.random_secure_string(length=32)} + "keycloak": {"initial_root_password": random_secure_string(length=32)} } with open(default_password_filename, "w") as f: f.write(config["security"]["keycloak"]["initial_root_password"]) diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index a78dc326e..dc2ee0f65 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -1,5 +1,7 @@ import io +import enum from inspect import cleandoc +import typing from typing import Any, Dict, List from _nebari.provider.cicd.github import gen_nebari_linter, gen_nebari_ops @@ -54,10 +56,40 @@ def gen_cicd(config): return cicd_files +@schema.yaml_object(schema.yaml) +class CiEnum(str, enum.Enum): + github_actions = "github-actions" + gitlab_ci = "gitlab-ci" + none = "none" + + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_str(node.value) + + +class CICD(schema.Base): + type: CiEnum = CiEnum.none + branch: str = "main" + commit_render: bool = True + before_script: typing.List[typing.Union[str, typing.Dict]] = [] + after_script: typing.List[typing.Union[str, typing.Dict]] = [] + + +class InputSchema(schema.Base): + ci_cd: CICD = CICD() + + +class OutputSchema(schema.Base): + pass + + class BootstrapStage(NebariStage): name = "bootstrap" priority = 0 + input_schema = InputSchema + output_schema = OutputSchema + def render(self) -> Dict[str, str]: contents = {} if self.config.ci_cd.type != schema.CiEnum.none: diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 5ddcdcc74..20f7b7125 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -3,9 +3,19 @@ import pathlib import sys import tempfile +import typing from typing import Any, Dict, List, Optional +import pydantic + +from _nebari import constants from _nebari.stages.base import NebariTerraformStage +from _nebari.provider.cloud import ( + amazon_web_services, + azure_cloud, + digital_ocean, + google_cloud, +) from _nebari.stages.tf_objects import ( NebariAWSProvider, NebariGCPProvider, @@ -20,10 +30,6 @@ def get_kubeconfig_filename(): return str(pathlib.Path(tempfile.gettempdir()) / "NEBARI_KUBECONFIG") -# TODO: -# - create schema for node group for each provider - - class LocalInputVars(schema.Base): kubeconfig_filename: str = get_kubeconfig_filename() kube_context: Optional[str] @@ -33,14 +39,20 @@ class ExistingInputVars(schema.Base): kube_context: str -class BaseCloudProviderInputVars(schema.Base): - name: str - environment: str - kubeconfig_filename: str = get_kubeconfig_filename() +class DigitalOceanNodeGroup(schema.Base): + instance: str + min_nodes: int + max_nodes: int -class DigitalOceanInputVars(BaseCloudProviderInputVars, schema.DigitalOceanProvider): - pass +class DigitalOceanInputVars(schema.Base): + name: str + environment: str + region: str + tags: typing.List[str] + kubernetes_version: str + node_groups: typing.Dict[str, DigitalOceanNodeGroup] + kubeconfig_filename: str class GCPGuestAccelerators(schema.Base): @@ -175,6 +187,315 @@ def kubernetes_provider_context(kubernetes_credentials: Dict[str, str]): yield +class KeyValueDict(schema.Base): + key: str + value: str + + +class DigitalOceanNodeGroup(schema.Base): + """Representation of a node group with Digital Ocean + + - Kubernetes limits: https://docs.digitalocean.com/products/kubernetes/details/limits/ + - Available instance types: https://slugs.do-api.dev/ + """ + + instance: str + min_nodes: pydantic.conint(ge=1) = 1 + max_nodes: pydantic.conint(ge=1) = 1 + + +class DigitalOceanProvider(schema.Base): + region: str = "nyc3" + kubernetes_version: typing.Optional[str] + # Digital Ocean image slugs are listed here https://slugs.do-api.dev/ + node_groups: typing.Dict[str, DigitalOceanNodeGroup] = { + "general": DigitalOceanNodeGroup( + instance="g-8vcpu-32gb", min_nodes=1, max_nodes=1 + ), + "user": DigitalOceanNodeGroup( + instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5 + ), + "worker": DigitalOceanNodeGroup( + instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5 + ), + } + tags: typing.Optional[typing.List[str]] = [] + + @pydantic.root_validator + def _validate_kubernetes_version(cls, values): + available_kubernetes_versions = digital_ocean.kubernetes_versions( + values["region"] + ) + if ( + values["kubernetes_version"] is not None + and values["kubernetes_version"] not in available_kubernetes_versions + ): + raise ValueError( + f"\nInvalid `kubernetes-version` provided: {values['kubernetes_version']}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." + ) + else: + values["kubernetes_version"] = available_kubernetes_versions[-1] + return values + + +class GCPIPAllocationPolicy(schema.Base): + cluster_secondary_range_name: str + services_secondary_range_name: str + cluster_ipv4_cidr_block: str + services_ipv4_cidr_block: str + + +class GCPCIDRBlock(schema.Base): + cidr_block: str + display_name: str + + +class GCPMasterAuthorizedNetworksConfig(schema.Base): + cidr_blocks: typing.List[GCPCIDRBlock] + + +class GCPPrivateClusterConfig(schema.Base): + enable_private_endpoint: bool + enable_private_nodes: bool + master_ipv4_cidr_block: str + + +class GCPGuestAccelerator(schema.Base): + """ + See general information regarding GPU support at: + # TODO: replace with nebari.dev new URL + https://docs.nebari.dev/en/stable/source/admin_guide/gpu.html?#add-gpu-node-group + """ + + name: str + count: pydantic.conint(ge=1) = 1 + + +class GCPNodeGroup(schema.Base): + instance: str + min_nodes: pydantic.conint(ge=0) = 0 + max_nodes: pydantic.conint(ge=1) = 1 + preemptible: bool = False + labels: typing.Dict[str, str] = {} + guest_accelerators: typing.List[GCPGuestAccelerator] = [] + + +class GoogleCloudPlatformProvider(schema.Base): + project: str = pydantic.Field(default_factory=lambda: os.environ["PROJECT_ID"]) + region: str = "us-central1" + availability_zones: typing.Optional[typing.List[str]] = [] + kubernetes_version: typing.Optional[str] + release_channel: str = constants.DEFAULT_GKE_RELEASE_CHANNEL + node_groups: typing.Dict[str, GCPNodeGroup] = { + "general": GCPNodeGroup(instance="n1-standard-8", min_nodes=1, max_nodes=1), + "user": GCPNodeGroup(instance="n1-standard-4", min_nodes=0, max_nodes=5), + "worker": GCPNodeGroup(instance="n1-standard-4", min_nodes=0, max_nodes=5), + } + tags: typing.Optional[typing.List[str]] = [] + networking_mode: str = "ROUTE" + network: str = "default" + subnetwork: typing.Optional[typing.Union[str, None]] = None + ip_allocation_policy: typing.Optional[ + typing.Union[GCPIPAllocationPolicy, None] + ] = None + master_authorized_networks_config: typing.Optional[ + typing.Union[GCPCIDRBlock, None] + ] = None + private_cluster_config: typing.Optional[ + typing.Union[GCPPrivateClusterConfig, None] + ] = None + + @pydantic.root_validator + def _validate_kubernetes_version(cls, values): + available_kubernetes_versions = google_cloud.kubernetes_versions( + values["region"] + ) + if ( + values["kubernetes_version"] is not None + and values["kubernetes_version"] not in available_kubernetes_versions + ): + raise ValueError( + f"\nInvalid `kubernetes-version` provided: {values['kubernetes_version']}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." + ) + else: + values["kubernetes_version"] = available_kubernetes_versions[-1] + return values + + +class AzureNodeGroup(schema.Base): + instance: str + min_nodes: int + max_nodes: int + + +class AzureProvider(schema.Base): + region: str = "Central US" + kubernetes_version: typing.Optional[str] + node_groups: typing.Dict[str, AzureNodeGroup] = { + "general": AzureNodeGroup(instance="Standard_D8_v3", min_nodes=1, max_nodes=1), + "user": AzureNodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), + "worker": AzureNodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), + } + storage_account_postfix: str = pydantic.Field( + default_factory=lambda: random_secure_string(length=4) + ) + vnet_subnet_id: typing.Optional[typing.Union[str, None]] = None + private_cluster_enabled: bool = False + + @pydantic.validator("kubernetes_version") + def _validate_kubernetes_version(cls, value): + available_kubernetes_versions = azure_cloud.kubernetes_versions() + if value is None: + value = available_kubernetes_versions[-1] + elif value not in available_kubernetes_versions: + raise ValueError( + f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." + ) + return value + + +class AWSNodeGroup(schema.Base): + instance: str + min_nodes: int = 0 + max_nodes: int + gpu: bool = False + single_subnet: bool = False + + +class AmazonWebServicesProvider(schema.Base): + region: str = pydantic.Field( + default_factory=lambda: os.environ.get("AWS_DEFAULT_REGION", "us-west-2") + ) + availability_zones: typing.Optional[typing.List[str]] + kubernetes_version: typing.Optional[str] + node_groups: typing.Dict[str, AWSNodeGroup] = { + "general": AWSNodeGroup(instance="m5.2xlarge", min_nodes=1, max_nodes=1), + "user": AWSNodeGroup( + instance="m5.xlarge", min_nodes=1, max_nodes=5, single_subnet=False + ), + "worker": AWSNodeGroup( + instance="m5.xlarge", min_nodes=1, max_nodes=5, single_subnet=False + ), + } + existing_subnet_ids: typing.List[str] = None + existing_security_group_ids: str = None + vpc_cidr_block: str = "10.10.0.0/16" + + @pydantic.root_validator + def _validate_kubernetes_version(cls, values): + available_kubernetes_versions = amazon_web_services.kubernetes_versions() + if values["kubernetes_version"] is None: + values["kubernetes_version"] = available_kubernetes_versions[-1] + elif values["kubernetes_version"] not in available_kubernetes_versions: + raise ValueError( + f"\nInvalid `kubernetes-version` provided: {values['kubernetes_version']}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." + ) + return values + + @pydantic.root_validator + def _validate_availability_zones(cls, values): + if values["availability_zones"] is None: + zones = amazon_web_services.zones(values["region"]) + values["availability_zones"] = list(sorted(zones))[:2] + return values + + +class LocalProvider(schema.Base): + kube_context: typing.Optional[str] + node_selectors: typing.Dict[str, KeyValueDict] = { + "general": KeyValueDict(key="kubernetes.io/os", value="linux"), + "user": KeyValueDict(key="kubernetes.io/os", value="linux"), + "worker": KeyValueDict(key="kubernetes.io/os", value="linux"), + } + + +class ExistingProvider(schema.Base): + kube_context: typing.Optional[str] + node_selectors: typing.Dict[str, KeyValueDict] = { + "general": KeyValueDict(key="kubernetes.io/os", value="linux"), + "user": KeyValueDict(key="kubernetes.io/os", value="linux"), + "worker": KeyValueDict(key="kubernetes.io/os", value="linux"), + } + + +class InputSchema(schema.Base): + provider: schema.ProviderEnum = schema.ProviderEnum.local + local: typing.Optional[LocalProvider] + existing: typing.Optional[ExistingProvider] + google_cloud_platform: typing.Optional[GoogleCloudPlatformProvider] + amazon_web_services: typing.Optional[AmazonWebServicesProvider] + azure: typing.Optional[AzureProvider] + digital_ocean: typing.Optional[DigitalOceanProvider] + + @pydantic.root_validator + def check_provider(cls, values): + if values["provider"] == schema.ProviderEnum.local and values.get("local") is None: + values["local"] = LocalProvider() + elif ( + values["provider"] == schema.ProviderEnum.existing + and values.get("existing") is None + ): + values["existing"] = ExistingProvider() + elif ( + values["provider"] == schema.ProviderEnum.gcp + and values.get("google_cloud_platform") is None + ): + values["google_cloud_platform"] = GoogleCloudPlatformProvider() + elif ( + values["provider"] == schema.ProviderEnum.aws + and values.get("amazon_web_services") is None + ): + values["amazon_web_services"] = schema.AmazonWebServicesProvider() + elif values["provider"] == schema.ProviderEnum.azure and values.get("azure") is None: + values["azure"] = AzureProvider() + elif ( + values["provider"] == schema.ProviderEnum.do + and values.get("digital_ocean") is None + ): + values["digital_ocean"] = DigitalOceanProvider() + + if ( + sum( + (_ in values and values[_] is not None) + for _ in { + "local", + "existing", + "google_cloud_platform", + "amazon_web_services", + "azure", + "digital_ocean", + } + ) + != 1 + ): + raise ValueError("multiple providers set or wrong provider fields set") + return values + + +class NodeSelectorKeyValue(schema.Base): + key: str + value: str + + +class KubernetesCredentials(schema.Base): + host: str + cluster_ca_certifiate: str + token: typing.Optional[str] + username: typing.Optional[str] + password: typing.Optional[str] + client_certificate: typing.Optional[str] + client_key: typing.Optional[str] + config_path: typing.Optional[str] + config_context: typing.Optional[str] + + +class OutputSchema(schema.Base): + node_selectors: Dict[str, NodeSelectorKeyValue] + kubernetes_credentials: KubernetesCredentials + kubeconfig_filename: str + nfs_endpoint: typing.Optional[str] + + class KubernetesInfrastructureStage(NebariTerraformStage): """Generalized method to provision infrastructure. @@ -194,6 +515,9 @@ class KubernetesInfrastructureStage(NebariTerraformStage): name = "02-infrastructure" priority = 20 + input_schema = InputSchema + output_schema = OutputSchema + @property def template_directory(self): return ( diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index bda8d6795..33761754d 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -1,6 +1,8 @@ import socket +import enum import sys import time +import typing from typing import Any, Dict, List from _nebari import constants @@ -126,10 +128,54 @@ def _attempt_dns_lookup( sys.exit(1) +@schema.yaml_object(schema.yaml) +class CertificateEnum(str, enum.Enum): + letsencrypt = "lets-encrypt" + selfsigned = "self-signed" + existing = "existing" + disabled = "disabled" + + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_str(node.value) + + +class Certificate(schema.Base): + type: CertificateEnum = CertificateEnum.selfsigned + # existing + secret_name: typing.Optional[str] + # lets-encrypt + acme_email: typing.Optional[str] + acme_server: str = "https://acme-v02.api.letsencrypt.org/directory" + + +class Ingress(schema.Base): + terraform_overrides: typing.Dict = {} + + +class InputSchema(schema.Base): + domain: typing.Optional[str] + certificate: Certificate = Certificate() + ingress: Ingress = Ingress() + + +class IngressEndpoint(schema.Base): + ip: str + hostname: str + + +class OutputSchema(schema.Base): + load_balancer_address: typing.List[IngressEndpoint] + domain: str + + class KubernetesIngressStage(NebariTerraformStage): name = "04-kubernetes-ingress" priority = 40 + input_schema = InputSchema + output_schema = OutputSchema + def tf_objects(self) -> List[Dict]: return [ NebariTerraformState(self.name, self.config), diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py index 202d8b875..02f8df6f9 100644 --- a/src/_nebari/stages/kubernetes_initialize/__init__.py +++ b/src/_nebari/stages/kubernetes_initialize/__init__.py @@ -1,6 +1,9 @@ import sys +import typing from typing import Any, Dict, List, Union +import pydantic + from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( NebariHelmProvider, @@ -11,20 +14,58 @@ from nebari.hookspecs import NebariStage, hookimpl +class ExtContainerReg(schema.Base): + enabled: bool = False + access_key_id: typing.Optional[str] + secret_access_key: typing.Optional[str] + extcr_account: typing.Optional[str] + extcr_region: typing.Optional[str] + + @pydantic.root_validator + def enabled_must_have_fields(cls, values): + if values["enabled"]: + for fldname in ( + "access_key_id", + "secret_access_key", + "extcr_account", + "extcr_region", + ): + if ( + fldname not in values + or values[fldname] is None + or values[fldname].strip() == "" + ): + raise ValueError( + f"external_container_reg must contain a non-blank {fldname} when enabled is true" + ) + return values + + class InputVars(schema.Base): name: str environment: str cloud_provider: str aws_region: Union[str, None] = None - external_container_reg: Union[schema.ExtContainerReg, None] = None + external_container_reg: Union[ExtContainerReg, None] = None gpu_enabled: bool = False gpu_node_group_names: List[str] = [] +class InputSchema(schema.Base): + external_container_reg: ExtContainerReg = ExtContainerReg() + + +class OutputSchema(schema.Base): + pass + + class KubernetesInitializeStage(NebariTerraformStage): name = "03-kubernetes-initialize" priority = 30 + input_schema = InputSchema + output_schema = OutputSchema + def tf_objects(self) -> List[Dict]: return [ NebariTerraformState(self.name, self.config), diff --git a/src/_nebari/stages/kubernetes_keycloak/__init__.py b/src/_nebari/stages/kubernetes_keycloak/__init__.py index 1eb9e0080..869d8eca9 100644 --- a/src/_nebari/stages/kubernetes_keycloak/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak/__init__.py @@ -1,8 +1,15 @@ +import enum +import secrets +import string import contextlib import json import sys import time +import typing from typing import Any, Dict, List +from abc import ABC + +import pydantic from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( @@ -42,10 +49,121 @@ def keycloak_provider_context(keycloak_credentials: Dict[str, str]): yield +@schema.yaml_object(schema.yaml) +class AuthenticationEnum(str, enum.Enum): + password = "password" + github = "GitHub" + auth0 = "Auth0" + custom = "custom" + + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_str(node.value) + + +class GitHubConfig(schema.Base): + client_id: str + client_secret: str + + +class Auth0Config(schema.Base): + client_id: str + client_secret: str + auth0_subdomain: str + + +class Authentication(schema.Base, ABC): + _types: typing.Dict[str, type] = {} + + type: AuthenticationEnum + + # Based on https://github.com/samuelcolvin/pydantic/issues/2177#issuecomment-739578307 + + # This allows type field to determine which subclass of Authentication should be used for validation. + + # Used to register automatically all the submodels in `_types`. + def __init_subclass__(cls): + cls._types[cls._typ.value] = cls + + @classmethod + def __get_validators__(cls): + yield cls.validate + + @classmethod + def validate(cls, value: typing.Dict[str, typing.Any]) -> "Authentication": + if "type" not in value: + raise ValueError("type field is missing from security.authentication") + + specified_type = value.get("type") + sub_class = cls._types.get(specified_type, None) + + if not sub_class: + raise ValueError( + f"No registered Authentication type called {specified_type}" + ) + + # init with right submodel + return sub_class(**value) + + +def random_secure_string( + length: int = 16, chars: str = string.ascii_lowercase + string.digits +): + return "".join(secrets.choice(chars) for i in range(length)) + + +class PasswordAuthentication(Authentication): + _typ = AuthenticationEnum.password + + +class Auth0Authentication(Authentication): + _typ = AuthenticationEnum.auth0 + config: Auth0Config + + +class GitHubAuthentication(Authentication): + _typ = AuthenticationEnum.github + config: GitHubConfig + + +class Keycloak(schema.Base): + initial_root_password: str = pydantic.Field(default_factory=random_secure_string) + overrides: typing.Dict = {} + realm_display_name: str = "Nebari" + + +class Security(schema.Base): + authentication: Authentication = PasswordAuthentication( + type=AuthenticationEnum.password + ) + shared_users_group: bool = True + keycloak: Keycloak = Keycloak() + + +class InputSchema(schema.Base): + security: Security = Security() + + +class KeycloakCredentials(schema.Base): + url: str + client_id: str + realm: str + username: str + password: str + + +class OutputSchema(schema.Base): + keycloak_credentials: KeycloakCredentials + keycloak_nebari_bot_password: str + + class KubernetesKeycloakStage(NebariTerraformStage): name = "05-kubernetes-keycloak" priority = 50 + input_schema = InputSchema + output_schema = OutputSchema + def tf_objects(self) -> List[Dict]: return [ NebariTerraformState(self.name, self.config), diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index d34c93dcc..a85667656 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -1,10 +1,16 @@ +import enum +import os import json import sys +import typing from typing import Any, Dict, List from urllib.parse import urlencode +import pydantic from pydantic import Field +from _nebari.version import __version__ +from _nebari import constants from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( NebariHelmProvider, @@ -19,6 +25,311 @@ TIMEOUT = 10 # seconds +def set_docker_image_tag() -> str: + """Set docker image tag for `jupyterlab`, `jupyterhub`, and `dask-worker`.""" + return os.environ.get("NEBARI_IMAGE_TAG", constants.DEFAULT_NEBARI_IMAGE_TAG) + + +def set_nebari_dask_version() -> str: + """Set version of `nebari-dask` meta package.""" + return os.environ.get("NEBARI_DASK_VERSION", constants.DEFAULT_NEBARI_DASK_VERSION) + + +@schema.yaml_object(schema.yaml) +class AccessEnum(str, enum.Enum): + all = "all" + yaml = "yaml" + keycloak = "keycloak" + + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_str(node.value) + + +class Prefect(schema.Base): + enabled: bool = False + image: typing.Optional[str] + overrides: typing.Dict = {} + token: typing.Optional[str] + + +class CDSDashboards(schema.Base): + enabled: bool = True + cds_hide_user_named_servers: bool = True + cds_hide_user_dashboard_servers: bool = False + + +class DefaultImages(schema.Base): + jupyterhub: str = f"quay.io/nebari/nebari-jupyterhub:{set_docker_image_tag()}" + jupyterlab: str = f"quay.io/nebari/nebari-jupyterlab:{set_docker_image_tag()}" + dask_worker: str = f"quay.io/nebari/nebari-dask-worker:{set_docker_image_tag()}" + + +class Storage(schema.Base): + conda_store: str = "200Gi" + shared_filesystem: str = "200Gi" + + +class JupyterHubTheme(schema.Base): + hub_title: str = "Nebari" + hub_subtitle: str = "Your open source data science platform" + welcome: str = """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""" + logo: str = "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg" + primary_color: str = "#4f4173" + secondary_color: str = "#957da6" + accent_color: str = "#32C574" + text_color: str = "#111111" + h1_color: str = "#652e8e" + h2_color: str = "#652e8e" + version: str = f"v{__version__}" + display_version: str = "True" # limitation of theme everything is a str + + +class Theme(schema.Base): + jupyterhub: JupyterHubTheme = JupyterHubTheme() + + +class KubeSpawner(schema.Base): + cpu_limit: int + cpu_guarantee: int + mem_limit: str + mem_guarantee: str + + class Config: + extra = "allow" + + +class JupyterLabProfile(schema.Base): + access: AccessEnum = AccessEnum.all + display_name: str + description: str + default: bool = False + users: typing.Optional[typing.List[str]] + groups: typing.Optional[typing.List[str]] + kubespawner_override: typing.Optional[KubeSpawner] + + @pydantic.root_validator + def only_yaml_can_have_groups_and_users(cls, values): + if values["access"] != AccessEnum.yaml: + if ( + values.get("users", None) is not None + or values.get("groups", None) is not None + ): + raise ValueError( + "Profile must not contain groups or users fields unless access = yaml" + ) + return values + + +class DaskWorkerProfile(schema.Base): + worker_cores_limit: int + worker_cores: int + worker_memory_limit: str + worker_memory: str + image: typing.Optional[str] + + class Config: + extra = "allow" + + +class Profiles(schema.Base): + jupyterlab: typing.List[JupyterLabProfile] = [ + JupyterLabProfile( + display_name="Small Instance", + description="Stable environment with 2 cpu / 8 GB ram", + default=True, + kubespawner_override=KubeSpawner( + cpu_limit=2, + cpu_guarantee=1.5, + mem_limit="8G", + mem_guarantee="5G", + ), + ), + JupyterLabProfile( + display_name="Medium Instance", + description="Stable environment with 4 cpu / 16 GB ram", + kubespawner_override=KubeSpawner( + cpu_limit=4, + cpu_guarantee=3, + mem_limit="16G", + mem_guarantee="10G", + ), + ), + ] + dask_worker: typing.Dict[str, DaskWorkerProfile] = { + "Small Worker": DaskWorkerProfile( + worker_cores_limit=2, + worker_cores=1.5, + worker_memory_limit="8G", + worker_memory="5G", + worker_threads=2, + ), + "Medium Worker": DaskWorkerProfile( + worker_cores_limit=4, + worker_cores=3, + worker_memory_limit="16G", + worker_memory="10G", + worker_threads=4, + ), + } + + @pydantic.validator("jupyterlab") + def check_default(cls, v, values): + """Check if only one default value is present.""" + default = [attrs["default"] for attrs in v if "default" in attrs] + if default.count(True) > 1: + raise TypeError( + "Multiple default Jupyterlab profiles may cause unexpected problems." + ) + return v + + +class CondaEnvironment(schema.Base): + name: str + channels: typing.Optional[typing.List[str]] + dependencies: typing.List[typing.Union[str, typing.Dict[str, typing.List[str]]]] + + +class CondaStore(schema.Base): + extra_settings: typing.Dict[str, typing.Any] = {} + extra_config: str = "" + image: str = "quansight/conda-store-server" + image_tag: str = constants.DEFAULT_CONDA_STORE_IMAGE_TAG + default_namespace: str = "nebari-git" + object_storage: str = "200Gi" + + +class NebariWorkflowController(schema.Base): + enabled: bool = True + image_tag: str = constants.DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG + + +class ArgoWorkflows(schema.Base): + enabled: bool = True + overrides: typing.Dict = {} + nebari_workflow_controller: NebariWorkflowController = NebariWorkflowController() + + +class KBatch(schema.Base): + enabled: bool = True + + +class Monitoring(schema.Base): + enabled: bool = True + + +class ClearML(schema.Base): + enabled: bool = False + enable_forward_auth: bool = False + overrides: typing.Dict = {} + + +class JupyterHub(schema.Base): + overrides: typing.Dict = {} + + + +class IdleCuller(schema.Base): + terminal_cull_inactive_timeout: int = 15 + terminal_cull_interval: int = 5 + kernel_cull_idle_timeout: int = 15 + kernel_cull_interval: int = 5 + kernel_cull_connected: bool = True + kernel_cull_busy: bool = False + server_shutdown_no_activity_timeout: int = 15 + + +class JupyterLab(schema.Base): + idle_culler: IdleCuller = IdleCuller() + + +class InputSchema(schema.Base): + prefect: Prefect = Prefect() + cdsdashboards: CDSDashboards = CDSDashboards() + default_images: DefaultImages = DefaultImages() + storage: Storage = Storage() + theme: Theme = Theme() + profiles: Profiles = Profiles() + environments: typing.Dict[str, CondaEnvironment] = { + "environment-dask.yaml": CondaEnvironment( + name="dask", + channels=["conda-forge"], + dependencies=[ + "python=3.10.8", + "ipykernel=6.21.0", + "ipywidgets==7.7.1", + f"nebari-dask =={set_nebari_dask_version()}", + "python-graphviz=0.20.1", + "pyarrow=10.0.1", + "s3fs=2023.1.0", + "gcsfs=2023.1.0", + "numpy=1.23.5", + "numba=0.56.4", + "pandas=1.5.3", + { + "pip": [ + "kbatch==0.4.1", + ], + }, + ], + ), + "environment-dashboard.yaml": CondaEnvironment( + name="dashboard", + channels=["conda-forge"], + dependencies=[ + "python=3.10", + "cdsdashboards-singleuser=0.6.3", + "cufflinks-py=0.17.3", + "dash=2.8.1", + "geopandas=0.12.2", + "geopy=2.3.0", + "geoviews=1.9.6", + "gunicorn=20.1.0", + "holoviews=1.15.4", + "ipykernel=6.21.2", + "ipywidgets=8.0.4", + "jupyter=1.0.0", + "jupyterlab=3.6.1", + "jupyter_bokeh=3.0.5", + "matplotlib=3.7.0", + f"nebari-dask=={set_nebari_dask_version()}", + "nodejs=18.12.1", + "numpy", + "openpyxl=3.1.1", + "pandas=1.5.3", + "panel=0.14.3", + "param=1.12.3", + "plotly=5.13.0", + "python-graphviz=0.20.1", + "rich=13.3.1", + "streamlit=1.9.0", + "sympy=1.11.1", + "voila=0.4.0", + "pip=23.0", + { + "pip": [ + "streamlit-image-comparison==0.0.3", + "noaa-coops==0.2.1", + "dash_core_components==2.0.0", + "dash_html_components==2.0.0", + ], + }, + ], + ), + } + conda_store: CondaStore = CondaStore() + argo_workflows: ArgoWorkflows = ArgoWorkflows() + kbatch: KBatch = KBatch() + monitoring: Monitoring = Monitoring() + clearml: ClearML = ClearML() + jupyterhub: JupyterHub = JupyterHub() + jupyterlab: JupyterLab = JupyterLab() + + +class OutputSchema(schema.Base): + pass + + # variables shared by multiple services class KubernetesServicesInputVars(schema.Base): name: str @@ -40,7 +351,7 @@ class ImageNameTag(schema.Base): class CondaStoreInputVars(schema.Base): - conda_store_environments: Dict[str, schema.CondaEnvironment] = Field( + conda_store_environments: Dict[str, CondaEnvironment] = Field( alias="conda-store-environments" ) conda_store_default_namespace: str = Field(alias="conda-store-default-namespace") @@ -64,7 +375,7 @@ class JupyterhubInputVars(schema.Base): jupyterhub_overrides: List[str] = Field(alias="jupyterhub-overrides") jupyterhub_stared_storage: str = Field(alias="jupyterhub-shared-storage") jupyterhub_shared_endpoint: str = Field(None, alias="jupyterhub-shared-endpoint") - jupyterhub_profiles: List[schema.JupyterLabProfile] = Field( + jupyterhub_profiles: List[JupyterLabProfile] = Field( alias="jupyterlab-profiles" ) jupyterhub_image: ImageNameTag = Field(alias="jupyterhub-image") @@ -112,6 +423,9 @@ class KubernetesServicesStage(NebariTerraformStage): name = "07-kubernetes-services" priority = 70 + input_schema = InputSchema + output_schema = OutputSchema + def tf_objects(self) -> List[Dict]: return [ NebariTerraformState(self.name, self.config), diff --git a/src/_nebari/stages/nebari_tf_extensions/__init__.py b/src/_nebari/stages/nebari_tf_extensions/__init__.py index ca50def3a..4e4c0515e 100644 --- a/src/_nebari/stages/nebari_tf_extensions/__init__.py +++ b/src/_nebari/stages/nebari_tf_extensions/__init__.py @@ -1,3 +1,4 @@ +import typing from typing import Any, Dict, List from _nebari.stages.base import NebariTerraformStage @@ -7,12 +8,51 @@ NebariTerraformState, ) from nebari.hookspecs import NebariStage, hookimpl +from nebari import schema + + +class NebariExtensionEnv(schema.Base): + name: str + value: str + + +class NebariExtension(schema.Base): + name: str + image: str + urlslug: str + private: bool = False + oauth2client: bool = False + keycloakadmin: bool = False + jwt: bool = False + nebariconfigyaml: bool = False + logout: typing.Optional[str] + envs: typing.Optional[typing.List[NebariExtensionEnv]] + + +class HelmExtension(schema.Base): + name: str + repository: str + chart: str + version: str + overrides: typing.Dict = {} + + +class InputSchema(schema.Base): + helm_extensions: typing.List[HelmExtension] = [] + tf_extensions: typing.List[NebariExtension] = [] + + +class OutputSchema(schema.Base): + pass class NebariTFExtensionsStage(NebariTerraformStage): name = "08-nebari-tf-extensions" priority = 80 + input_schema = InputSchema + output_schema = OutputSchema + def tf_objects(self) -> List[Dict]: return [ NebariTerraformState(self.name, self.config), diff --git a/src/_nebari/stages/terraform_state/__init__.py b/src/_nebari/stages/terraform_state/__init__.py index fdb6dcd08..a7944d8b1 100644 --- a/src/_nebari/stages/terraform_state/__init__.py +++ b/src/_nebari/stages/terraform_state/__init__.py @@ -1,6 +1,8 @@ +import enum import inspect import os import pathlib +import typing from typing import Any, Dict, List, Tuple from _nebari.stages.base import NebariTerraformStage @@ -8,26 +10,53 @@ from nebari.hookspecs import NebariStage, hookimpl -class BaseCloudProviderInputVars(schema.Base): +class DigitalOceanInputVars(schema.Base): name: str namespace: str - - -class DigitalOceanInputVars(BaseCloudProviderInputVars): region: str -class GCPInputVars(BaseCloudProviderInputVars): +class GCPInputVars(schema.Base): + name: str + namespace: str region: str -class AzureInputVars(BaseCloudProviderInputVars): +class AzureInputVars(schema.Base): + name: str + namespace: str region: str storage_account_postfix: str state_resource_group_name: str -class AWSInputVars(BaseCloudProviderInputVars): +class AWSInputVars(schema.Base): + name: str + namespace: str + + +@schema.yaml_object(schema.yaml) +class TerraformStateEnum(str, enum.Enum): + remote = "remote" + local = "local" + existing = "existing" + + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_str(node.value) + + +class TerraformState(schema.Base): + type: TerraformStateEnum = TerraformStateEnum.remote + backend: typing.Optional[str] + config: typing.Dict[str, str] = {} + + +class InputSchema(schema.Base): + terraform_state: TerraformState = TerraformState() + + +class OutputSchema(schema.Base): pass @@ -35,6 +64,9 @@ class TerraformStateStage(NebariTerraformStage): name = "01-terraform-state" priority = 10 + input_schema = InputSchema + output_schema = OutputSchema + @property def template_directory(self): return ( diff --git a/src/_nebari/subcommands/info.py b/src/_nebari/subcommands/info.py new file mode 100644 index 000000000..1a36afceb --- /dev/null +++ b/src/_nebari/subcommands/info.py @@ -0,0 +1,43 @@ +import collections + +import rich +import typer +from rich.table import Table + +from _nebari.version import __version__ +from nebari.hookspecs import hookimpl + + +@hookimpl +def nebari_subcommand(cli: typer.Typer): + @cli.command() + def info(ctx: typer.Context): + from nebari.plugins import nebari_plugin_manager + + rich.print(f"Nebari version: {__version__}") + + hooks = collections.defaultdict(list) + for plugin in nebari_plugin_manager.plugin_manager.get_plugins(): + for hook in nebari_plugin_manager.plugin_manager.get_hookcallers(plugin): + hooks[hook.name].append(plugin.__name__) + + table = Table(title="Hooks") + table.add_column("hook", justify="left", no_wrap=True) + table.add_column("module", justify="left", no_wrap=True) + + for hook_name, modules in hooks.items(): + for module in modules: + table.add_row(hook_name, module) + + rich.print(table) + + table = Table(title="Runtime Stage Ordering") + table.add_column("name") + table.add_column("priority") + table.add_column("module") + for stage in nebari_plugin_manager.ordered_stages: + table.add_row( + stage.name, str(stage.priority), f"{stage.__module__}.{stage.__name__}" + ) + + rich.print(table) diff --git a/src/_nebari/subcommands/validate.py b/src/_nebari/subcommands/validate.py index 5dfa81308..465aaf643 100644 --- a/src/_nebari/subcommands/validate.py +++ b/src/_nebari/subcommands/validate.py @@ -30,8 +30,5 @@ def validate( pass else: from nebari.plugins import nebari_plugin_manager - - config_schema = nebari_plugin_manager.config_schema - - schema.read_configuration(config_filename, config_schema=config_schema) + schema.read_configuration(config_filename, config_schema=nebari_plugin_manager.config_schema) print("[bold purple]Successfully validated configuration.[/bold purple]") diff --git a/src/nebari/__main__.py b/src/nebari/__main__.py index 4a884d3c9..b18eaf428 100644 --- a/src/nebari/__main__.py +++ b/src/nebari/__main__.py @@ -1,8 +1,9 @@ -from nebari.plugins import nebari_plugin_manager +from _nebari.cli import create_cli def main(): - nebari_plugin_manager.create_cli() + cli = create_cli() + cli() if __name__ == "__main__": diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index 7e4c47009..72990774c 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -4,7 +4,7 @@ import typer from pluggy import HookimplMarker, HookspecMarker -from pydantic import BaseModel +import pydantic from nebari import schema @@ -15,7 +15,9 @@ class NebariStage: name: str = None priority: int = None - stage_schema: BaseModel = None + + input_schema: pydantic.BaseModel = None + output_schema: pydantic.BaseModel = None def __init__(self, output_directory: pathlib.Path, config: schema.Main): self.output_directory = output_directory diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index 465ceb27e..205c6ffd1 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -1,4 +1,3 @@ -import collections import importlib import itertools import os @@ -8,17 +7,15 @@ from pathlib import Path import pluggy -import rich -import typer -from pydantic import BaseModel, create_model -from rich.table import Table -from typer.core import TyperGroup +import pydantic from _nebari.version import __version__ from nebari import hookspecs, schema + DEFAULT_SUBCOMMAND_PLUGINS = [ # subcommands + "_nebari.subcommands.info", "_nebari.subcommands.init", "_nebari.subcommands.dev", "_nebari.subcommands.deploy", @@ -47,16 +44,11 @@ class NebariPluginManager: plugin_manager = pluggy.PluginManager("nebari") - ordered_stages: typing.List[hookspecs.NebariStage] = [] exclude_default_stages: bool = False exclude_stages: typing.List[str] = [] - cli: typer.Typer = None - - schema_name: str = "NebariConfig" - config_schema: typing.Union[BaseModel, None] = None config_path: typing.Union[Path, None] = None - config: typing.Union[BaseModel, None] = None + config: typing.Union[pydantic.BaseModel, None] = None def __init__(self) -> None: self.plugin_manager.add_hookspecs(hookspecs) @@ -65,17 +57,9 @@ def __init__(self) -> None: # Only load plugins if not running tests self.plugin_manager.load_setuptools_entrypoints("nebari") - self.load_subcommands(DEFAULT_SUBCOMMAND_PLUGINS) - self.ordered_stages = self.get_available_stages() - self.config_schema = self.extend_schema() - - def load_subcommands(self, subcommand: typing.List[str]): - self._load_plugins(subcommand) + self.load_plugins(DEFAULT_SUBCOMMAND_PLUGINS) - def load_stages(self, stages: typing.List[str]): - self._load_plugins(stages) - - def _load_plugins(self, plugins: typing.List[str]): + def load_plugins(self, plugins: typing.List[str]): def _import_module_from_filename(plugin: str): module_name = f"_nebari.stages._files.{plugin.replace(os.sep, '.')}" spec = importlib.util.spec_from_file_location(module_name, plugin) @@ -98,7 +82,7 @@ def _import_module_from_filename(plugin: str): def get_available_stages(self): if not self.exclude_default_stages: - self.load_stages(DEFAULT_STAGES_PLUGINS) + self.load_plugins(DEFAULT_STAGES_PLUGINS) stages = itertools.chain.from_iterable(self.plugin_manager.hook.nebari_stage()) @@ -135,136 +119,16 @@ def load_config(self, config_path: typing.Union[str, Path]): self.config_path = config_path self.config = schema.read_configuration(config_path) - def _create_dynamic_schema( - self, base: BaseModel, stage: BaseModel, stage_name: str - ) -> BaseModel: - stage_fields = { - n: (f.type_, f.default if f.default is not None else ...) - for n, f in stage.__fields__.items() - } - # ensure top-level key for `stage` is set to `stage_name` - stage_model = create_model(stage_name, __base__=schema.Base, **stage_fields) - extra_fields = {stage_name: (stage_model, None)} - return create_model(self.schema_name, __base__=base, **extra_fields) - - def extend_schema(self, base_schema: BaseModel = schema.Main) -> BaseModel: - config_schema = base_schema - for stages in self.ordered_stages: - if stages.stage_schema: - config_schema = self._create_dynamic_schema( - config_schema, - stages.stage_schema, - stages.name, - ) - return config_schema - - def _version_callback(self, value: bool): - if value: - typer.echo(__version__) - raise typer.Exit() - - def create_cli(self) -> typer.Typer: - class OrderCommands(TyperGroup): - def list_commands(self, ctx: typer.Context): - """Return list of commands in the order appear.""" - return list(self.commands) - - cli = typer.Typer( - cls=OrderCommands, - help="Nebari CLI 🪴", - add_completion=False, - no_args_is_help=True, - rich_markup_mode="rich", - pretty_exceptions_show_locals=False, - context_settings={"help_option_names": ["-h", "--help"]}, - ) - - @cli.callback() - def common( - ctx: typer.Context, - version: bool = typer.Option( - None, - "-V", - "--version", - help="Nebari version number", - callback=self._version_callback, - ), - extra_stages: typing.List[str] = typer.Option( - [], - "--import-plugin", - help="Import nebari plugin", - ), - extra_subcommands: typing.List[str] = typer.Option( - [], - "--import-subcommand", - help="Import nebari subcommand", - ), - excluded_stages: typing.List[str] = typer.Option( - [], - "--exclude-stage", - help="Exclude nebari stage(s) by name or regex", - ), - exclude_default_stages: bool = typer.Option( - False, - "--exclude-default-stages", - help="Exclude default nebari included stages", - ), - ): - try: - self.load_stages(extra_stages) - self.load_subcommands(extra_subcommands) - except ModuleNotFoundError: - typer.echo( - "ERROR: Python module {e.name} not found. Make sure that the module is in your python path {sys.path}" - ) - typer.Exit() - - self.exclude_default_stages = exclude_default_stages - self.exclude_stages = excluded_stages - self.ordered_stages = self.get_available_stages() - self.config_schema = self.extend_schema() - - @cli.command() - def info(ctx: typer.Context): - """ - Display the version and available hooks for Nebari. - """ - rich.print(f"Nebari version: {__version__}") - - hooks = collections.defaultdict(list) - for plugin in self.plugin_manager.get_plugins(): - for hook in self.plugin_manager.get_hookcallers(plugin): - hooks[hook.name].append(plugin.__name__) - - table = Table(title="Hooks") - table.add_column("hook", justify="left", no_wrap=True) - table.add_column("module", justify="left", no_wrap=True) - - for hook_name, modules in hooks.items(): - for module in modules: - table.add_row(hook_name, module) - - rich.print(table) - - table = Table(title="Runtime Stage Ordering") - table.add_column("name") - table.add_column("priority") - table.add_column("module") - for stage in self.ordered_stages: - table.add_row( - stage.name, - str(stage.priority), - f"{stage.__module__}.{stage.__name__}", - ) - - rich.print(table) - - self.plugin_manager.hook.nebari_subcommand(cli=cli) - - self.cli = cli - self.cli() + @property + def ordered_stages(self): + return self.get_available_stages() - return cli + @property + def config_schema(self): + classes = [schema.Main] + [ + _.input_schema for _ in self.ordered_stages if _.input_schema is not None + ] + return type('ConfigSchema', tuple(classes), {}) nebari_plugin_manager = NebariPluginManager() diff --git a/src/nebari/schema.py b/src/nebari/schema.py index da8a62acc..2f65c9484 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -13,12 +13,6 @@ from ruamel.yaml import YAML, yaml_object from _nebari import constants -from _nebari.provider.cloud import ( - amazon_web_services, - azure_cloud, - digital_ocean, - google_cloud, -) from _nebari.version import __version__, rounded_ver_parse yaml = YAML() @@ -29,34 +23,6 @@ namestr_regex = r"^[A-Za-z][A-Za-z\-_]*[A-Za-z]$" -def random_secure_string( - length: int = 16, chars: str = string.ascii_lowercase + string.digits -): - return "".join(secrets.choice(chars) for i in range(length)) - - -def set_docker_image_tag() -> str: - """Set docker image tag for `jupyterlab`, `jupyterhub`, and `dask-worker`.""" - return os.environ.get("NEBARI_IMAGE_TAG", constants.DEFAULT_NEBARI_IMAGE_TAG) - - -def set_nebari_dask_version() -> str: - """Set version of `nebari-dask` meta package.""" - return os.environ.get("NEBARI_DASK_VERSION", constants.DEFAULT_NEBARI_DASK_VERSION) - - -@yaml_object(yaml) -class CertificateEnum(str, enum.Enum): - letsencrypt = "lets-encrypt" - selfsigned = "self-signed" - existing = "existing" - disabled = "disabled" - - @classmethod - def to_yaml(cls, representer, node): - return representer.represent_str(node.value) - - @yaml_object(yaml) class TerraformStateEnum(str, enum.Enum): remote = "remote" @@ -115,17 +81,6 @@ def to_yaml(cls, representer, node): return representer.represent_str(node.value) -@yaml_object(yaml) -class AccessEnum(str, enum.Enum): - all = "all" - yaml = "yaml" - keycloak = "keycloak" - - @classmethod - def to_yaml(cls, representer, node): - return representer.represent_str(node.value) - - class Base(pydantic.BaseModel): ... @@ -135,669 +90,6 @@ class Config: allow_population_by_field_name = True -# ============== CI/CD ============= - - -class CICD(Base): - type: CiEnum = CiEnum.none - branch: str = "main" - commit_render: bool = True - before_script: typing.List[typing.Union[str, typing.Dict]] = [] - after_script: typing.List[typing.Union[str, typing.Dict]] = [] - - -# ======== Generic Helm Extensions ======== -class HelmExtension(Base): - name: str - repository: str - chart: str - version: str - overrides: typing.Dict = {} - - -# ============== Argo-Workflows ========= - - -class NebariWorkflowController(Base): - enabled: bool = True - image_tag: str = constants.DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG - - -class ArgoWorkflows(Base): - enabled: bool = True - overrides: typing.Dict = {} - nebari_workflow_controller: NebariWorkflowController = NebariWorkflowController() - - -# ============== kbatch ============= - - -class KBatch(Base): - enabled: bool = True - - -# ============== Monitoring ============= - - -class Monitoring(Base): - enabled: bool = True - - -# ============== ClearML ============= - - -class ClearML(Base): - enabled: bool = False - enable_forward_auth: bool = False - overrides: typing.Dict = {} - - -# ============== Prefect ============= - - -class Prefect(Base): - enabled: bool = False - image: typing.Optional[str] - overrides: typing.Dict = {} - token: typing.Optional[str] - - -# =========== Conda-Store ============== - - -class CondaStore(Base): - extra_settings: typing.Dict[str, typing.Any] = {} - extra_config: str = "" - image: str = "quansight/conda-store-server" - image_tag: str = constants.DEFAULT_CONDA_STORE_IMAGE_TAG - default_namespace: str = "nebari-git" - object_storage: str = "200Gi" - - -# ============= Terraform =============== - - -class TerraformState(Base): - type: TerraformStateEnum = TerraformStateEnum.remote - backend: typing.Optional[str] - config: typing.Dict[str, str] = {} - - -# ============ Certificate ============= - - -class Certificate(Base): - type: CertificateEnum = CertificateEnum.selfsigned - # existing - secret_name: typing.Optional[str] - # lets-encrypt - acme_email: typing.Optional[str] - acme_server: str = "https://acme-v02.api.letsencrypt.org/directory" - - -# ========== Default Images ============== - - -class DefaultImages(Base): - jupyterhub: str = f"quay.io/nebari/nebari-jupyterhub:{set_docker_image_tag()}" - jupyterlab: str = f"quay.io/nebari/nebari-jupyterlab:{set_docker_image_tag()}" - dask_worker: str = f"quay.io/nebari/nebari-dask-worker:{set_docker_image_tag()}" - - -# =========== Storage ============= - - -class Storage(Base): - conda_store: str = "200Gi" - shared_filesystem: str = "200Gi" - - -# =========== Authentication ============== - - -class GitHubConfig(Base): - client_id: str - client_secret: str - - -class Auth0Config(Base): - client_id: str - client_secret: str - auth0_subdomain: str - - -class Authentication(Base, ABC): - _types: typing.Dict[str, type] = {} - - type: AuthenticationEnum - - # Based on https://github.com/samuelcolvin/pydantic/issues/2177#issuecomment-739578307 - - # This allows type field to determine which subclass of Authentication should be used for validation. - - # Used to register automatically all the submodels in `_types`. - def __init_subclass__(cls): - cls._types[cls._typ.value] = cls - - @classmethod - def __get_validators__(cls): - yield cls.validate - - @classmethod - def validate(cls, value: typing.Dict[str, typing.Any]) -> "Authentication": - if "type" not in value: - raise ValueError("type field is missing from security.authentication") - - specified_type = value.get("type") - sub_class = cls._types.get(specified_type, None) - - if not sub_class: - raise ValueError( - f"No registered Authentication type called {specified_type}" - ) - - # init with right submodel - return sub_class(**value) - - -class PasswordAuthentication(Authentication): - _typ = AuthenticationEnum.password - - -class Auth0Authentication(Authentication): - _typ = AuthenticationEnum.auth0 - config: Auth0Config - - -class GitHubAuthentication(Authentication): - _typ = AuthenticationEnum.github - config: GitHubConfig - - -# ================= Keycloak ================== - - -class Keycloak(Base): - initial_root_password: str = Field(default_factory=random_secure_string) - overrides: typing.Dict = {} - realm_display_name: str = "Nebari" - - -# ============== Security ================ - - -class Security(Base): - authentication: Authentication = PasswordAuthentication( - type=AuthenticationEnum.password - ) - shared_users_group: bool = True - keycloak: Keycloak = Keycloak() - - -# ================ Providers =============== - - -class KeyValueDict(Base): - key: str - value: str - - -class NodeSelector(Base): - general: KeyValueDict - user: KeyValueDict - worker: KeyValueDict - - -class DigitalOceanNodeGroup(Base): - """Representation of a node group with Digital Ocean - - - Kubernetes limits: https://docs.digitalocean.com/products/kubernetes/details/limits/ - - Available instance types: https://slugs.do-api.dev/ - """ - - instance: str - min_nodes: conint(ge=1) = 1 - max_nodes: conint(ge=1) = 1 - - -class DigitalOceanProvider(Base): - region: str = "nyc3" - kubernetes_version: typing.Optional[str] - # Digital Ocean image slugs are listed here https://slugs.do-api.dev/ - node_groups: typing.Dict[str, DigitalOceanNodeGroup] = { - "general": DigitalOceanNodeGroup( - instance="g-8vcpu-32gb", min_nodes=1, max_nodes=1 - ), - "user": DigitalOceanNodeGroup( - instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5 - ), - "worker": DigitalOceanNodeGroup( - instance="g-4vcpu-16gb", min_nodes=1, max_nodes=5 - ), - } - tags: typing.Optional[typing.List[str]] = [] - - @root_validator - def _validate_kubernetes_version(cls, values): - available_kubernetes_versions = digital_ocean.kubernetes_versions( - values["region"] - ) - if ( - values["kubernetes_version"] is not None - and values["kubernetes_version"] not in available_kubernetes_versions - ): - raise ValueError( - f"\nInvalid `kubernetes-version` provided: {values['kubernetes_version']}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." - ) - else: - values["kubernetes_version"] = available_kubernetes_versions[-1] - return values - - -class GCPIPAllocationPolicy(Base): - cluster_secondary_range_name: str - services_secondary_range_name: str - cluster_ipv4_cidr_block: str - services_ipv4_cidr_block: str - - -class GCPCIDRBlock(Base): - cidr_block: str - display_name: str - - -class GCPMasterAuthorizedNetworksConfig(Base): - cidr_blocks: typing.List[GCPCIDRBlock] - - -class GCPPrivateClusterConfig(Base): - enable_private_endpoint: bool - enable_private_nodes: bool - master_ipv4_cidr_block: str - - -class GCPGuestAccelerator(Base): - """ - See general information regarding GPU support at: - # TODO: replace with nebari.dev new URL - https://docs.nebari.dev/en/stable/source/admin_guide/gpu.html?#add-gpu-node-group - """ - - name: str - count: conint(ge=1) = 1 - - -class GCPNodeGroup(Base): - instance: str - min_nodes: conint(ge=0) = 0 - max_nodes: conint(ge=1) = 1 - preemptible: bool = False - labels: typing.Dict[str, str] = {} - guest_accelerators: typing.List[GCPGuestAccelerator] = [] - - -class GoogleCloudPlatformProvider(Base): - project: str = Field(default_factory=lambda: os.environ["PROJECT_ID"]) - region: str = "us-central1" - availability_zones: typing.Optional[typing.List[str]] = [] - kubernetes_version: typing.Optional[str] - release_channel: str = constants.DEFAULT_GKE_RELEASE_CHANNEL - node_groups: typing.Dict[str, GCPNodeGroup] = { - "general": GCPNodeGroup(instance="n1-standard-8", min_nodes=1, max_nodes=1), - "user": GCPNodeGroup(instance="n1-standard-4", min_nodes=0, max_nodes=5), - "worker": GCPNodeGroup(instance="n1-standard-4", min_nodes=0, max_nodes=5), - } - tags: typing.Optional[typing.List[str]] = [] - networking_mode: str = "ROUTE" - network: str = "default" - subnetwork: typing.Optional[typing.Union[str, None]] = None - ip_allocation_policy: typing.Optional[ - typing.Union[GCPIPAllocationPolicy, None] - ] = None - master_authorized_networks_config: typing.Optional[ - typing.Union[GCPCIDRBlock, None] - ] = None - private_cluster_config: typing.Optional[ - typing.Union[GCPPrivateClusterConfig, None] - ] = None - - @root_validator - def _validate_kubernetes_version(cls, values): - available_kubernetes_versions = google_cloud.kubernetes_versions( - values["region"] - ) - if ( - values["kubernetes_version"] is not None - and values["kubernetes_version"] not in available_kubernetes_versions - ): - raise ValueError( - f"\nInvalid `kubernetes-version` provided: {values['kubernetes_version']}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." - ) - else: - values["kubernetes_version"] = available_kubernetes_versions[-1] - return values - - -class AzureNodeGroup(Base): - instance: str - min_nodes: int - max_nodes: int - - -class AzureProvider(Base): - region: str = "Central US" - kubernetes_version: typing.Optional[str] - node_groups: typing.Dict[str, AzureNodeGroup] = { - "general": AzureNodeGroup(instance="Standard_D8_v3", min_nodes=1, max_nodes=1), - "user": AzureNodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), - "worker": AzureNodeGroup(instance="Standard_D4_v3", min_nodes=0, max_nodes=5), - } - storage_account_postfix: str = Field( - default_factory=lambda: random_secure_string(length=4) - ) - vnet_subnet_id: typing.Optional[typing.Union[str, None]] = None - private_cluster_enabled: bool = False - - @validator("kubernetes_version") - def _validate_kubernetes_version(cls, value): - available_kubernetes_versions = azure_cloud.kubernetes_versions() - if value is None: - value = available_kubernetes_versions[-1] - elif value not in available_kubernetes_versions: - raise ValueError( - f"\nInvalid `kubernetes-version` provided: {value}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." - ) - return value - - -class AWSNodeGroup(Base): - instance: str - min_nodes: int = 0 - max_nodes: int - gpu: bool = False - single_subnet: bool = False - - -class AmazonWebServicesProvider(Base): - region: str = Field( - default_factory=lambda: os.environ.get("AWS_DEFAULT_REGION", "us-west-2") - ) - availability_zones: typing.Optional[typing.List[str]] - kubernetes_version: typing.Optional[str] - node_groups: typing.Dict[str, AWSNodeGroup] = { - "general": AWSNodeGroup(instance="m5.2xlarge", min_nodes=1, max_nodes=1), - "user": AWSNodeGroup( - instance="m5.xlarge", min_nodes=1, max_nodes=5, single_subnet=False - ), - "worker": AWSNodeGroup( - instance="m5.xlarge", min_nodes=1, max_nodes=5, single_subnet=False - ), - } - existing_subnet_ids: typing.List[str] = None - existing_security_group_ids: str = None - vpc_cidr_block: str = "10.10.0.0/16" - - @root_validator - def _validate_kubernetes_version(cls, values): - available_kubernetes_versions = amazon_web_services.kubernetes_versions() - if values["kubernetes_version"] is None: - values["kubernetes_version"] = available_kubernetes_versions[-1] - elif values["kubernetes_version"] not in available_kubernetes_versions: - raise ValueError( - f"\nInvalid `kubernetes-version` provided: {values['kubernetes_version']}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available." - ) - return values - - @root_validator - def _validate_availability_zones(cls, values): - if values["availability_zones"] is None: - zones = amazon_web_services.zones(values["region"]) - values["availability_zones"] = list(sorted(zones))[:2] - return values - - -class LocalProvider(Base): - kube_context: typing.Optional[str] - node_selectors: typing.Dict[str, KeyValueDict] = { - "general": KeyValueDict(key="kubernetes.io/os", value="linux"), - "user": KeyValueDict(key="kubernetes.io/os", value="linux"), - "worker": KeyValueDict(key="kubernetes.io/os", value="linux"), - } - - -class ExistingProvider(Base): - kube_context: typing.Optional[str] - node_selectors: typing.Dict[str, KeyValueDict] = { - "general": KeyValueDict(key="kubernetes.io/os", value="linux"), - "user": KeyValueDict(key="kubernetes.io/os", value="linux"), - "worker": KeyValueDict(key="kubernetes.io/os", value="linux"), - } - - -# ================= Theme ================== - - -class JupyterHubTheme(Base): - hub_title: str = "Nebari" - hub_subtitle: str = "Your open source data science platform" - welcome: str = """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""" - logo: str = "https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg" - primary_color: str = "#4f4173" - secondary_color: str = "#957da6" - accent_color: str = "#32C574" - text_color: str = "#111111" - h1_color: str = "#652e8e" - h2_color: str = "#652e8e" - version: str = f"v{__version__}" - display_version: str = "True" # limitation of theme everything is a str - - -class Theme(Base): - jupyterhub: JupyterHubTheme = JupyterHubTheme() - - -# ================= Theme ================== - - -class JupyterHub(Base): - overrides: typing.Dict = {} - - -# ================= JupyterLab ================== - - -class IdleCuller(Base): - terminal_cull_inactive_timeout: int = 15 - terminal_cull_interval: int = 5 - kernel_cull_idle_timeout: int = 15 - kernel_cull_interval: int = 5 - kernel_cull_connected: bool = True - kernel_cull_busy: bool = False - server_shutdown_no_activity_timeout: int = 15 - - -class JupyterLab(Base): - idle_culler: IdleCuller = IdleCuller() - - -# ================== Profiles ================== - - -class KubeSpawner(Base): - cpu_limit: int - cpu_guarantee: int - mem_limit: str - mem_guarantee: str - - class Config: - extra = "allow" - - -class JupyterLabProfile(Base): - access: AccessEnum = AccessEnum.all - display_name: str - description: str - default: bool = False - users: typing.Optional[typing.List[str]] - groups: typing.Optional[typing.List[str]] - kubespawner_override: typing.Optional[KubeSpawner] - - @root_validator - def only_yaml_can_have_groups_and_users(cls, values): - if values["access"] != AccessEnum.yaml: - if ( - values.get("users", None) is not None - or values.get("groups", None) is not None - ): - raise ValueError( - "Profile must not contain groups or users fields unless access = yaml" - ) - return values - - -class DaskWorkerProfile(Base): - worker_cores_limit: int - worker_cores: int - worker_memory_limit: str - worker_memory: str - image: typing.Optional[str] - - class Config: - extra = "allow" - - -class Profiles(Base): - jupyterlab: typing.List[JupyterLabProfile] = [ - JupyterLabProfile( - display_name="Small Instance", - description="Stable environment with 2 cpu / 8 GB ram", - default=True, - kubespawner_override=KubeSpawner( - cpu_limit=2, - cpu_guarantee=1.5, - mem_limit="8G", - mem_guarantee="5G", - ), - ), - JupyterLabProfile( - display_name="Medium Instance", - description="Stable environment with 4 cpu / 16 GB ram", - kubespawner_override=KubeSpawner( - cpu_limit=4, - cpu_guarantee=3, - mem_limit="16G", - mem_guarantee="10G", - ), - ), - ] - dask_worker: typing.Dict[str, DaskWorkerProfile] = { - "Small Worker": DaskWorkerProfile( - worker_cores_limit=2, - worker_cores=1.5, - worker_memory_limit="8G", - worker_memory="5G", - worker_threads=2, - ), - "Medium Worker": DaskWorkerProfile( - worker_cores_limit=4, - worker_cores=3, - worker_memory_limit="16G", - worker_memory="10G", - worker_threads=4, - ), - } - - @validator("jupyterlab") - def check_default(cls, v, values): - """Check if only one default value is present.""" - default = [attrs["default"] for attrs in v if "default" in attrs] - if default.count(True) > 1: - raise TypeError( - "Multiple default Jupyterlab profiles may cause unexpected problems." - ) - return v - - -# ================ Environment ================ - - -class CondaEnvironment(Base): - name: str - channels: typing.Optional[typing.List[str]] - dependencies: typing.List[typing.Union[str, typing.Dict[str, typing.List[str]]]] - - -# =============== CDSDashboards ============== - - -class CDSDashboards(Base): - enabled: bool = True - cds_hide_user_named_servers: bool = True - cds_hide_user_dashboard_servers: bool = False - - -# =============== Extensions = = ============== - - -class NebariExtensionEnv(Base): - name: str - value: str - - -class NebariExtension(Base): - name: str - image: str - urlslug: str - private: bool = False - oauth2client: bool = False - keycloakadmin: bool = False - jwt: bool = False - nebariconfigyaml: bool = False - logout: typing.Optional[str] - envs: typing.Optional[typing.List[NebariExtensionEnv]] - - -class Ingress(Base): - terraform_overrides: typing.Dict = {} - - -# ======== External Container Registry ======== - -# This allows the user to set a private AWS ECR as a replacement for -# Docker Hub for some images - those where you provide the full path -# to the image on the ECR. -# extcr_account and extcr_region are the AWS account number and region -# of the ECR respectively. access_key_id and secret_access_key are -# AWS access keys that should have read access to the ECR. - - -class ExtContainerReg(Base): - enabled: bool = False - access_key_id: typing.Optional[str] - secret_access_key: typing.Optional[str] - extcr_account: typing.Optional[str] - extcr_region: typing.Optional[str] - - @root_validator - def enabled_must_have_fields(cls, values): - if values["enabled"]: - for fldname in ( - "access_key_id", - "secret_access_key", - "extcr_account", - "extcr_region", - ): - if ( - fldname not in values - or values[fldname] is None - or values[fldname].strip() == "" - ): - raise ValueError( - f"external_container_reg must contain a non-blank {fldname} when enabled is true" - ) - return values - - # ==================== Main =================== letter_dash_underscore_pydantic = pydantic.constr(regex=namestr_regex) @@ -858,114 +150,15 @@ class InitInputs(Base): output: pathlib.Path = pathlib.Path("nebari-config.yaml") -class CLIContext(Base): - stages: typing.List = [] - - class Main(Base): - provider: ProviderEnum = ProviderEnum.local project_name: str namespace: letter_dash_underscore_pydantic = "dev" # In nebari_version only use major.minor.patch version - drop any pre/post/dev suffixes nebari_version: str = __version__ - ci_cd: CICD = CICD() - domain: typing.Optional[str] - terraform_state: TerraformState = TerraformState() - certificate: Certificate = Certificate() - helm_extensions: typing.List[HelmExtension] = [] - prefect: Prefect = Prefect() - cdsdashboards: CDSDashboards = CDSDashboards() - security: Security = Security() - external_container_reg: ExtContainerReg = ExtContainerReg() - default_images: DefaultImages = DefaultImages() - storage: Storage = Storage() - local: typing.Optional[LocalProvider] - existing: typing.Optional[ExistingProvider] - google_cloud_platform: typing.Optional[GoogleCloudPlatformProvider] - amazon_web_services: typing.Optional[AmazonWebServicesProvider] - azure: typing.Optional[AzureProvider] - digital_ocean: typing.Optional[DigitalOceanProvider] - theme: Theme = Theme() - profiles: Profiles = Profiles() - environments: typing.Dict[str, CondaEnvironment] = { - "environment-dask.yaml": CondaEnvironment( - name="dask", - channels=["conda-forge"], - dependencies=[ - "python=3.10.8", - "ipykernel=6.21.0", - "ipywidgets==7.7.1", - f"nebari-dask =={set_nebari_dask_version()}", - "python-graphviz=0.20.1", - "pyarrow=10.0.1", - "s3fs=2023.1.0", - "gcsfs=2023.1.0", - "numpy=1.23.5", - "numba=0.56.4", - "pandas=1.5.3", - { - "pip": [ - "kbatch==0.4.1", - ], - }, - ], - ), - "environment-dashboard.yaml": CondaEnvironment( - name="dashboard", - channels=["conda-forge"], - dependencies=[ - "python=3.10", - "cdsdashboards-singleuser=0.6.3", - "cufflinks-py=0.17.3", - "dash=2.8.1", - "geopandas=0.12.2", - "geopy=2.3.0", - "geoviews=1.9.6", - "gunicorn=20.1.0", - "holoviews=1.15.4", - "ipykernel=6.21.2", - "ipywidgets=8.0.4", - "jupyter=1.0.0", - "jupyterlab=3.6.1", - "jupyter_bokeh=3.0.5", - "matplotlib=3.7.0", - f"nebari-dask=={set_nebari_dask_version()}", - "nodejs=18.12.1", - "numpy", - "openpyxl=3.1.1", - "pandas=1.5.3", - "panel=0.14.3", - "param=1.12.3", - "plotly=5.13.0", - "python-graphviz=0.20.1", - "rich=13.3.1", - "streamlit=1.9.0", - "sympy=1.11.1", - "voila=0.4.0", - "pip=23.0", - { - "pip": [ - "streamlit-image-comparison==0.0.3", - "noaa-coops==0.2.1", - "dash_core_components==2.0.0", - "dash_html_components==2.0.0", - ], - }, - ], - ), - } - conda_store: CondaStore = CondaStore() - argo_workflows: ArgoWorkflows = ArgoWorkflows() - kbatch: KBatch = KBatch() - monitoring: Monitoring = Monitoring() - clearml: ClearML = ClearML() - tf_extensions: typing.List[NebariExtension] = [] - jupyterhub: JupyterHub = JupyterHub() - jupyterlab: JupyterLab = JupyterLab() + prevent_deploy: bool = ( False # Optional, but will be given default value if not present ) - ingress: Ingress = Ingress() # If the nebari_version in the schema is old # we must tell the user to first run nebari upgrade @@ -983,50 +176,6 @@ def check_default(cls, v): ) return v - @root_validator - def check_provider(cls, values): - if values["provider"] == ProviderEnum.local and values.get("local") is None: - values["local"] = LocalProvider() - elif ( - values["provider"] == ProviderEnum.existing - and values.get("existing") is None - ): - values["existing"] = ExistingProvider() - elif ( - values["provider"] == ProviderEnum.gcp - and values.get("google_cloud_platform") is None - ): - values["google_cloud_platform"] = GoogleCloudPlatformProvider() - elif ( - values["provider"] == ProviderEnum.aws - and values.get("amazon_web_services") is None - ): - values["amazon_web_services"] = AmazonWebServicesProvider() - elif values["provider"] == ProviderEnum.azure and values.get("azure") is None: - values["azure"] = AzureProvider() - elif ( - values["provider"] == ProviderEnum.do - and values.get("digital_ocean") is None - ): - values["digital_ocean"] = DigitalOceanProvider() - - if ( - sum( - (_ in values and values[_] is not None) - for _ in { - "local", - "existing", - "google_cloud_platform", - "amazon_web_services", - "azure", - "digital_ocean", - } - ) - != 1 - ): - raise ValueError("multiple providers set or wrong provider fields set") - return values - @classmethod def is_version_accepted(cls, v): return v != "" and rounded_ver_parse(v) == rounded_ver_parse(__version__) @@ -1083,7 +232,7 @@ def _set_attr(d: typing.Any, attr: str, value: typing.Any): def set_config_from_environment_variables( - config: Main, keyword: str = "NEBARI_SECRET", separator: str = "__" + config: Base, keyword: str = "NEBARI_SECRET", separator: str = "__" ): """Setting nebari configuration values from environment variables From dd46e27dbf1cd2e82c91371a35735d6994748b75 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Jun 2023 02:47:11 +0000 Subject: [PATCH 088/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/cli.py | 1 - src/_nebari/initialize.py | 4 ++-- src/_nebari/stages/bootstrap/__init__.py | 4 ++-- src/_nebari/stages/infrastructure/__init__.py | 12 +++++++++--- src/_nebari/stages/kubernetes_ingress/__init__.py | 2 +- src/_nebari/stages/kubernetes_keycloak/__init__.py | 6 +++--- src/_nebari/stages/kubernetes_services/__init__.py | 9 +++------ src/_nebari/stages/nebari_tf_extensions/__init__.py | 2 +- src/_nebari/subcommands/validate.py | 5 ++++- src/nebari/hookspecs.py | 2 +- src/nebari/plugins.py | 4 +--- src/nebari/schema.py | 6 +----- 12 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/_nebari/cli.py b/src/_nebari/cli.py index 272ab21d4..de91cc185 100644 --- a/src/_nebari/cli.py +++ b/src/_nebari/cli.py @@ -4,7 +4,6 @@ from typer.core import TyperGroup from _nebari.version import __version__ -from nebari import schema from nebari.plugins import nebari_plugin_manager diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index 78ebaeac9..9e721de7b 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -1,9 +1,9 @@ import logging import os import re -import tempfile -import string import secrets +import string +import tempfile import requests diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index dc2ee0f65..935997801 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -1,7 +1,7 @@ -import io import enum -from inspect import cleandoc +import io import typing +from inspect import cleandoc from typing import Any, Dict, List from _nebari.provider.cicd.github import gen_nebari_linter, gen_nebari_ops diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 20f7b7125..a1f44f958 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -9,13 +9,13 @@ import pydantic from _nebari import constants -from _nebari.stages.base import NebariTerraformStage from _nebari.provider.cloud import ( amazon_web_services, azure_cloud, digital_ocean, google_cloud, ) +from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( NebariAWSProvider, NebariGCPProvider, @@ -429,7 +429,10 @@ class InputSchema(schema.Base): @pydantic.root_validator def check_provider(cls, values): - if values["provider"] == schema.ProviderEnum.local and values.get("local") is None: + if ( + values["provider"] == schema.ProviderEnum.local + and values.get("local") is None + ): values["local"] = LocalProvider() elif ( values["provider"] == schema.ProviderEnum.existing @@ -446,7 +449,10 @@ def check_provider(cls, values): and values.get("amazon_web_services") is None ): values["amazon_web_services"] = schema.AmazonWebServicesProvider() - elif values["provider"] == schema.ProviderEnum.azure and values.get("azure") is None: + elif ( + values["provider"] == schema.ProviderEnum.azure + and values.get("azure") is None + ): values["azure"] = AzureProvider() elif ( values["provider"] == schema.ProviderEnum.do diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index 33761754d..0697cd2b0 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -1,5 +1,5 @@ -import socket import enum +import socket import sys import time import typing diff --git a/src/_nebari/stages/kubernetes_keycloak/__init__.py b/src/_nebari/stages/kubernetes_keycloak/__init__.py index 869d8eca9..ac8882df2 100644 --- a/src/_nebari/stages/kubernetes_keycloak/__init__.py +++ b/src/_nebari/stages/kubernetes_keycloak/__init__.py @@ -1,13 +1,13 @@ +import contextlib import enum +import json import secrets import string -import contextlib -import json import sys import time import typing -from typing import Any, Dict, List from abc import ABC +from typing import Any, Dict, List import pydantic diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index a85667656..f913267ca 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -1,6 +1,6 @@ import enum -import os import json +import os import sys import typing from typing import Any, Dict, List @@ -9,7 +9,6 @@ import pydantic from pydantic import Field -from _nebari.version import __version__ from _nebari import constants from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( @@ -17,6 +16,7 @@ NebariKubernetesProvider, NebariTerraformState, ) +from _nebari.version import __version__ from nebari import schema from nebari.hookspecs import NebariStage, hookimpl @@ -228,7 +228,6 @@ class JupyterHub(schema.Base): overrides: typing.Dict = {} - class IdleCuller(schema.Base): terminal_cull_inactive_timeout: int = 15 terminal_cull_interval: int = 5 @@ -375,9 +374,7 @@ class JupyterhubInputVars(schema.Base): jupyterhub_overrides: List[str] = Field(alias="jupyterhub-overrides") jupyterhub_stared_storage: str = Field(alias="jupyterhub-shared-storage") jupyterhub_shared_endpoint: str = Field(None, alias="jupyterhub-shared-endpoint") - jupyterhub_profiles: List[JupyterLabProfile] = Field( - alias="jupyterlab-profiles" - ) + jupyterhub_profiles: List[JupyterLabProfile] = Field(alias="jupyterlab-profiles") jupyterhub_image: ImageNameTag = Field(alias="jupyterhub-image") jupyterhub_hub_extraEnv: str = Field(alias="jupyterhub-hub-extraEnv") idle_culler_settings: Dict[str, Any] = Field(alias="idle-culler-settings") diff --git a/src/_nebari/stages/nebari_tf_extensions/__init__.py b/src/_nebari/stages/nebari_tf_extensions/__init__.py index 4e4c0515e..cf2bf7e5a 100644 --- a/src/_nebari/stages/nebari_tf_extensions/__init__.py +++ b/src/_nebari/stages/nebari_tf_extensions/__init__.py @@ -7,8 +7,8 @@ NebariKubernetesProvider, NebariTerraformState, ) -from nebari.hookspecs import NebariStage, hookimpl from nebari import schema +from nebari.hookspecs import NebariStage, hookimpl class NebariExtensionEnv(schema.Base): diff --git a/src/_nebari/subcommands/validate.py b/src/_nebari/subcommands/validate.py index 465aaf643..8cfbced11 100644 --- a/src/_nebari/subcommands/validate.py +++ b/src/_nebari/subcommands/validate.py @@ -30,5 +30,8 @@ def validate( pass else: from nebari.plugins import nebari_plugin_manager - schema.read_configuration(config_filename, config_schema=nebari_plugin_manager.config_schema) + + schema.read_configuration( + config_filename, config_schema=nebari_plugin_manager.config_schema + ) print("[bold purple]Successfully validated configuration.[/bold purple]") diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index 72990774c..8c01f412c 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -2,9 +2,9 @@ import pathlib from typing import Any, Dict, List +import pydantic import typer from pluggy import HookimplMarker, HookspecMarker -import pydantic from nebari import schema diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index 205c6ffd1..8ef489804 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -9,10 +9,8 @@ import pluggy import pydantic -from _nebari.version import __version__ from nebari import hookspecs, schema - DEFAULT_SUBCOMMAND_PLUGINS = [ # subcommands "_nebari.subcommands.info", @@ -128,7 +126,7 @@ def config_schema(self): classes = [schema.Main] + [ _.input_schema for _ in self.ordered_stages if _.input_schema is not None ] - return type('ConfigSchema', tuple(classes), {}) + return type("ConfigSchema", tuple(classes), {}) nebari_plugin_manager = NebariPluginManager() diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 2f65c9484..3d44c607b 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -2,17 +2,13 @@ import os import pathlib import re -import secrets -import string import sys import typing -from abc import ABC import pydantic -from pydantic import Field, conint, root_validator, validator +from pydantic import validator from ruamel.yaml import YAML, yaml_object -from _nebari import constants from _nebari.version import __version__, rounded_ver_parse yaml = YAML() From c26006b0b103cb84fc7cd14c4d9fd0ef16fe5df1 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 28 Jun 2023 22:17:40 -0400 Subject: [PATCH 089/147] Moving validation of environment variables to schema and providers --- src/_nebari/initialize.py | 7 +- .../provider/cloud/amazon_web_services.py | 15 ++ src/_nebari/provider/cloud/azure_cloud.py | 17 +++ src/_nebari/provider/cloud/digital_ocean.py | 33 +++++ src/_nebari/provider/cloud/google_cloud.py | 12 ++ src/_nebari/stages/bootstrap/__init__.py | 12 +- src/_nebari/stages/infrastructure/__init__.py | 69 ++++++--- src/_nebari/utils.py | 132 ------------------ src/nebari/hookspecs.py | 3 - 9 files changed, 130 insertions(+), 170 deletions(-) diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index 9e721de7b..a93e31d3c 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -10,7 +10,6 @@ from _nebari.provider import git from _nebari.provider.cicd import github from _nebari.provider.oauth.auth0 import create_client -from _nebari.utils import check_cloud_credentials from _nebari.version import __version__ from nebari import schema @@ -131,6 +130,8 @@ def render_config( config["certificate"] = {"type": schema.CertificateEnum.letsencrypt.value} config["certificate"]["acme_email"] = ssl_cert_email + # TODO: attempt to load config into schema for validation + if repository_auto_provision: GITHUB_REGEX = "(https://)?github.com/([^/]+)/([^/]+)/?" if re.search(GITHUB_REGEX, repository): @@ -148,10 +149,6 @@ def render_config( def github_auto_provision(config: schema.Main, owner: str, repo: str): - check_cloud_credentials( - config - ) # We may need env vars such as AWS_ACCESS_KEY_ID depending on provider - already_exists = True try: github.get_repository(owner, repo) diff --git a/src/_nebari/provider/cloud/amazon_web_services.py b/src/_nebari/provider/cloud/amazon_web_services.py index 218aebecf..ab552b4f3 100644 --- a/src/_nebari/provider/cloud/amazon_web_services.py +++ b/src/_nebari/provider/cloud/amazon_web_services.py @@ -8,6 +8,21 @@ from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version +def check_credentials(): + AWS_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-aws" + + for variable in { + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + }: + if variable not in os.environ: + raise ValueError( + f"""Missing the following required environment variable: {variable}\n + Please see the documentation for more information: {AWS_ENV_DOCS}""" + ) + + + @functools.lru_cache() def regions(): output = subprocess.check_output(["aws", "ec2", "describe-regions"]) diff --git a/src/_nebari/provider/cloud/azure_cloud.py b/src/_nebari/provider/cloud/azure_cloud.py index 7ef13da22..bfd2637cb 100644 --- a/src/_nebari/provider/cloud/azure_cloud.py +++ b/src/_nebari/provider/cloud/azure_cloud.py @@ -11,6 +11,23 @@ logger.setLevel(logging.ERROR) +def check_credentials(): + AZURE_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-azure" + + for variable in { + "ARM_CLIENT_ID", + "ARM_CLIENT_SECRET", + "ARM_SUBSCRIPTION_ID", + "ARM_TENANT_ID", + }: + if variable not in os.environ: + raise ValueError( + f"""Missing the following required environment variable: {variable}\n + Please see the documentation for more information: {AZURE_ENV_DOCS}""" + ) + + + @functools.lru_cache() def initiate_container_service_client(): subscription_id = os.environ.get("ARM_SUBSCRIPTION_ID", None) diff --git a/src/_nebari/provider/cloud/digital_ocean.py b/src/_nebari/provider/cloud/digital_ocean.py index fd55672f1..20d15ff82 100644 --- a/src/_nebari/provider/cloud/digital_ocean.py +++ b/src/_nebari/provider/cloud/digital_ocean.py @@ -6,6 +6,39 @@ from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version +def check_credentials(): + DO_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-do" + + for variable in { + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "SPACES_ACCESS_KEY_ID", + "SPACES_SECRET_ACCESS_KEY", + "DIGITALOCEAN_TOKEN", + }: + if variable not in os.environ: + raise ValueError( + f"""Missing the following required environment variable: {variable}\n + Please see the documentation for more information: {DO_ENV_DOCS}""" + ) + + if os.environ["AWS_ACCESS_KEY_ID"] != os.environ["SPACES_ACCESS_KEY_ID"]: + raise ValueError( + f"""The environment variables AWS_ACCESS_KEY_ID and SPACES_ACCESS_KEY_ID must be equal\n + See {DO_ENV_DOCS} for more information""" + ) + + if ( + os.environ["AWS_SECRET_ACCESS_KEY"] + != os.environ["SPACES_SECRET_ACCESS_KEY"] + ): + raise ValueError( + f"""The environment variables AWS_SECRET_ACCESS_KEY and SPACES_SECRET_ACCESS_KEY must be equal\n + See {DO_ENV_DOCS} for more information""" + ) + + + def digital_ocean_request(url, method="GET", json=None): BASE_DIGITALOCEAN_URL = "https://api.digitalocean.com/v2/" diff --git a/src/_nebari/provider/cloud/google_cloud.py b/src/_nebari/provider/cloud/google_cloud.py index a8e6d542f..accf26ce7 100644 --- a/src/_nebari/provider/cloud/google_cloud.py +++ b/src/_nebari/provider/cloud/google_cloud.py @@ -1,3 +1,4 @@ +import os import functools import json import subprocess @@ -5,6 +6,17 @@ from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version +def check_credentials(): + GCP_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-gcp" + for variable in {"GOOGLE_CREDENTIALS"}: + if variable not in os.environ: + raise ValueError( + f"""Missing the following required environment variable: {variable}\n + Please see the documentation for more information: {GCP_ENV_DOCS}""" + ) + + + @functools.lru_cache() def projects(): output = subprocess.check_output(["gcloud", "projects", "list", "--format=json"]) diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index 935997801..d7824f89b 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -4,9 +4,10 @@ from inspect import cleandoc from typing import Any, Dict, List +import questionary + from _nebari.provider.cicd.github import gen_nebari_linter, gen_nebari_ops from _nebari.provider.cicd.gitlab import gen_gitlab_ci -from _nebari.utils import check_cloud_credentials from nebari import schema from nebari.hookspecs import NebariStage, hookimpl @@ -29,7 +30,7 @@ def gen_gitignore(): return {".gitignore": cleandoc(filestoignore)} -def gen_cicd(config): +def gen_cicd(config: schema.Main): """ Use cicd schema to generate workflow files based on the `ci_cd` key in the `config`. @@ -40,12 +41,12 @@ def gen_cicd(config): """ cicd_files = {} - if config.ci_cd.type == schema.CiEnum.github_actions: + if config.ci_cd.type == CiEnum.github_actions: gha_dir = ".github/workflows/" cicd_files[gha_dir + "nebari-ops.yaml"] = gen_nebari_ops(config) cicd_files[gha_dir + "nebari-linter.yaml"] = gen_nebari_linter(config) - elif config.ci_cd.type == schema.CiEnum.gitlab_ci: + elif config.ci_cd.type == CiEnum.gitlab_ci: cicd_files[".gitlab-ci.yml"] = gen_gitlab_ci(config) else: @@ -106,9 +107,6 @@ def render(self) -> Dict[str, str]: contents.update(gen_gitignore()) return contents - def check(self, stage_outputs: Dict[str, Dict[str, Any]]): - check_cloud_credentials(self.config) - @hookimpl def nebari_stage() -> List[NebariStage]: diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index a1f44f958..a1096715f 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -1,3 +1,4 @@ +import enum import contextlib import inspect import pathlib @@ -30,6 +31,20 @@ def get_kubeconfig_filename(): return str(pathlib.Path(tempfile.gettempdir()) / "NEBARI_KUBECONFIG") +@schema.yaml_object(schema.yaml) +class ProviderEnum(str, enum.Enum): + local = "local" + existing = "existing" + do = "do" + aws = "aws" + gcp = "gcp" + azure = "azure" + + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_str(node.value) + + class LocalInputVars(schema.Base): kubeconfig_filename: str = get_kubeconfig_filename() kube_context: Optional[str] @@ -138,27 +153,27 @@ class AWSInputVars(schema.Base): def _calculate_node_groups(config: schema.Main): - if config.provider == schema.ProviderEnum.aws: + if config.provider == ProviderEnum.aws: return { group: {"key": "eks.amazonaws.com/nodegroup", "value": group} for group in ["general", "user", "worker"] } - elif config.provider == schema.ProviderEnum.gcp: + elif config.provider == ProviderEnum.gcp: return { group: {"key": "cloud.google.com/gke-nodepool", "value": group} for group in ["general", "user", "worker"] } - elif config.provider == schema.ProviderEnum.azure: + elif config.provider == ProviderEnum.azure: return { group: {"key": "azure-node-pool", "value": group} for group in ["general", "user", "worker"] } - elif config.provider == schema.ProviderEnum.do: + elif config.provider == ProviderEnum.do: return { group: {"key": "doks.digitalocean.com/node-pool", "value": group} for group in ["general", "user", "worker"] } - elif config.provider == schema.ProviderEnum.existing: + elif config.provider == ProviderEnum.existing: return config.existing.node_selectors else: return config.local.dict()["node_selectors"] @@ -223,6 +238,8 @@ class DigitalOceanProvider(schema.Base): @pydantic.root_validator def _validate_kubernetes_version(cls, values): + digital_ocean.check_credentials() + available_kubernetes_versions = digital_ocean.kubernetes_versions( values["region"] ) @@ -307,6 +324,8 @@ class GoogleCloudPlatformProvider(schema.Base): @pydantic.root_validator def _validate_kubernetes_version(cls, values): + google_cloud.check_credentials() + available_kubernetes_versions = google_cloud.kubernetes_versions( values["region"] ) @@ -344,6 +363,8 @@ class AzureProvider(schema.Base): @pydantic.validator("kubernetes_version") def _validate_kubernetes_version(cls, value): + azure_cloud.check_credentials() + available_kubernetes_versions = azure_cloud.kubernetes_versions() if value is None: value = available_kubernetes_versions[-1] @@ -383,6 +404,8 @@ class AmazonWebServicesProvider(schema.Base): @pydantic.root_validator def _validate_kubernetes_version(cls, values): + amazon_web_services.check_credentials() + available_kubernetes_versions = amazon_web_services.kubernetes_versions() if values["kubernetes_version"] is None: values["kubernetes_version"] = available_kubernetes_versions[-1] @@ -419,7 +442,7 @@ class ExistingProvider(schema.Base): class InputSchema(schema.Base): - provider: schema.ProviderEnum = schema.ProviderEnum.local + provider: ProviderEnum = ProviderEnum.local local: typing.Optional[LocalProvider] existing: typing.Optional[ExistingProvider] google_cloud_platform: typing.Optional[GoogleCloudPlatformProvider] @@ -430,32 +453,32 @@ class InputSchema(schema.Base): @pydantic.root_validator def check_provider(cls, values): if ( - values["provider"] == schema.ProviderEnum.local + values["provider"] == ProviderEnum.local and values.get("local") is None ): values["local"] = LocalProvider() elif ( - values["provider"] == schema.ProviderEnum.existing + values["provider"] == ProviderEnum.existing and values.get("existing") is None ): values["existing"] = ExistingProvider() elif ( - values["provider"] == schema.ProviderEnum.gcp + values["provider"] == ProviderEnum.gcp and values.get("google_cloud_platform") is None ): values["google_cloud_platform"] = GoogleCloudPlatformProvider() elif ( - values["provider"] == schema.ProviderEnum.aws + values["provider"] == ProviderEnum.aws and values.get("amazon_web_services") is None ): - values["amazon_web_services"] = schema.AmazonWebServicesProvider() + values["amazon_web_services"] = AmazonWebServicesProvider() elif ( - values["provider"] == schema.ProviderEnum.azure + values["provider"] == ProviderEnum.azure and values.get("azure") is None ): values["azure"] = AzureProvider() elif ( - values["provider"] == schema.ProviderEnum.do + values["provider"] == ProviderEnum.do and values.get("digital_ocean") is None ): values["digital_ocean"] = DigitalOceanProvider() @@ -537,20 +560,20 @@ def stage_prefix(self): return pathlib.Path("stages") / self.name / self.config.provider.value def tf_objects(self) -> List[Dict]: - if self.config.provider == schema.ProviderEnum.gcp: + if self.config.provider == ProviderEnum.gcp: return [ NebariGCPProvider(self.config), NebariTerraformState(self.name, self.config), ] - elif self.config.provider == schema.ProviderEnum.do: + elif self.config.provider == ProviderEnum.do: return [ NebariTerraformState(self.name, self.config), ] - elif self.config.provider == schema.ProviderEnum.azure: + elif self.config.provider == ProviderEnum.azure: return [ NebariTerraformState(self.name, self.config), ] - elif self.config.provider == schema.ProviderEnum.aws: + elif self.config.provider == ProviderEnum.aws: return [ NebariAWSProvider(self.config), NebariTerraformState(self.name, self.config), @@ -559,13 +582,13 @@ def tf_objects(self) -> List[Dict]: return [] def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): - if self.config.provider == schema.ProviderEnum.local: + if self.config.provider == ProviderEnum.local: return LocalInputVars(kube_context=self.config.local.kube_context).dict() - elif self.config.provider == schema.ProviderEnum.existing: + elif self.config.provider == ProviderEnum.existing: return ExistingInputVars( kube_context=self.config.existing.kube_context ).dict() - elif self.config.provider == schema.ProviderEnum.do: + elif self.config.provider == ProviderEnum.do: return DigitalOceanInputVars( name=self.config.project_name, environment=self.config.namespace, @@ -574,7 +597,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): kubernetes_version=self.config.digital_ocean.kubernetes_version, node_groups=self.config.digital_ocean.node_groups, ).dict() - elif self.config.provider == schema.ProviderEnum.gcp: + elif self.config.provider == ProviderEnum.gcp: return GCPInputVars( name=self.config.project_name, environment=self.config.namespace, @@ -603,7 +626,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): master_authorized_networks_config=self.config.google_cloud_platform.master_authorized_networks_config, private_cluster_config=self.config.google_cloud_platform.private_cluster_config, ).dict() - elif self.config.provider == schema.ProviderEnum.azure: + elif self.config.provider == ProviderEnum.azure: return AzureInputVars( name=self.config.project_name, environment=self.config.namespace, @@ -622,7 +645,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): vnet_subnet_id=self.config.azure.vnet_subnet_id, private_cluster_enabled=self.config.azure.private_cluster_enabled, ).dict() - elif self.config.provider == schema.ProviderEnum.aws: + elif self.config.provider == ProviderEnum.aws: return AWSInputVars( name=self.config.project_name, environment=self.config.namespace, diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 26fc44489..60d4e07ad 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -12,23 +12,10 @@ from ruamel.yaml import YAML -from _nebari.provider.cloud import ( - amazon_web_services, - azure_cloud, - digital_ocean, - google_cloud, -) -from nebari import schema - # environment variable overrides NEBARI_K8S_VERSION = os.getenv("NEBARI_K8S_VERSION", None) NEBARI_GH_BRANCH = os.getenv("NEBARI_GH_BRANCH", None) -DO_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-do" -AWS_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-aws" -GCP_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-gcp" -AZURE_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-azure" - CONDA_FORGE_CHANNEL_DATA_URL = "https://conda.anaconda.org/conda-forge/channeldata.json" # Create a ruamel object with our favored config, for universal use @@ -137,125 +124,6 @@ def backup_config_file(filename: Path, extrasuffix: str = ""): print(f"Backing up {filename} as {backup_filename}") -def check_cloud_credentials(config: schema.Main): - if config.provider == schema.ProviderEnum.gcp: - for variable in {"GOOGLE_CREDENTIALS"}: - if variable not in os.environ: - raise ValueError( - f"""Missing the following required environment variable: {variable}\n - Please see the documentation for more information: {GCP_ENV_DOCS}""" - ) - elif config.provider == schema.ProviderEnum.azure: - for variable in { - "ARM_CLIENT_ID", - "ARM_CLIENT_SECRET", - "ARM_SUBSCRIPTION_ID", - "ARM_TENANT_ID", - }: - if variable not in os.environ: - raise ValueError( - f"""Missing the following required environment variable: {variable}\n - Please see the documentation for more information: {AZURE_ENV_DOCS}""" - ) - elif config.provider == schema.ProviderEnum.aws: - for variable in { - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - }: - if variable not in os.environ: - raise ValueError( - f"""Missing the following required environment variable: {variable}\n - Please see the documentation for more information: {AWS_ENV_DOCS}""" - ) - elif config.provider == schema.ProviderEnum.do: - for variable in { - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "SPACES_ACCESS_KEY_ID", - "SPACES_SECRET_ACCESS_KEY", - "DIGITALOCEAN_TOKEN", - }: - if variable not in os.environ: - raise ValueError( - f"""Missing the following required environment variable: {variable}\n - Please see the documentation for more information: {DO_ENV_DOCS}""" - ) - - if os.environ["AWS_ACCESS_KEY_ID"] != os.environ["SPACES_ACCESS_KEY_ID"]: - raise ValueError( - f"""The environment variables AWS_ACCESS_KEY_ID and SPACES_ACCESS_KEY_ID must be equal\n - See {DO_ENV_DOCS} for more information""" - ) - - if ( - os.environ["AWS_SECRET_ACCESS_KEY"] - != os.environ["SPACES_SECRET_ACCESS_KEY"] - ): - raise ValueError( - f"""The environment variables AWS_SECRET_ACCESS_KEY and SPACES_SECRET_ACCESS_KEY must be equal\n - See {DO_ENV_DOCS} for more information""" - ) - - -def set_kubernetes_version( - config, kubernetes_version, cloud_provider, grab_latest_version=True -): - cloud_provider_dict = { - "aws": { - "full_name": "amazon_web_services", - "k8s_version_checker_func": amazon_web_services.kubernetes_versions, - }, - "azure": { - "full_name": "azure", - "k8s_version_checker_func": azure_cloud.kubernetes_versions, - }, - "do": { - "full_name": "digital_ocean", - "k8s_version_checker_func": digital_ocean.kubernetes_versions, - }, - "gcp": { - "full_name": "google_cloud_platform", - "k8s_version_checker_func": google_cloud.kubernetes_versions, - }, - } - cloud_full_name = cloud_provider_dict[cloud_provider]["full_name"] - func = cloud_provider_dict[cloud_provider]["k8s_version_checker_func"] - cloud_config = config[cloud_full_name] - - def _raise_value_error(cloud_provider, k8s_versions): - raise ValueError( - f"\nInvalid `kubernetes-version` provided: {kubernetes_version}.\nPlease select from one of the following {cloud_provider.upper()} supported Kubernetes versions: {k8s_versions} or omit flag to use latest Kubernetes version available." - ) - - def _check_and_set_kubernetes_version( - kubernetes_version=kubernetes_version, - cloud_provider=cloud_provider, - cloud_config=cloud_config, - func=func, - ): - region = cloud_config["region"] - - # to avoid using cloud provider SDK - # set NEBARI_K8S_VERSION environment variable - if not NEBARI_K8S_VERSION: - k8s_versions = func(region) - else: - k8s_versions = [NEBARI_K8S_VERSION] - - if kubernetes_version: - if kubernetes_version in k8s_versions: - cloud_config["kubernetes_version"] = kubernetes_version - else: - _raise_value_error(cloud_provider, k8s_versions) - elif grab_latest_version: - cloud_config["kubernetes_version"] = k8s_versions[-1] - else: - # grab oldest version - cloud_config["kubernetes_version"] = k8s_versions[0] - - return _check_and_set_kubernetes_version() - - @contextlib.contextmanager def modified_environ(*remove: List[str], **update: Dict[str, str]): """ diff --git a/src/nebari/hookspecs.py b/src/nebari/hookspecs.py index 8c01f412c..789dfe2d7 100644 --- a/src/nebari/hookspecs.py +++ b/src/nebari/hookspecs.py @@ -23,9 +23,6 @@ def __init__(self, output_directory: pathlib.Path, config: schema.Main): self.output_directory = output_directory self.config = config - def validate(self): - pass - def render(self) -> Dict[str, str]: return {} From fd5e71a499089b880a6eeb4738e876f9fdcd1c74 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Jun 2023 02:18:43 +0000 Subject: [PATCH 090/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/provider/cloud/amazon_web_services.py | 1 - src/_nebari/provider/cloud/azure_cloud.py | 1 - src/_nebari/provider/cloud/digital_ocean.py | 6 +----- src/_nebari/provider/cloud/google_cloud.py | 3 +-- src/_nebari/stages/bootstrap/__init__.py | 4 +--- src/_nebari/stages/infrastructure/__init__.py | 12 +++--------- 6 files changed, 6 insertions(+), 21 deletions(-) diff --git a/src/_nebari/provider/cloud/amazon_web_services.py b/src/_nebari/provider/cloud/amazon_web_services.py index ab552b4f3..a18a64ca2 100644 --- a/src/_nebari/provider/cloud/amazon_web_services.py +++ b/src/_nebari/provider/cloud/amazon_web_services.py @@ -22,7 +22,6 @@ def check_credentials(): ) - @functools.lru_cache() def regions(): output = subprocess.check_output(["aws", "ec2", "describe-regions"]) diff --git a/src/_nebari/provider/cloud/azure_cloud.py b/src/_nebari/provider/cloud/azure_cloud.py index bfd2637cb..f344dadcc 100644 --- a/src/_nebari/provider/cloud/azure_cloud.py +++ b/src/_nebari/provider/cloud/azure_cloud.py @@ -27,7 +27,6 @@ def check_credentials(): ) - @functools.lru_cache() def initiate_container_service_client(): subscription_id = os.environ.get("ARM_SUBSCRIPTION_ID", None) diff --git a/src/_nebari/provider/cloud/digital_ocean.py b/src/_nebari/provider/cloud/digital_ocean.py index 20d15ff82..984dc219e 100644 --- a/src/_nebari/provider/cloud/digital_ocean.py +++ b/src/_nebari/provider/cloud/digital_ocean.py @@ -28,17 +28,13 @@ def check_credentials(): See {DO_ENV_DOCS} for more information""" ) - if ( - os.environ["AWS_SECRET_ACCESS_KEY"] - != os.environ["SPACES_SECRET_ACCESS_KEY"] - ): + if os.environ["AWS_SECRET_ACCESS_KEY"] != os.environ["SPACES_SECRET_ACCESS_KEY"]: raise ValueError( f"""The environment variables AWS_SECRET_ACCESS_KEY and SPACES_SECRET_ACCESS_KEY must be equal\n See {DO_ENV_DOCS} for more information""" ) - def digital_ocean_request(url, method="GET", json=None): BASE_DIGITALOCEAN_URL = "https://api.digitalocean.com/v2/" diff --git a/src/_nebari/provider/cloud/google_cloud.py b/src/_nebari/provider/cloud/google_cloud.py index accf26ce7..3ad2ce50c 100644 --- a/src/_nebari/provider/cloud/google_cloud.py +++ b/src/_nebari/provider/cloud/google_cloud.py @@ -1,6 +1,6 @@ -import os import functools import json +import os import subprocess from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version @@ -16,7 +16,6 @@ def check_credentials(): ) - @functools.lru_cache() def projects(): output = subprocess.check_output(["gcloud", "projects", "list", "--format=json"]) diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index d7824f89b..bff3fd371 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -2,9 +2,7 @@ import io import typing from inspect import cleandoc -from typing import Any, Dict, List - -import questionary +from typing import Dict, List from _nebari.provider.cicd.github import gen_nebari_linter, gen_nebari_ops from _nebari.provider.cicd.gitlab import gen_gitlab_ci diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index a1096715f..6f4e3c9e2 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -1,5 +1,5 @@ -import enum import contextlib +import enum import inspect import pathlib import sys @@ -452,10 +452,7 @@ class InputSchema(schema.Base): @pydantic.root_validator def check_provider(cls, values): - if ( - values["provider"] == ProviderEnum.local - and values.get("local") is None - ): + if values["provider"] == ProviderEnum.local and values.get("local") is None: values["local"] = LocalProvider() elif ( values["provider"] == ProviderEnum.existing @@ -472,10 +469,7 @@ def check_provider(cls, values): and values.get("amazon_web_services") is None ): values["amazon_web_services"] = AmazonWebServicesProvider() - elif ( - values["provider"] == ProviderEnum.azure - and values.get("azure") is None - ): + elif values["provider"] == ProviderEnum.azure and values.get("azure") is None: values["azure"] = AzureProvider() elif ( values["provider"] == ProviderEnum.do From a70485bfe93d6b598bf583d0be3e03351426526f Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 29 Jun 2023 16:59:12 -0400 Subject: [PATCH 091/147] Further work to uncouple stages --- src/_nebari/config.py | 117 +++++++++ src/_nebari/initialize.py | 58 ++--- src/_nebari/stages/bootstrap/__init__.py | 2 +- src/_nebari/stages/infrastructure/__init__.py | 38 ++- .../stages/kubernetes_services/__init__.py | 1 + src/_nebari/stages/tf_objects.py | 20 -- src/_nebari/subcommands/deploy.py | 4 +- src/_nebari/subcommands/destroy.py | 4 +- src/_nebari/subcommands/dev.py | 4 +- src/_nebari/subcommands/init.py | 161 +++++++----- src/_nebari/subcommands/keycloak.py | 15 +- src/_nebari/subcommands/render.py | 4 +- src/_nebari/subcommands/support.py | 7 +- src/_nebari/subcommands/validate.py | 4 +- src/_nebari/upgrade.py | 9 +- src/_nebari/utils.py | 39 ++- src/nebari/plugins.py | 2 +- src/nebari/schema.py | 229 ++---------------- 18 files changed, 342 insertions(+), 376 deletions(-) create mode 100644 src/_nebari/config.py diff --git a/src/_nebari/config.py b/src/_nebari/config.py new file mode 100644 index 000000000..ca8b8db96 --- /dev/null +++ b/src/_nebari/config.py @@ -0,0 +1,117 @@ +import os +import pathlib +import typing + +import pydantic + +from _nebari.utils import yaml + + +def set_nested_attribute(data: typing.Any, attrs: typing.List[str], value: typing.Any): + """Takes an arbitrary set of attributes and accesses the deep + nested object config to set value + + """ + + def _get_attr(d: typing.Any, attr: str): + if hasattr(d, "__getitem__"): + if re.fullmatch(r"\d+", attr): + try: + return d[int(attr)] + except Exception: + return d[attr] + else: + return d[attr] + else: + return getattr(d, attr) + + def _set_attr(d: typing.Any, attr: str, value: typing.Any): + if hasattr(d, "__getitem__"): + if re.fullmatch(r"\d+", attr): + try: + d[int(attr)] = value + except Exception: + d[attr] = value + else: + d[attr] = value + else: + return setattr(d, attr, value) + + data_pos = data + for attr in attrs[:-1]: + data_pos = _get_attr(data_pos, attr) + _set_attr(data_pos, attrs[-1], value) + + +def set_config_from_environment_variables( + config: pydantic.BaseModel, keyword: str = "NEBARI_SECRET", separator: str = "__" +): + """Setting nebari configuration values from environment variables + + For example `NEBARI_SECRET__ci_cd__branch=master` would set `ci_cd.branch = "master"` + """ + nebari_secrets = [_ for _ in os.environ if _.startswith(keyword + separator)] + for secret in nebari_secrets: + attrs = secret[len(keyword + separator) :].split(separator) + try: + set_nested_attribute(config, attrs, os.environ[secret]) + except Exception as e: + print( + f"FAILED: setting secret from environment variable={secret} due to the following error\n {e}" + ) + sys.exit(1) + return config + + +def read_configuration( + config_filename: pathlib.Path, + config_schema: pydantic.BaseModel, + read_environment: bool = True, +): + """Read configuration from multiple sources and apply validation""" + filename = pathlib.Path(config_filename) + + if not filename.is_file(): + raise ValueError( + f"passed in configuration filename={config_filename} does not exist" + ) + + with filename.open() as f: + config = config_schema(**yaml.load(f.read())) + + if read_environment: + config = set_config_from_environment_variables(config) + + return config + + +def write_configuration( + config_filename: pathlib.Path, + config: typing.Union[pydantic.BaseModel, typing.Dict], + mode: str = "w", +): + with config_filename.open(mode) as f: + if isinstance(config, pydantic.BaseModel): + yaml.dump(config.dict(), f) + else: + yaml.dump(config, f) + + +def backup_configuration(filename: pathlib.Path, extrasuffix: str = ""): + if not filename.exists(): + return + + # Backup old file + backup_filename = pathlib.Path(f"{filename}{extrasuffix}.backup") + + if backup_filename.exists(): + i = 1 + while True: + next_backup_filename = pathlib.Path(f"{backup_filename}~{i}") + if not next_backup_filename.exists(): + backup_filename = next_backup_filename + break + i = i + 1 + + filename.rename(backup_filename) + print(f"Backing up {filename} as {backup_filename}") diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index a93e31d3c..0d7e9df42 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -6,35 +6,37 @@ import tempfile import requests +import pydantic +from _nebari.utils import random_secure_string from _nebari.provider import git from _nebari.provider.cicd import github from _nebari.provider.oauth.auth0 import create_client from _nebari.version import __version__ -from nebari import schema -logger = logging.getLogger(__name__) +from _nebari.stages.bootstrap import CiEnum +from _nebari.stages.infrastructure import ProviderEnum +from _nebari.stages.kubernetes_ingress import CertificateEnum +from _nebari.stages.kubernetes_keycloak import AuthenticationEnum +from _nebari.stages.terraform_state import TerraformStateEnum -WELCOME_HEADER_TEXT = "Your open source data science platform, hosted" +logger = logging.getLogger(__name__) -def random_secure_string( - length: int = 16, chars: str = string.ascii_lowercase + string.digits -): - return "".join(secrets.choice(chars) for i in range(length)) +WELCOME_HEADER_TEXT = "Your open source data science platform, hosted" def render_config( project_name: str, nebari_domain: str = None, - cloud_provider: schema.ProviderEnum = schema.ProviderEnum.local, - ci_provider: schema.CiEnum = schema.CiEnum.none, + cloud_provider: ProviderEnum = ProviderEnum.local, + ci_provider: CiEnum = CiEnum.none, repository: str = None, - auth_provider: schema.AuthenticationEnum = schema.AuthenticationEnum.password, + auth_provider: AuthenticationEnum = AuthenticationEnum.password, namespace: str = "dev", repository_auto_provision: bool = False, auth_auto_provision: bool = False, - terraform_state: schema.TerraformStateEnum = schema.TerraformStateEnum.remote, + terraform_state: TerraformStateEnum = TerraformStateEnum.remote, kubernetes_version: str = None, disable_prompt: bool = False, ssl_cert_email: str = None, @@ -72,13 +74,13 @@ def render_config( ] = """Welcome! Learn about Nebari's features and configurations in the documentation. If you have any questions or feedback, reach the team on Nebari's support forums.""" config["security"]["authentication"] = {"type": auth_provider.value} - if auth_provider == schema.AuthenticationEnum.github: + if auth_provider == AuthenticationEnum.github: if not disable_prompt: config["security"]["authentication"]["config"] = { "client_id": input("Github client_id: "), "client_secret": input("Github client_secret: "), } - elif auth_provider == schema.AuthenticationEnum.auth0: + elif auth_provider == AuthenticationEnum.auth0: if auth_auto_provision: auth0_config = create_client(config.domain, config.project_name) config["security"]["authentication"]["config"] = auth0_config @@ -89,13 +91,13 @@ def render_config( "auth0_subdomain": input("Auth0 subdomain: "), } - if cloud_provider == schema.ProviderEnum.do: + if cloud_provider == ProviderEnum.do: config["theme"]["jupyterhub"][ "hub_subtitle" ] = f"{WELCOME_HEADER_TEXT} on Digital Ocean" if kubernetes_version is not None: config["digital_ocean"] = {"kubernetes_version": kubernetes_version} - elif cloud_provider == schema.ProviderEnum.gcp: + elif cloud_provider == ProviderEnum.gcp: config["theme"]["jupyterhub"][ "hub_subtitle" ] = f"{WELCOME_HEADER_TEXT} on Google Cloud Platform" @@ -109,35 +111,37 @@ def render_config( if kubernetes_version is not None: config["google_cloud_platform"]["kubernetes_version"] = kubernetes_version - elif cloud_provider == schema.ProviderEnum.azure: + elif cloud_provider == ProviderEnum.azure: config["theme"]["jupyterhub"][ "hub_subtitle" ] = f"{WELCOME_HEADER_TEXT} on Azure" if kubernetes_version is not None: config["azure"] = {"kubernetes_version": kubernetes_version} - elif cloud_provider == schema.ProviderEnum.aws: + elif cloud_provider == ProviderEnum.aws: config["theme"]["jupyterhub"][ "hub_subtitle" ] = f"{WELCOME_HEADER_TEXT} on Amazon Web Services" if kubernetes_version is not None: config["amazon_web_services"] = {"kubernetes_version": kubernetes_version} - elif cloud_provider == schema.ProviderEnum.existing: + elif cloud_provider == ProviderEnum.existing: config["theme"]["jupyterhub"]["hub_subtitle"] = WELCOME_HEADER_TEXT - elif cloud_provider == schema.ProviderEnum.local: + elif cloud_provider == ProviderEnum.local: config["theme"]["jupyterhub"]["hub_subtitle"] = WELCOME_HEADER_TEXT if ssl_cert_email: - config["certificate"] = {"type": schema.CertificateEnum.letsencrypt.value} + config["certificate"] = {"type": CertificateEnum.letsencrypt.value} config["certificate"]["acme_email"] = ssl_cert_email - # TODO: attempt to load config into schema for validation + # validate configuration and convert to model + from nebari.plugins import nebari_plugin_manager + config_model = nebari_plugin_manager.config_schema.parse_obj(config) if repository_auto_provision: GITHUB_REGEX = "(https://)?github.com/([^/]+)/([^/]+)/?" if re.search(GITHUB_REGEX, repository): match = re.search(GITHUB_REGEX, repository) git_repository = github_auto_provision( - schema.Main.parse_obj(config), match.group(2), match.group(3) + config_model, match.group(2), match.group(3) ) git_repository_initialize(git_repository) else: @@ -148,7 +152,7 @@ def render_config( return config -def github_auto_provision(config: schema.Main, owner: str, repo: str): +def github_auto_provision(config: pydantic.BaseModel, owner: str, repo: str): already_exists = True try: github.get_repository(owner, repo) @@ -173,7 +177,7 @@ def github_auto_provision(config: schema.Main, owner: str, repo: str): try: # Secrets - if config.provider == schema.ProviderEnum.do: + if config.provider == ProviderEnum.do: for name in { "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", @@ -182,17 +186,17 @@ def github_auto_provision(config: schema.Main, owner: str, repo: str): "DIGITALOCEAN_TOKEN", }: github.update_secret(owner, repo, name, os.environ[name]) - elif config.provider == schema.ProviderEnum.aws: + elif config.provider == ProviderEnum.aws: for name in { "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", }: github.update_secret(owner, repo, name, os.environ[name]) - elif config.provider == schema.ProviderEnum.gcp: + elif config.provider == ProviderEnum.gcp: github.update_secret(owner, repo, "PROJECT_ID", os.environ["PROJECT_ID"]) with open(os.environ["GOOGLE_CREDENTIALS"]) as f: github.update_secret(owner, repo, "GOOGLE_CREDENTIALS", f.read()) - elif config.provider == schema.ProviderEnum.azure: + elif config.provider == ProviderEnum.azure: for name in { "ARM_CLIENT_ID", "ARM_CLIENT_SECRET", diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index bff3fd371..6229417bd 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -91,7 +91,7 @@ class BootstrapStage(NebariStage): def render(self) -> Dict[str, str]: contents = {} - if self.config.ci_cd.type != schema.CiEnum.none: + if self.config.ci_cd.type != CiEnum.none: for fn, workflow in gen_cicd(self.config).items(): stream = io.StringIO() schema.yaml.dump( diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 6f4e3c9e2..f5da3f864 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -5,6 +5,7 @@ import sys import tempfile import typing +import string from typing import Any, Dict, List, Optional import pydantic @@ -17,12 +18,11 @@ google_cloud, ) from _nebari.stages.base import NebariTerraformStage +from _nebari.provider import terraform from _nebari.stages.tf_objects import ( - NebariAWSProvider, - NebariGCPProvider, NebariTerraformState, ) -from _nebari.utils import modified_environ +from _nebari.utils import modified_environ, escape_string, deep_merge from nebari import schema from nebari.hookspecs import NebariStage, hookimpl @@ -556,7 +556,11 @@ def stage_prefix(self): def tf_objects(self) -> List[Dict]: if self.config.provider == ProviderEnum.gcp: return [ - NebariGCPProvider(self.config), + terraform.Provider( + "google", + project=nebari_config.google_cloud_platform.project, + region=nebari_config.google_cloud_platform.region, + ), NebariTerraformState(self.name, self.config), ] elif self.config.provider == ProviderEnum.do: @@ -569,13 +573,29 @@ def tf_objects(self) -> List[Dict]: ] elif self.config.provider == ProviderEnum.aws: return [ - NebariAWSProvider(self.config), + terraform.Provider("aws", region=nebari_config.amazon_web_services.region), NebariTerraformState(self.name, self.config), ] else: return [] def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): + def escape_project_name(project_name: str, provider: ProviderEnum): + if provider == ProviderEnum.azure and '-' in project_name: + project_name = escape_string( + project_name, + escape_char='a' + ) + + if provider == ProviderEnum.aws and project_name.startswith('aws'): + project_name = 'a' + project_name + + if len(project_name) > 16: + project_name = project_name[:16] + + return project_name + + if self.config.provider == ProviderEnum.local: return LocalInputVars(kube_context=self.config.local.kube_context).dict() elif self.config.provider == ProviderEnum.existing: @@ -584,7 +604,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): ).dict() elif self.config.provider == ProviderEnum.do: return DigitalOceanInputVars( - name=self.config.project_name, + name=escape_project_name(self.config.project_name, self.config.provider), environment=self.config.namespace, region=self.config.digital_ocean.region, tags=self.config.digital_ocean.tags, @@ -593,7 +613,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): ).dict() elif self.config.provider == ProviderEnum.gcp: return GCPInputVars( - name=self.config.project_name, + name=escape_project_name(self.config.project_name, self.config.provider), environment=self.config.namespace, region=self.config.google_cloud_platform.region, project_id=self.config.google_cloud_platform.project, @@ -622,7 +642,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): ).dict() elif self.config.provider == ProviderEnum.azure: return AzureInputVars( - name=self.config.project_name, + name=escape_project_name(self.config.project_name, self.config.provider), environment=self.config.namespace, region=self.config.azure.region, kubernetes_version=self.config.azure.kubernetes_version, @@ -641,7 +661,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): ).dict() elif self.config.provider == ProviderEnum.aws: return AWSInputVars( - name=self.config.project_name, + name=escape_project_name(self.config.project_name, self.config.provider), environment=self.config.namespace, existing_subnet_ids=self.config.amazon_web_services.existing_subnet_ids, existing_security_group_id=self.config.amazon_web_services.existing_security_group_ids, diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index f913267ca..9b4e25495 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -1,3 +1,4 @@ +import time import enum import json import os diff --git a/src/_nebari/stages/tf_objects.py b/src/_nebari/stages/tf_objects.py index 343fc5116..58bf25a1e 100644 --- a/src/_nebari/stages/tf_objects.py +++ b/src/_nebari/stages/tf_objects.py @@ -3,26 +3,6 @@ from nebari import schema -def NebariAWSProvider(nebari_config: schema.Main): - return Provider("aws", region=nebari_config.amazon_web_services.region) - - -def NebariGCPProvider(nebari_config: schema.Main): - return Provider( - "google", - project=nebari_config.google_cloud_platform.project, - region=nebari_config.google_cloud_platform.region, - ) - - -def NebariAzureProvider(nebari_config: schema.Main): - return Provider("azurerm", features={}) - - -def NebariDigitalOceanProvider(nebari_config: schema.Main): - return Provider("digitalocean") - - def NebariKubernetesProvider(nebari_config: schema.Main): if nebari_config.provider == schema.ProviderEnum.aws: cluster_name = f"{nebari_config.project_name}-{nebari_config.namespace}" diff --git a/src/_nebari/subcommands/deploy.py b/src/_nebari/subcommands/deploy.py index 412238f75..765e788af 100644 --- a/src/_nebari/subcommands/deploy.py +++ b/src/_nebari/subcommands/deploy.py @@ -4,7 +4,7 @@ from _nebari.deploy import deploy_configuration from _nebari.render import render_template -from nebari import schema +from _nebari.config import read_configuration from nebari.hookspecs import hookimpl @@ -64,7 +64,7 @@ def deploy( stages = nebari_plugin_manager.ordered_stages config_schema = nebari_plugin_manager.config_schema - config = schema.read_configuration(config_filename, config_schema=config_schema) + config = read_configuration(config_filename, config_schema=config_schema) if not disable_render: render_template(output_directory, config, stages) diff --git a/src/_nebari/subcommands/destroy.py b/src/_nebari/subcommands/destroy.py index 5b1e56656..6c317116d 100644 --- a/src/_nebari/subcommands/destroy.py +++ b/src/_nebari/subcommands/destroy.py @@ -4,7 +4,7 @@ from _nebari.destroy import destroy_configuration from _nebari.render import render_template -from nebari import schema +from _nebari.config import read_configuration from nebari.hookspecs import hookimpl @@ -44,7 +44,7 @@ def destroy( def _run_destroy( config_filename=config_filename, disable_render=disable_render ): - config = schema.read_configuration( + config = read_configuration( config_filename, config_schema=config_schema ) diff --git a/src/_nebari/subcommands/dev.py b/src/_nebari/subcommands/dev.py index 54ee0f6b9..c3c104f06 100644 --- a/src/_nebari/subcommands/dev.py +++ b/src/_nebari/subcommands/dev.py @@ -4,7 +4,7 @@ import typer from _nebari.keycloak import keycloak_rest_api_call -from nebari import schema +from _nebari.config import read_configuration from nebari.hookspecs import hookimpl @@ -50,6 +50,6 @@ def keycloak_api( config_schema = nebari_plugin_manager.config_schema - schema.read_configuration(config_filename, config_schema=config_schema) + read_configuration(config_filename, config_schema=config_schema) r = keycloak_rest_api_call(config_filename, request=request) print(json.dumps(r, indent=4)) diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index cff725001..d05393fd3 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -2,6 +2,7 @@ import pathlib import re import typing +import enum import questionary import rich @@ -9,9 +10,16 @@ from pydantic import BaseModel from _nebari.initialize import render_config +from _nebari.config import write_configuration from nebari import schema from nebari.hookspecs import hookimpl +from _nebari.stages.bootstrap import CiEnum +from _nebari.stages.infrastructure import ProviderEnum +from _nebari.stages.kubernetes_ingress import CertificateEnum +from _nebari.stages.kubernetes_keycloak import AuthenticationEnum +from _nebari.stages.terraform_state import TerraformStateEnum + MISSING_CREDS_TEMPLATE = "Unable to locate your {provider} credentials, refer to this guide on how to generate them:\n\n[green]\t{link_to_docs}[/green]\n\n" LINKS_TO_DOCS_TEMPLATE = ( "For more details, refer to the Nebari docs:\n\n\t[green]{link_to_docs}[/green]\n\n" @@ -42,11 +50,33 @@ ) +class GitRepoEnum(str, enum.Enum): + github = "github.com" + gitlab = "gitlab.com" + + +class InitInputs(schema.Base): + cloud_provider: ProviderEnum = ProviderEnum.local + project_name: schema.letter_dash_underscore_pydantic = "" + domain_name: typing.Optional[str] = None + namespace: typing.Optional[schema.letter_dash_underscore_pydantic] = "dev" + auth_provider: AuthenticationEnum = AuthenticationEnum.password + auth_auto_provision: bool = False + repository: typing.Union[str, None] = None + repository_auto_provision: bool = False + ci_provider: CiEnum = CiEnum.none + terraform_state: TerraformStateEnum = TerraformStateEnum.remote + kubernetes_version: typing.Union[str, None] = None + ssl_cert_email: typing.Union[schema.email_pydantic, None] = None + disable_prompt: bool = False + output: pathlib.Path = pathlib.Path("nebari-config.yaml") + + def enum_to_list(enum_cls): return [e.value for e in enum_cls] -def handle_init(inputs: schema.InitInputs, config_schema: BaseModel): +def handle_init(inputs: InitInputs, config_schema: BaseModel): """ Take the inputs from the `nebari init` command, render the config and write it to a local yaml file. """ @@ -72,8 +102,8 @@ def handle_init(inputs: schema.InitInputs, config_schema: BaseModel): ) try: - schema.write_configuration( - inputs.output, config, mode="x", config_schema=config_schema + write_configuration( + inputs.output, config, mode="x", ) except FileExistsError: raise ValueError( @@ -81,28 +111,11 @@ def handle_init(inputs: schema.InitInputs, config_schema: BaseModel): ) -def check_project_name(ctx: typer.Context, project_name: str): - """Validate the project_name is acceptable. Depends on `cloud_provider`.""" - schema.project_name_convention( - project_name.lower(), {"provider": ctx.params["cloud_provider"]} - ) - - return project_name - - -def check_ssl_cert_email(ctx: typer.Context, ssl_cert_email: str): - """Validate the email used for SSL cert is in a valid format.""" - if ssl_cert_email and not re.match("^[^ @]+@[^ @]+\\.[^ @]+$", ssl_cert_email): - raise ValueError("ssl-cert-email should be a valid email address") - - return ssl_cert_email - - def check_repository_creds(ctx: typer.Context, git_provider: str): """Validate the necessary Git provider (GitHub) credentials are set.""" if ( - git_provider == schema.GitRepoEnum.github.value.lower() + git_provider == GitRepoEnum.github.value.lower() and not os.environ.get("GITHUB_USERNAME") or not os.environ.get("GITHUB_TOKEN") ): @@ -116,6 +129,28 @@ def check_repository_creds(ctx: typer.Context, git_provider: str): ) +def typer_validate_regex(regex: str, error_message: str = None): + def callback(value): + if value is None: + return value + + if re.fullmatch(regex, value): + return value + message = error_message or f"Does not match {regex}" + raise typer.BadParameter(message) + return callback + + +def questionary_validate_regex(regex: str, error_message: str = None): + def callback(value): + if re.fullmatch(regex, value): + return True + + message = error_message or f"Invalid input. Does not match {regex}" + return message + return callback + + def check_auth_provider_creds(ctx: typer.Context, auth_provider: str): """Validate the the necessary auth provider credentials have been set as environment variables.""" if ctx.params.get("disable_prompt"): @@ -124,7 +159,7 @@ def check_auth_provider_creds(ctx: typer.Context, auth_provider: str): auth_provider = auth_provider.lower() # Auth0 - if auth_provider == schema.AuthenticationEnum.auth0.value.lower() and ( + if auth_provider == AuthenticationEnum.auth0.value.lower() and ( not os.environ.get("AUTH0_CLIENT_ID") or not os.environ.get("AUTH0_CLIENT_SECRET") or not os.environ.get("AUTH0_DOMAIN") @@ -149,7 +184,7 @@ def check_auth_provider_creds(ctx: typer.Context, auth_provider: str): ) # GitHub - elif auth_provider == schema.AuthenticationEnum.github.value.lower() and ( + elif auth_provider == AuthenticationEnum.github.value.lower() and ( not os.environ.get("GITHUB_CLIENT_ID") or not os.environ.get("GITHUB_CLIENT_SECRET") ): @@ -171,7 +206,7 @@ def check_auth_provider_creds(ctx: typer.Context, auth_provider: str): return auth_provider -def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: schema.ProviderEnum): +def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: ProviderEnum): """Validate that the necessary cloud credentials have been set as environment variables.""" if ctx.params.get("disable_prompt"): @@ -180,7 +215,7 @@ def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: schema.Provid cloud_provider = cloud_provider.lower() # AWS - if cloud_provider == schema.ProviderEnum.aws.value.lower() and ( + if cloud_provider == ProviderEnum.aws.value.lower() and ( not os.environ.get("AWS_ACCESS_KEY_ID") or not os.environ.get("AWS_SECRET_ACCESS_KEY") ): @@ -200,7 +235,7 @@ def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: schema.Provid ) # GCP - elif cloud_provider == schema.ProviderEnum.gcp.value.lower() and ( + elif cloud_provider == ProviderEnum.gcp.value.lower() and ( not os.environ.get("GOOGLE_CREDENTIALS") or not os.environ.get("PROJECT_ID") ): rich.print( @@ -219,7 +254,7 @@ def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: schema.Provid ) # DO - elif cloud_provider == schema.ProviderEnum.do.value.lower() and ( + elif cloud_provider == ProviderEnum.do.value.lower() and ( not os.environ.get("DIGITALOCEAN_TOKEN") or not os.environ.get("SPACES_ACCESS_KEY_ID") or not os.environ.get("SPACES_SECRET_ACCESS_KEY") @@ -246,7 +281,7 @@ def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: schema.Provid os.environ["AWS_SECRET_ACCESS_KEY"] = os.getenv("AWS_SECRET_ACCESS_KEY") # AZURE - elif cloud_provider == schema.ProviderEnum.azure.value.lower() and ( + elif cloud_provider == ProviderEnum.azure.value.lower() and ( not os.environ.get("ARM_CLIENT_ID") or not os.environ.get("ARM_CLIENT_SECRET") or not os.environ.get("ARM_SUBSCRIPTION_ID") @@ -281,9 +316,9 @@ def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: schema.Provid def nebari_subcommand(cli: typer.Typer): @cli.command() def init( - cloud_provider: schema.ProviderEnum = typer.Argument( - schema.ProviderEnum.local, - help=f"options: {enum_to_list(schema.ProviderEnum)}", + cloud_provider: ProviderEnum = typer.Argument( + ProviderEnum.local, + help=f"options: {enum_to_list(ProviderEnum)}", callback=check_cloud_provider_creds, is_eager=True, ), @@ -300,46 +335,47 @@ def init( "--project-name", "--project", "-p", - callback=check_project_name, + callback=typer_validate_regex(schema.namestr_regex, "Project name must begin with a letter and consist of letters, numbers, dashes, or underscores."), ), domain_name: typing.Optional[str] = typer.Option( - ..., + None, "--domain-name", "--domain", "-d", ), namespace: str = typer.Option( "dev", + callback=typer_validate_regex(schema.namestr_regex, "Namespace must begin with a letter and consist of letters, numbers, dashes, or underscores."), ), - auth_provider: schema.AuthenticationEnum = typer.Option( - schema.AuthenticationEnum.password, - help=f"options: {enum_to_list(schema.AuthenticationEnum)}", + auth_provider: AuthenticationEnum = typer.Option( + AuthenticationEnum.password, + help=f"options: {enum_to_list(AuthenticationEnum)}", callback=check_auth_provider_creds, ), auth_auto_provision: bool = typer.Option( False, ), - repository: schema.GitRepoEnum = typer.Option( + repository: GitRepoEnum = typer.Option( None, - help=f"options: {enum_to_list(schema.GitRepoEnum)}", + help=f"options: {enum_to_list(GitRepoEnum)}", ), repository_auto_provision: bool = typer.Option( False, ), - ci_provider: schema.CiEnum = typer.Option( - schema.CiEnum.none, - help=f"options: {enum_to_list(schema.CiEnum)}", + ci_provider: CiEnum = typer.Option( + CiEnum.none, + help=f"options: {enum_to_list(CiEnum)}", ), - terraform_state: schema.TerraformStateEnum = typer.Option( - schema.TerraformStateEnum.remote, - help=f"options: {enum_to_list(schema.TerraformStateEnum)}", + terraform_state: TerraformStateEnum = typer.Option( + TerraformStateEnum.remote, + help=f"options: {enum_to_list(TerraformStateEnum)}", ), kubernetes_version: str = typer.Option( "latest", ), ssl_cert_email: str = typer.Option( None, - callback=check_ssl_cert_email, + callback=typer_validate_regex(schema.email_regex, f"Email must be valid and match the regex {schema.email_regex}"), ), disable_prompt: bool = typer.Option( False, @@ -366,7 +402,7 @@ def init( [green]nebari init --guided-init[/green] """ - inputs = schema.InitInputs() + inputs = InitInputs() inputs.cloud_provider = cloud_provider inputs.project_name = project_name @@ -420,7 +456,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): ) # pull in default values for each of the below - inputs = schema.InitInputs() + inputs = InitInputs() # CLOUD PROVIDER rich.print( @@ -436,7 +472,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): # try: inputs.cloud_provider = questionary.select( "Where would you like to deploy your Nebari cluster?", - choices=enum_to_list(schema.ProviderEnum), + choices=enum_to_list(ProviderEnum), qmark=qmark, ).unsafe_ask() @@ -451,9 +487,9 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): - Letters from A to Z (upper and lower case) and numbers - Maximum accepted length of the name string is 16 characters """ - if inputs.cloud_provider == schema.ProviderEnum.aws.value.lower(): + if inputs.cloud_provider == ProviderEnum.aws.value.lower(): name_guidelines += "- Should NOT start with the string `aws`\n" - elif inputs.cloud_provider == schema.ProviderEnum.azure.value.lower(): + elif inputs.cloud_provider == ProviderEnum.azure.value.lower(): name_guidelines += "- Should NOT contain `-`\n" # PROJECT NAME @@ -465,12 +501,9 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): inputs.project_name = questionary.text( "What project name would you like to use?", qmark=qmark, - validate=lambda text: True if len(text) > 0 else "Please enter a value", + validate=questionary_validate_regex(schema.namestr_regex), ).unsafe_ask() - if not disable_checks: - check_project_name(ctx, inputs.project_name) - # DOMAIN NAME rich.print( ( @@ -497,7 +530,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): ) inputs.auth_provider = questionary.select( "What authentication provider would you like?", - choices=enum_to_list(schema.AuthenticationEnum), + choices=enum_to_list(AuthenticationEnum), qmark=qmark, ).unsafe_ask() @@ -506,7 +539,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): if ( inputs.auth_provider.lower() - == schema.AuthenticationEnum.auth0.value.lower() + == AuthenticationEnum.auth0.value.lower() ): inputs.auth_auto_provision = questionary.confirm( "Would you like us to auto provision the Auth0 Machine-to-Machine app?", @@ -517,7 +550,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): elif ( inputs.auth_provider.lower() - == schema.AuthenticationEnum.github.value.lower() + == AuthenticationEnum.github.value.lower() ): rich.print( ( @@ -545,7 +578,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): git_provider = questionary.select( "Which git provider would you like to use?", - choices=enum_to_list(schema.GitRepoEnum), + choices=enum_to_list(GitRepoEnum), qmark=qmark, ).unsafe_ask() @@ -563,7 +596,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): git_provider=git_provider, org_name=org_name, repo_name=repo_name ) - if git_provider == schema.GitRepoEnum.github.value.lower(): + if git_provider == GitRepoEnum.github.value.lower(): inputs.repository_auto_provision = questionary.confirm( f"Would you like nebari to create a remote repository on {git_provider}?", default=False, @@ -574,10 +607,10 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): if not disable_checks and inputs.repository_auto_provision: check_repository_creds(ctx, git_provider) - if git_provider == schema.GitRepoEnum.github.value.lower(): - inputs.ci_provider = schema.CiEnum.github_actions.value.lower() - elif git_provider == schema.GitRepoEnum.gitlab.value.lower(): - inputs.ci_provider = schema.CiEnum.gitlab_ci.value.lower() + if git_provider == GitRepoEnum.github.value.lower(): + inputs.ci_provider = CiEnum.github_actions.value.lower() + elif git_provider == GitRepoEnum.gitlab.value.lower(): + inputs.ci_provider = CiEnum.gitlab_ci.value.lower() # SSL CERTIFICATE if inputs.domain_name: @@ -621,7 +654,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): # TERRAFORM STATE inputs.terraform_state = questionary.select( "Where should the Terraform State be provisioned?", - choices=enum_to_list(schema.TerraformStateEnum), + choices=enum_to_list(TerraformStateEnum), qmark=qmark, ).unsafe_ask() diff --git a/src/_nebari/subcommands/keycloak.py b/src/_nebari/subcommands/keycloak.py index ddf144ced..c8d4efeaa 100644 --- a/src/_nebari/subcommands/keycloak.py +++ b/src/_nebari/subcommands/keycloak.py @@ -5,7 +5,7 @@ import typer from _nebari.keycloak import do_keycloak, export_keycloak_users -from nebari import schema +from _nebari.config import read_configuration from nebari.hookspecs import hookimpl @@ -38,8 +38,11 @@ def add_user( ), ): """Add a user to Keycloak. User will be automatically added to the [italic]analyst[/italic] group.""" + from nebari.plugins import nebari_plugin_manager + args = ["adduser", add_users[0], add_users[1]] - config = schema.read_configuration(config_filename) + config_schema = nebari_plugin_manager.config_schema + config = read_configuration(config_filename, config_schema) do_keycloak(config, *args) @app_keycloak.command(name="listusers") @@ -52,8 +55,11 @@ def list_users( ) ): """List the users in Keycloak.""" + from nebari.plugins import nebari_plugin_manager + args = ["listusers"] - config = schema.read_configuration(config_filename) + config_schema = nebari_plugin_manager.config_schema + config = read_configuration(config_filename, config_schema) do_keycloak(config, *args) @app_keycloak.command(name="export-users") @@ -74,7 +80,6 @@ def export_users( from nebari.plugins import nebari_plugin_manager config_schema = nebari_plugin_manager.config_schema - - config = schema.read_configuration(config_filename, config_schema=config_schema) + config = read_configuration(config_filename, config_schema=config_schema) r = export_keycloak_users(config, realm=realm) print(json.dumps(r, indent=4)) diff --git a/src/_nebari/subcommands/render.py b/src/_nebari/subcommands/render.py index a8f1f0448..1d7aa4823 100644 --- a/src/_nebari/subcommands/render.py +++ b/src/_nebari/subcommands/render.py @@ -3,7 +3,7 @@ import typer from _nebari.render import render_template -from nebari import schema +from _nebari.config import read_configuration from nebari.hookspecs import hookimpl @@ -38,5 +38,5 @@ def render( stages = nebari_plugin_manager.ordered_stages config_schema = nebari_plugin_manager.config_schema - config = schema.read_configuration(config_filename, config_schema=config_schema) + config = read_configuration(config_filename, config_schema=config_schema) render_template(output_directory, config, stages, dry_run=dry_run) diff --git a/src/_nebari/subcommands/support.py b/src/_nebari/subcommands/support.py index c3a82d40e..93b185fa2 100644 --- a/src/_nebari/subcommands/support.py +++ b/src/_nebari/subcommands/support.py @@ -5,7 +5,7 @@ import kubernetes.config import typer -from nebari import schema +from _nebari.config import read_configuration from nebari.hookspecs import hookimpl @@ -32,11 +32,14 @@ def support( The Nebari team recommends k9s to manage and inspect the state of the cluster. However, this command occasionally helpful for debugging purposes should the logs need to be shared. """ + from nebari.plugins import nebari_plugin_manager + kubernetes.config.kube_config.load_kube_config() v1 = kubernetes.client.CoreV1Api() - namespace = schema.read_configuration(config_filename).namespace + config_schema = nebari_plugin_manager.config_schema + namespace = read_configuration(config_filename, config_schema).namespace pods = v1.list_namespaced_pod(namespace=namespace) diff --git a/src/_nebari/subcommands/validate.py b/src/_nebari/subcommands/validate.py index 8cfbced11..37747269f 100644 --- a/src/_nebari/subcommands/validate.py +++ b/src/_nebari/subcommands/validate.py @@ -3,7 +3,7 @@ import typer from rich import print -from nebari import schema +from _nebari.config import read_configuration from nebari.hookspecs import hookimpl @@ -31,7 +31,7 @@ def validate( else: from nebari.plugins import nebari_plugin_manager - schema.read_configuration( + read_configuration( config_filename, config_schema=nebari_plugin_manager.config_schema ) print("[bold purple]Successfully validated configuration.[/bold purple]") diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 1d05d5637..b2c335282 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -12,8 +12,9 @@ from nebari import schema -from .utils import backup_config_file, load_yaml, yaml -from .version import __version__, rounded_ver_parse +from _nebari.config import backup_configuration +from _nebari.utils import load_yaml, yaml +from _nebari.version import __version__, rounded_ver_parse logger = logging.getLogger(__name__) @@ -53,7 +54,7 @@ def do_upgrade(config_filename, attempt_fixes=False): ) # Backup old file - backup_config_file(config_filename, f".{start_version or 'old'}") + backup_configuration(config_filename, f".{start_version or 'old'}") with config_filename.open("wt") as f: yaml.dump(config, f) @@ -349,7 +350,7 @@ def _version_specific_upgrade( if k not in {"users", "admin"} ] - backup_config_file(realm_import_filename) + backup_configuration(realm_import_filename) with realm_import_filename.open("wt") as f: json.dump(realm, f, indent=2) diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 60d4e07ad..da87e6109 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -7,13 +7,14 @@ import sys import threading import time -from pathlib import Path +import string +import secrets from typing import Dict, List +import escapism from ruamel.yaml import YAML # environment variable overrides -NEBARI_K8S_VERSION = os.getenv("NEBARI_K8S_VERSION", None) NEBARI_GH_BRANCH = os.getenv("NEBARI_GH_BRANCH", None) CONDA_FORGE_CHANNEL_DATA_URL = "https://conda.anaconda.org/conda-forge/channeldata.json" @@ -104,26 +105,6 @@ def load_yaml(config_filename: pathlib.Path): return config -def backup_config_file(filename: Path, extrasuffix: str = ""): - if not filename.exists(): - return - - # Backup old file - backup_filename = Path(f"{filename}{extrasuffix}.backup") - - if backup_filename.exists(): - i = 1 - while True: - next_backup_filename = Path(f"{backup_filename}~{i}") - if not next_backup_filename.exists(): - backup_filename = next_backup_filename - break - i = i + 1 - - filename.rename(backup_filename) - print(f"Backing up {filename} as {backup_filename}") - - @contextlib.contextmanager def modified_environ(*remove: List[str], **update: Dict[str, str]): """ @@ -199,3 +180,17 @@ def deep_merge(*args): return [*d1, *d2] else: # if they don't match use left one return d1 + + +def escape_string( + s: str, + safe_chars=string.ascii_letters + string.digits, + escape_char='-' +): + return escapism.escape(s, safe=safe_chars, escape_char=escape_char) + + +def random_secure_string( + length: int = 16, chars: str = string.ascii_lowercase + string.digits +): + return "".join(secrets.choice(chars) for i in range(length)) diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index 8ef489804..ef74d1827 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -115,7 +115,7 @@ def load_config(self, config_path: typing.Union[str, Path]): raise FileNotFoundError(f"Config file {config_path} not found") self.config_path = config_path - self.config = schema.read_configuration(config_path) + self.config = schema.read_configuration(config_path, self.config_schema) @property def ordered_stages(self): diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 3d44c607b..09d4e2398 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -7,27 +7,26 @@ import pydantic from pydantic import validator -from ruamel.yaml import YAML, yaml_object +from ruamel.yaml import yaml_object +from _nebari.utils import yaml from _nebari.version import __version__, rounded_ver_parse -yaml = YAML() -yaml.preserve_quotes = True -yaml.default_flow_style = False - # Regex for suitable project names namestr_regex = r"^[A-Za-z][A-Za-z\-_]*[A-Za-z]$" +letter_dash_underscore_pydantic = pydantic.constr(regex=namestr_regex) +email_regex = "^[^ @]+@[^ @]+\\.[^ @]+$" +email_pydantic = pydantic.constr(regex=email_regex) -@yaml_object(yaml) -class TerraformStateEnum(str, enum.Enum): - remote = "remote" - local = "local" - existing = "existing" - @classmethod - def to_yaml(cls, representer, node): - return representer.represent_str(node.value) +class Base(pydantic.BaseModel): + ... + + class Config: + extra = "forbid" + validate_assignment = True + allow_population_by_field_name = True @yaml_object(yaml) @@ -45,109 +44,18 @@ def to_yaml(cls, representer, node): @yaml_object(yaml) -class GitRepoEnum(str, enum.Enum): - github = "github.com" - gitlab = "gitlab.com" - - @classmethod - def to_yaml(cls, representer, node): - return representer.represent_str(node.value) - - -@yaml_object(yaml) -class CiEnum(str, enum.Enum): - github_actions = "github-actions" - gitlab_ci = "gitlab-ci" - none = "none" - - @classmethod - def to_yaml(cls, representer, node): - return representer.represent_str(node.value) - - -@yaml_object(yaml) -class AuthenticationEnum(str, enum.Enum): - password = "password" - github = "GitHub" - auth0 = "Auth0" - custom = "custom" +class TerraformStateEnum(str, enum.Enum): + remote = "remote" + local = "local" + existing = "existing" @classmethod def to_yaml(cls, representer, node): return representer.represent_str(node.value) -class Base(pydantic.BaseModel): - ... - - class Config: - extra = "forbid" - validate_assignment = True - allow_population_by_field_name = True - - -# ==================== Main =================== -letter_dash_underscore_pydantic = pydantic.constr(regex=namestr_regex) - - -def project_name_convention(value: typing.Any, values): - convention = """ - There are some project naming conventions which need to be followed. - First, ensure your name is compatible with the specific one for - your chosen Cloud provider. In addition, the project name should also obey the following - format requirements: - - Letters from A to Z (upper and lower case) and numbers; - - Maximum accepted length of the name string is 16 characters. - - If using AWS: names should not start with the string "aws"; - - If using Azure: names should not contain "-". - """ - if len(value) > 16: - raise ValueError( - "\n".join( - [ - convention, - "Maximum accepted length of the project name string is 16 characters.", - ] - ) - ) - elif values["provider"] == "azure" and ("-" in value): - raise ValueError( - "\n".join( - [convention, "Provider [azure] does not allow '-' in project name."] - ) - ) - elif values["provider"] == "aws" and value.startswith("aws"): - raise ValueError( - "\n".join( - [ - convention, - "Provider [aws] does not allow 'aws' as starting sequence in project name.", - ] - ) - ) - else: - return letter_dash_underscore_pydantic - - -class InitInputs(Base): - cloud_provider: ProviderEnum = ProviderEnum.local - project_name: str = "" - domain_name: typing.Optional[str] = None - namespace: typing.Optional[letter_dash_underscore_pydantic] = "dev" - auth_provider: AuthenticationEnum = AuthenticationEnum.password - auth_auto_provision: bool = False - repository: typing.Union[str, None] = None - repository_auto_provision: bool = False - ci_provider: CiEnum = CiEnum.none - terraform_state: TerraformStateEnum = TerraformStateEnum.remote - kubernetes_version: typing.Union[str, None] = None - ssl_cert_email: typing.Union[str, None] = None - disable_prompt: bool = False - output: pathlib.Path = pathlib.Path("nebari-config.yaml") - - class Main(Base): - project_name: str + project_name: letter_dash_underscore_pydantic namespace: letter_dash_underscore_pydantic = "dev" # In nebari_version only use major.minor.patch version - drop any pre/post/dev suffixes nebari_version: str = __version__ @@ -176,11 +84,6 @@ def check_default(cls, v): def is_version_accepted(cls, v): return v != "" and rounded_ver_parse(v) == rounded_ver_parse(__version__) - @validator("project_name") - def _project_name_convention(cls, value: typing.Any, values): - project_name_convention(value=value, values=values) - return value - def is_version_accepted(v): """ @@ -191,100 +94,4 @@ def is_version_accepted(v): return Main.is_version_accepted(v) -def set_nested_attribute(data: typing.Any, attrs: typing.List[str], value: typing.Any): - """Takes an arbitrary set of attributes and accesses the deep - nested object config to set value - - """ - - def _get_attr(d: typing.Any, attr: str): - if hasattr(d, "__getitem__"): - if re.fullmatch(r"\d+", attr): - try: - return d[int(attr)] - except Exception: - return d[attr] - else: - return d[attr] - else: - return getattr(d, attr) - - def _set_attr(d: typing.Any, attr: str, value: typing.Any): - if hasattr(d, "__getitem__"): - if re.fullmatch(r"\d+", attr): - try: - d[int(attr)] = value - except Exception: - d[attr] = value - else: - d[attr] = value - else: - return setattr(d, attr, value) - - data_pos = data - for attr in attrs[:-1]: - data_pos = _get_attr(data_pos, attr) - _set_attr(data_pos, attrs[-1], value) - - -def set_config_from_environment_variables( - config: Base, keyword: str = "NEBARI_SECRET", separator: str = "__" -): - """Setting nebari configuration values from environment variables - - For example `NEBARI_SECRET__ci_cd__branch=master` would set `ci_cd.branch = "master"` - """ - nebari_secrets = [_ for _ in os.environ if _.startswith(keyword + separator)] - for secret in nebari_secrets: - attrs = secret[len(keyword + separator) :].split(separator) - try: - set_nested_attribute(config, attrs, os.environ[secret]) - except Exception as e: - print( - f"FAILED: setting secret from environment variable={secret} due to the following error\n {e}" - ) - sys.exit(1) - return config - - -def read_configuration( - config_filename: pathlib.Path, - read_environment: bool = True, - config_schema: pydantic.BaseModel = Main, -): - """Read configuration from multiple sources and apply validation""" - filename = pathlib.Path(config_filename) - - yaml = YAML() - yaml.preserve_quotes = True - yaml.default_flow_style = False - - if not filename.is_file(): - raise ValueError( - f"passed in configuration filename={config_filename} does not exist" - ) - - with filename.open() as f: - config = config_schema(**yaml.load(f.read())) - - if read_environment: - config = set_config_from_environment_variables(config) - - return config - - -def write_configuration( - config_filename: pathlib.Path, - config: typing.Union[Main, typing.Dict], - mode: str = "w", - config_schema: pydantic.BaseModel = Main, -): - yaml = YAML() - yaml.preserve_quotes = True - yaml.default_flow_style = False - - with config_filename.open(mode) as f: - if isinstance(config, config_schema): - yaml.dump(config.dict(), f) - else: - yaml.dump(config, f) +# from _nebari.config import read_configuration, write_configuration From ac04e965ccebe11b0d913224048a9b2641d43ca9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Jun 2023 21:01:59 +0000 Subject: [PATCH 092/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/initialize.py | 11 ++--- src/_nebari/stages/infrastructure/__init__.py | 41 ++++++++++--------- .../stages/kubernetes_services/__init__.py | 2 +- src/_nebari/subcommands/deploy.py | 2 +- src/_nebari/subcommands/destroy.py | 6 +-- src/_nebari/subcommands/dev.py | 2 +- src/_nebari/subcommands/init.py | 41 +++++++++++-------- src/_nebari/subcommands/keycloak.py | 2 +- src/_nebari/subcommands/render.py | 2 +- src/_nebari/upgrade.py | 3 +- src/_nebari/utils.py | 8 ++-- src/nebari/schema.py | 5 --- 12 files changed, 60 insertions(+), 65 deletions(-) diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index 0d7e9df42..3a6750983 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -1,25 +1,21 @@ import logging import os import re -import secrets -import string import tempfile -import requests import pydantic +import requests -from _nebari.utils import random_secure_string from _nebari.provider import git from _nebari.provider.cicd import github from _nebari.provider.oauth.auth0 import create_client -from _nebari.version import __version__ - from _nebari.stages.bootstrap import CiEnum from _nebari.stages.infrastructure import ProviderEnum from _nebari.stages.kubernetes_ingress import CertificateEnum from _nebari.stages.kubernetes_keycloak import AuthenticationEnum from _nebari.stages.terraform_state import TerraformStateEnum - +from _nebari.utils import random_secure_string +from _nebari.version import __version__ logger = logging.getLogger(__name__) @@ -134,6 +130,7 @@ def render_config( # validate configuration and convert to model from nebari.plugins import nebari_plugin_manager + config_model = nebari_plugin_manager.config_schema.parse_obj(config) if repository_auto_provision: diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index f5da3f864..17dc0dffd 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -5,12 +5,12 @@ import sys import tempfile import typing -import string from typing import Any, Dict, List, Optional import pydantic from _nebari import constants +from _nebari.provider import terraform from _nebari.provider.cloud import ( amazon_web_services, azure_cloud, @@ -18,11 +18,8 @@ google_cloud, ) from _nebari.stages.base import NebariTerraformStage -from _nebari.provider import terraform -from _nebari.stages.tf_objects import ( - NebariTerraformState, -) -from _nebari.utils import modified_environ, escape_string, deep_merge +from _nebari.stages.tf_objects import NebariTerraformState +from _nebari.utils import escape_string, modified_environ from nebari import schema from nebari.hookspecs import NebariStage, hookimpl @@ -573,7 +570,9 @@ def tf_objects(self) -> List[Dict]: ] elif self.config.provider == ProviderEnum.aws: return [ - terraform.Provider("aws", region=nebari_config.amazon_web_services.region), + terraform.Provider( + "aws", region=nebari_config.amazon_web_services.region + ), NebariTerraformState(self.name, self.config), ] else: @@ -581,21 +580,17 @@ def tf_objects(self) -> List[Dict]: def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): def escape_project_name(project_name: str, provider: ProviderEnum): - if provider == ProviderEnum.azure and '-' in project_name: - project_name = escape_string( - project_name, - escape_char='a' - ) + if provider == ProviderEnum.azure and "-" in project_name: + project_name = escape_string(project_name, escape_char="a") - if provider == ProviderEnum.aws and project_name.startswith('aws'): - project_name = 'a' + project_name + if provider == ProviderEnum.aws and project_name.startswith("aws"): + project_name = "a" + project_name if len(project_name) > 16: project_name = project_name[:16] return project_name - if self.config.provider == ProviderEnum.local: return LocalInputVars(kube_context=self.config.local.kube_context).dict() elif self.config.provider == ProviderEnum.existing: @@ -604,7 +599,9 @@ def escape_project_name(project_name: str, provider: ProviderEnum): ).dict() elif self.config.provider == ProviderEnum.do: return DigitalOceanInputVars( - name=escape_project_name(self.config.project_name, self.config.provider), + name=escape_project_name( + self.config.project_name, self.config.provider + ), environment=self.config.namespace, region=self.config.digital_ocean.region, tags=self.config.digital_ocean.tags, @@ -613,7 +610,9 @@ def escape_project_name(project_name: str, provider: ProviderEnum): ).dict() elif self.config.provider == ProviderEnum.gcp: return GCPInputVars( - name=escape_project_name(self.config.project_name, self.config.provider), + name=escape_project_name( + self.config.project_name, self.config.provider + ), environment=self.config.namespace, region=self.config.google_cloud_platform.region, project_id=self.config.google_cloud_platform.project, @@ -642,7 +641,9 @@ def escape_project_name(project_name: str, provider: ProviderEnum): ).dict() elif self.config.provider == ProviderEnum.azure: return AzureInputVars( - name=escape_project_name(self.config.project_name, self.config.provider), + name=escape_project_name( + self.config.project_name, self.config.provider + ), environment=self.config.namespace, region=self.config.azure.region, kubernetes_version=self.config.azure.kubernetes_version, @@ -661,7 +662,9 @@ def escape_project_name(project_name: str, provider: ProviderEnum): ).dict() elif self.config.provider == ProviderEnum.aws: return AWSInputVars( - name=escape_project_name(self.config.project_name, self.config.provider), + name=escape_project_name( + self.config.project_name, self.config.provider + ), environment=self.config.namespace, existing_subnet_ids=self.config.amazon_web_services.existing_subnet_ids, existing_security_group_id=self.config.amazon_web_services.existing_security_group_ids, diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index 9b4e25495..ca74cee8c 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -1,8 +1,8 @@ -import time import enum import json import os import sys +import time import typing from typing import Any, Dict, List from urllib.parse import urlencode diff --git a/src/_nebari/subcommands/deploy.py b/src/_nebari/subcommands/deploy.py index 765e788af..6c0564a50 100644 --- a/src/_nebari/subcommands/deploy.py +++ b/src/_nebari/subcommands/deploy.py @@ -2,9 +2,9 @@ import typer +from _nebari.config import read_configuration from _nebari.deploy import deploy_configuration from _nebari.render import render_template -from _nebari.config import read_configuration from nebari.hookspecs import hookimpl diff --git a/src/_nebari/subcommands/destroy.py b/src/_nebari/subcommands/destroy.py index 6c317116d..d94f8cd05 100644 --- a/src/_nebari/subcommands/destroy.py +++ b/src/_nebari/subcommands/destroy.py @@ -2,9 +2,9 @@ import typer +from _nebari.config import read_configuration from _nebari.destroy import destroy_configuration from _nebari.render import render_template -from _nebari.config import read_configuration from nebari.hookspecs import hookimpl @@ -44,9 +44,7 @@ def destroy( def _run_destroy( config_filename=config_filename, disable_render=disable_render ): - config = read_configuration( - config_filename, config_schema=config_schema - ) + config = read_configuration(config_filename, config_schema=config_schema) if not disable_render: render_template(output_directory, config, stages) diff --git a/src/_nebari/subcommands/dev.py b/src/_nebari/subcommands/dev.py index c3c104f06..59bfc77d5 100644 --- a/src/_nebari/subcommands/dev.py +++ b/src/_nebari/subcommands/dev.py @@ -3,8 +3,8 @@ import typer -from _nebari.keycloak import keycloak_rest_api_call from _nebari.config import read_configuration +from _nebari.keycloak import keycloak_rest_api_call from nebari.hookspecs import hookimpl diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index d05393fd3..cf128ca99 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -1,24 +1,22 @@ +import enum import os import pathlib import re import typing -import enum import questionary import rich import typer from pydantic import BaseModel -from _nebari.initialize import render_config from _nebari.config import write_configuration -from nebari import schema -from nebari.hookspecs import hookimpl - +from _nebari.initialize import render_config from _nebari.stages.bootstrap import CiEnum from _nebari.stages.infrastructure import ProviderEnum -from _nebari.stages.kubernetes_ingress import CertificateEnum from _nebari.stages.kubernetes_keycloak import AuthenticationEnum from _nebari.stages.terraform_state import TerraformStateEnum +from nebari import schema +from nebari.hookspecs import hookimpl MISSING_CREDS_TEMPLATE = "Unable to locate your {provider} credentials, refer to this guide on how to generate them:\n\n[green]\t{link_to_docs}[/green]\n\n" LINKS_TO_DOCS_TEMPLATE = ( @@ -103,7 +101,9 @@ def handle_init(inputs: InitInputs, config_schema: BaseModel): try: write_configuration( - inputs.output, config, mode="x", + inputs.output, + config, + mode="x", ) except FileExistsError: raise ValueError( @@ -138,6 +138,7 @@ def callback(value): return value message = error_message or f"Does not match {regex}" raise typer.BadParameter(message) + return callback @@ -148,6 +149,7 @@ def callback(value): message = error_message or f"Invalid input. Does not match {regex}" return message + return callback @@ -335,7 +337,10 @@ def init( "--project-name", "--project", "-p", - callback=typer_validate_regex(schema.namestr_regex, "Project name must begin with a letter and consist of letters, numbers, dashes, or underscores."), + callback=typer_validate_regex( + schema.namestr_regex, + "Project name must begin with a letter and consist of letters, numbers, dashes, or underscores.", + ), ), domain_name: typing.Optional[str] = typer.Option( None, @@ -345,7 +350,10 @@ def init( ), namespace: str = typer.Option( "dev", - callback=typer_validate_regex(schema.namestr_regex, "Namespace must begin with a letter and consist of letters, numbers, dashes, or underscores."), + callback=typer_validate_regex( + schema.namestr_regex, + "Namespace must begin with a letter and consist of letters, numbers, dashes, or underscores.", + ), ), auth_provider: AuthenticationEnum = typer.Option( AuthenticationEnum.password, @@ -375,7 +383,10 @@ def init( ), ssl_cert_email: str = typer.Option( None, - callback=typer_validate_regex(schema.email_regex, f"Email must be valid and match the regex {schema.email_regex}"), + callback=typer_validate_regex( + schema.email_regex, + f"Email must be valid and match the regex {schema.email_regex}", + ), ), disable_prompt: bool = typer.Option( False, @@ -537,10 +548,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): if not disable_checks: check_auth_provider_creds(ctx, auth_provider=inputs.auth_provider) - if ( - inputs.auth_provider.lower() - == AuthenticationEnum.auth0.value.lower() - ): + if inputs.auth_provider.lower() == AuthenticationEnum.auth0.value.lower(): inputs.auth_auto_provision = questionary.confirm( "Would you like us to auto provision the Auth0 Machine-to-Machine app?", default=False, @@ -548,10 +556,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): auto_enter=False, ).unsafe_ask() - elif ( - inputs.auth_provider.lower() - == AuthenticationEnum.github.value.lower() - ): + elif inputs.auth_provider.lower() == AuthenticationEnum.github.value.lower(): rich.print( ( ":warning: If you haven't done so already, please ensure the following:\n" diff --git a/src/_nebari/subcommands/keycloak.py b/src/_nebari/subcommands/keycloak.py index c8d4efeaa..8f57d3417 100644 --- a/src/_nebari/subcommands/keycloak.py +++ b/src/_nebari/subcommands/keycloak.py @@ -4,8 +4,8 @@ import typer -from _nebari.keycloak import do_keycloak, export_keycloak_users from _nebari.config import read_configuration +from _nebari.keycloak import do_keycloak, export_keycloak_users from nebari.hookspecs import hookimpl diff --git a/src/_nebari/subcommands/render.py b/src/_nebari/subcommands/render.py index 1d7aa4823..9c260061f 100644 --- a/src/_nebari/subcommands/render.py +++ b/src/_nebari/subcommands/render.py @@ -2,8 +2,8 @@ import typer -from _nebari.render import render_template from _nebari.config import read_configuration +from _nebari.render import render_template from nebari.hookspecs import hookimpl diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index b2c335282..81ac57bde 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -10,11 +10,10 @@ from pydantic.error_wrappers import ValidationError from rich.prompt import Prompt -from nebari import schema - from _nebari.config import backup_configuration from _nebari.utils import load_yaml, yaml from _nebari.version import __version__, rounded_ver_parse +from nebari import schema logger = logging.getLogger(__name__) diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index da87e6109..0781f951c 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -2,13 +2,13 @@ import functools import os import re +import secrets import signal +import string import subprocess import sys import threading import time -import string -import secrets from typing import Dict, List import escapism @@ -183,9 +183,7 @@ def deep_merge(*args): def escape_string( - s: str, - safe_chars=string.ascii_letters + string.digits, - escape_char='-' + s: str, safe_chars=string.ascii_letters + string.digits, escape_char="-" ): return escapism.escape(s, safe=safe_chars, escape_char=escape_char) diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 09d4e2398..0440c2fb2 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -1,9 +1,4 @@ import enum -import os -import pathlib -import re -import sys -import typing import pydantic from pydantic import validator From 73776ac8668c3a6707c95c6ad3366a3a2c7b2205 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 29 Jun 2023 17:33:05 -0400 Subject: [PATCH 093/147] Adding changes --- src/_nebari/initialize.py | 2 +- src/_nebari/stages/infrastructure/__init__.py | 70 ++++++++----------- src/_nebari/stages/tf_objects.py | 20 +++--- src/_nebari/subcommands/init.py | 3 +- src/nebari/schema.py | 12 +--- 5 files changed, 43 insertions(+), 64 deletions(-) diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index 3a6750983..c55bae1ca 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -10,7 +10,7 @@ from _nebari.provider.cicd import github from _nebari.provider.oauth.auth0 import create_client from _nebari.stages.bootstrap import CiEnum -from _nebari.stages.infrastructure import ProviderEnum +from nebari.schema import ProviderEnum from _nebari.stages.kubernetes_ingress import CertificateEnum from _nebari.stages.kubernetes_keycloak import AuthenticationEnum from _nebari.stages.terraform_state import TerraformStateEnum diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 17dc0dffd..000312096 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -28,20 +28,6 @@ def get_kubeconfig_filename(): return str(pathlib.Path(tempfile.gettempdir()) / "NEBARI_KUBECONFIG") -@schema.yaml_object(schema.yaml) -class ProviderEnum(str, enum.Enum): - local = "local" - existing = "existing" - do = "do" - aws = "aws" - gcp = "gcp" - azure = "azure" - - @classmethod - def to_yaml(cls, representer, node): - return representer.represent_str(node.value) - - class LocalInputVars(schema.Base): kubeconfig_filename: str = get_kubeconfig_filename() kube_context: Optional[str] @@ -150,27 +136,27 @@ class AWSInputVars(schema.Base): def _calculate_node_groups(config: schema.Main): - if config.provider == ProviderEnum.aws: + if config.provider == schema.ProviderEnum.aws: return { group: {"key": "eks.amazonaws.com/nodegroup", "value": group} for group in ["general", "user", "worker"] } - elif config.provider == ProviderEnum.gcp: + elif config.provider == schema.ProviderEnum.gcp: return { group: {"key": "cloud.google.com/gke-nodepool", "value": group} for group in ["general", "user", "worker"] } - elif config.provider == ProviderEnum.azure: + elif config.provider == schema.ProviderEnum.azure: return { group: {"key": "azure-node-pool", "value": group} for group in ["general", "user", "worker"] } - elif config.provider == ProviderEnum.do: + elif config.provider == schema.ProviderEnum.do: return { group: {"key": "doks.digitalocean.com/node-pool", "value": group} for group in ["general", "user", "worker"] } - elif config.provider == ProviderEnum.existing: + elif config.provider == schema.ProviderEnum.existing: return config.existing.node_selectors else: return config.local.dict()["node_selectors"] @@ -439,7 +425,6 @@ class ExistingProvider(schema.Base): class InputSchema(schema.Base): - provider: ProviderEnum = ProviderEnum.local local: typing.Optional[LocalProvider] existing: typing.Optional[ExistingProvider] google_cloud_platform: typing.Optional[GoogleCloudPlatformProvider] @@ -449,27 +434,27 @@ class InputSchema(schema.Base): @pydantic.root_validator def check_provider(cls, values): - if values["provider"] == ProviderEnum.local and values.get("local") is None: + if values["provider"] == schema.ProviderEnum.local and values.get("local") is None: values["local"] = LocalProvider() elif ( - values["provider"] == ProviderEnum.existing + values["provider"] == schema.ProviderEnum.existing and values.get("existing") is None ): values["existing"] = ExistingProvider() elif ( - values["provider"] == ProviderEnum.gcp + values["provider"] == schema.ProviderEnum.gcp and values.get("google_cloud_platform") is None ): values["google_cloud_platform"] = GoogleCloudPlatformProvider() elif ( - values["provider"] == ProviderEnum.aws + values["provider"] == schema.ProviderEnum.aws and values.get("amazon_web_services") is None ): values["amazon_web_services"] = AmazonWebServicesProvider() - elif values["provider"] == ProviderEnum.azure and values.get("azure") is None: + elif values["provider"] == schema.ProviderEnum.azure and values.get("azure") is None: values["azure"] = AzureProvider() elif ( - values["provider"] == ProviderEnum.do + values["provider"] == schema.ProviderEnum.do and values.get("digital_ocean") is None ): values["digital_ocean"] = DigitalOceanProvider() @@ -551,7 +536,7 @@ def stage_prefix(self): return pathlib.Path("stages") / self.name / self.config.provider.value def tf_objects(self) -> List[Dict]: - if self.config.provider == ProviderEnum.gcp: + if self.config.provider == schema.ProviderEnum.gcp: return [ terraform.Provider( "google", @@ -560,15 +545,15 @@ def tf_objects(self) -> List[Dict]: ), NebariTerraformState(self.name, self.config), ] - elif self.config.provider == ProviderEnum.do: + elif self.config.provider == schema.ProviderEnum.do: return [ NebariTerraformState(self.name, self.config), ] - elif self.config.provider == ProviderEnum.azure: + elif self.config.provider == schema.ProviderEnum.azure: return [ NebariTerraformState(self.name, self.config), ] - elif self.config.provider == ProviderEnum.aws: + elif self.config.provider == schema.ProviderEnum.aws: return [ terraform.Provider( "aws", region=nebari_config.amazon_web_services.region @@ -579,25 +564,28 @@ def tf_objects(self) -> List[Dict]: return [] def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): - def escape_project_name(project_name: str, provider: ProviderEnum): - if provider == ProviderEnum.azure and "-" in project_name: - project_name = escape_string(project_name, escape_char="a") + def escape_project_name(project_name: str, provider: schema.ProviderEnum): + if provider == schema.ProviderEnum.azure and '-' in project_name: + project_name = escape_string( + project_name, + escape_char='a' + ) - if provider == ProviderEnum.aws and project_name.startswith("aws"): - project_name = "a" + project_name + if provider == schema.ProviderEnum.aws and project_name.startswith('aws'): + project_name = 'a' + project_name if len(project_name) > 16: project_name = project_name[:16] return project_name - if self.config.provider == ProviderEnum.local: + if self.config.provider == schema.ProviderEnum.local: return LocalInputVars(kube_context=self.config.local.kube_context).dict() - elif self.config.provider == ProviderEnum.existing: + elif self.config.provider == schema.ProviderEnum.existing: return ExistingInputVars( kube_context=self.config.existing.kube_context ).dict() - elif self.config.provider == ProviderEnum.do: + elif self.config.provider == schema.ProviderEnum.do: return DigitalOceanInputVars( name=escape_project_name( self.config.project_name, self.config.provider @@ -608,7 +596,7 @@ def escape_project_name(project_name: str, provider: ProviderEnum): kubernetes_version=self.config.digital_ocean.kubernetes_version, node_groups=self.config.digital_ocean.node_groups, ).dict() - elif self.config.provider == ProviderEnum.gcp: + elif self.config.provider == schema.ProviderEnum.gcp: return GCPInputVars( name=escape_project_name( self.config.project_name, self.config.provider @@ -639,7 +627,7 @@ def escape_project_name(project_name: str, provider: ProviderEnum): master_authorized_networks_config=self.config.google_cloud_platform.master_authorized_networks_config, private_cluster_config=self.config.google_cloud_platform.private_cluster_config, ).dict() - elif self.config.provider == ProviderEnum.azure: + elif self.config.provider == schema.ProviderEnum.azure: return AzureInputVars( name=escape_project_name( self.config.project_name, self.config.provider @@ -660,7 +648,7 @@ def escape_project_name(project_name: str, provider: ProviderEnum): vnet_subnet_id=self.config.azure.vnet_subnet_id, private_cluster_enabled=self.config.azure.private_cluster_enabled, ).dict() - elif self.config.provider == ProviderEnum.aws: + elif self.config.provider == schema.ProviderEnum.aws: return AWSInputVars( name=escape_project_name( self.config.project_name, self.config.provider diff --git a/src/_nebari/stages/tf_objects.py b/src/_nebari/stages/tf_objects.py index 58bf25a1e..c4e619613 100644 --- a/src/_nebari/stages/tf_objects.py +++ b/src/_nebari/stages/tf_objects.py @@ -4,7 +4,7 @@ def NebariKubernetesProvider(nebari_config: schema.Main): - if nebari_config.provider == schema.ProviderEnum.aws: + if nebari_config.provider == "aws": cluster_name = f"{nebari_config.project_name}-{nebari_config.namespace}" # The AWS provider needs to be added, as we are using aws related resources #1254 return deep_merge( @@ -26,7 +26,7 @@ def NebariKubernetesProvider(nebari_config: schema.Main): def NebariHelmProvider(nebari_config: schema.Main): - if nebari_config.provider == schema.ProviderEnum.aws: + if nebari_config.provider == "aws": cluster_name = f"{nebari_config.project_name}-{nebari_config.namespace}" return deep_merge( @@ -45,14 +45,14 @@ def NebariHelmProvider(nebari_config: schema.Main): def NebariTerraformState(directory: str, nebari_config: schema.Main): - if nebari_config.terraform_state.type == schema.TerraformStateEnum.local: + if nebari_config.terraform_state.type == "local": return {} - elif nebari_config.terraform_state.type == schema.TerraformStateEnum.existing: + elif nebari_config.terraform_state.type == "existing": return TerraformBackend( nebari_config["terraform_state"]["backend"], **nebari_config["terraform_state"]["config"], ) - elif nebari_config.provider == schema.ProviderEnum.aws: + elif nebari_config.provider == "aws": return TerraformBackend( "s3", bucket=f"{nebari_config.project_name}-{nebari_config.namespace}-terraform-state", @@ -61,13 +61,13 @@ def NebariTerraformState(directory: str, nebari_config: schema.Main): encrypt=True, dynamodb_table=f"{nebari_config.project_name}-{nebari_config.namespace}-terraform-state-lock", ) - elif nebari_config.provider == schema.ProviderEnum.gcp: + elif nebari_config.provider == "gcp": return TerraformBackend( "gcs", bucket=f"{nebari_config.project_name}-{nebari_config.namespace}-terraform-state", prefix=f"terraform/{nebari_config.project_name}/{directory}", ) - elif nebari_config.provider == schema.ProviderEnum.do: + elif nebari_config.provider == "do": return TerraformBackend( "s3", endpoint=f"{nebari_config.digital_ocean.region}.digitaloceanspaces.com", @@ -77,7 +77,7 @@ def NebariTerraformState(directory: str, nebari_config: schema.Main): skip_credentials_validation=True, skip_metadata_api_check=True, ) - elif nebari_config.provider == schema.ProviderEnum.azure: + elif nebari_config.provider == "azure": return TerraformBackend( "azurerm", resource_group_name=f"{nebari_config.project_name}-{nebari_config.namespace}-state", @@ -86,7 +86,7 @@ def NebariTerraformState(directory: str, nebari_config: schema.Main): container_name=f"{nebari_config.project_name}-{nebari_config.namespace}-state", key=f"terraform/{nebari_config.project_name}-{nebari_config.namespace}/{directory}", ) - elif nebari_config.provider == schema.ProviderEnum.existing: + elif nebari_config.provider == "existing": optional_kwargs = {} if "kube_context" in nebari_config.existing: optional_kwargs["config_context"] = nebari_config.existing.kube_context @@ -96,7 +96,7 @@ def NebariTerraformState(directory: str, nebari_config: schema.Main): load_config_file=True, **optional_kwargs, ) - elif nebari_config.provider == schema.ProviderEnum.local: + elif nebari_config.provider == "local": optional_kwargs = {} if "kube_context" in nebari_config.local: optional_kwargs["config_context"] = nebari_config.local.kube_context diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index cf128ca99..472637ea5 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -12,7 +12,8 @@ from _nebari.config import write_configuration from _nebari.initialize import render_config from _nebari.stages.bootstrap import CiEnum -from _nebari.stages.infrastructure import ProviderEnum +from nebari.schema import ProviderEnum +from _nebari.stages.kubernetes_ingress import CertificateEnum from _nebari.stages.kubernetes_keycloak import AuthenticationEnum from _nebari.stages.terraform_state import TerraformStateEnum from nebari import schema diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 0440c2fb2..3d17912da 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -38,20 +38,10 @@ def to_yaml(cls, representer, node): return representer.represent_str(node.value) -@yaml_object(yaml) -class TerraformStateEnum(str, enum.Enum): - remote = "remote" - local = "local" - existing = "existing" - - @classmethod - def to_yaml(cls, representer, node): - return representer.represent_str(node.value) - - class Main(Base): project_name: letter_dash_underscore_pydantic namespace: letter_dash_underscore_pydantic = "dev" + provider: ProviderEnum = ProviderEnum.local # In nebari_version only use major.minor.patch version - drop any pre/post/dev suffixes nebari_version: str = __version__ From c894cf639ed715d89b5b28ae3d600907f41ff541 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Jun 2023 21:33:21 +0000 Subject: [PATCH 094/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/initialize.py | 2 +- src/_nebari/stages/infrastructure/__init__.py | 24 ++++++++++--------- src/_nebari/subcommands/init.py | 3 +-- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index c55bae1ca..1b33bfa69 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -10,12 +10,12 @@ from _nebari.provider.cicd import github from _nebari.provider.oauth.auth0 import create_client from _nebari.stages.bootstrap import CiEnum -from nebari.schema import ProviderEnum from _nebari.stages.kubernetes_ingress import CertificateEnum from _nebari.stages.kubernetes_keycloak import AuthenticationEnum from _nebari.stages.terraform_state import TerraformStateEnum from _nebari.utils import random_secure_string from _nebari.version import __version__ +from nebari.schema import ProviderEnum logger = logging.getLogger(__name__) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 000312096..06c6f1702 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -1,5 +1,4 @@ import contextlib -import enum import inspect import pathlib import sys @@ -434,7 +433,10 @@ class InputSchema(schema.Base): @pydantic.root_validator def check_provider(cls, values): - if values["provider"] == schema.ProviderEnum.local and values.get("local") is None: + if ( + values["provider"] == schema.ProviderEnum.local + and values.get("local") is None + ): values["local"] = LocalProvider() elif ( values["provider"] == schema.ProviderEnum.existing @@ -451,7 +453,10 @@ def check_provider(cls, values): and values.get("amazon_web_services") is None ): values["amazon_web_services"] = AmazonWebServicesProvider() - elif values["provider"] == schema.ProviderEnum.azure and values.get("azure") is None: + elif ( + values["provider"] == schema.ProviderEnum.azure + and values.get("azure") is None + ): values["azure"] = AzureProvider() elif ( values["provider"] == schema.ProviderEnum.do @@ -565,14 +570,11 @@ def tf_objects(self) -> List[Dict]: def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): def escape_project_name(project_name: str, provider: schema.ProviderEnum): - if provider == schema.ProviderEnum.azure and '-' in project_name: - project_name = escape_string( - project_name, - escape_char='a' - ) - - if provider == schema.ProviderEnum.aws and project_name.startswith('aws'): - project_name = 'a' + project_name + if provider == schema.ProviderEnum.azure and "-" in project_name: + project_name = escape_string(project_name, escape_char="a") + + if provider == schema.ProviderEnum.aws and project_name.startswith("aws"): + project_name = "a" + project_name if len(project_name) > 16: project_name = project_name[:16] diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index 472637ea5..32551c9f4 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -12,12 +12,11 @@ from _nebari.config import write_configuration from _nebari.initialize import render_config from _nebari.stages.bootstrap import CiEnum -from nebari.schema import ProviderEnum -from _nebari.stages.kubernetes_ingress import CertificateEnum from _nebari.stages.kubernetes_keycloak import AuthenticationEnum from _nebari.stages.terraform_state import TerraformStateEnum from nebari import schema from nebari.hookspecs import hookimpl +from nebari.schema import ProviderEnum MISSING_CREDS_TEMPLATE = "Unable to locate your {provider} credentials, refer to this guide on how to generate them:\n\n[green]\t{link_to_docs}[/green]\n\n" LINKS_TO_DOCS_TEMPLATE = ( From 31ef601753e078fca742f7d2521d5b90158a01c6 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 29 Jun 2023 17:44:12 -0400 Subject: [PATCH 095/147] Missing import --- src/_nebari/stages/infrastructure/__init__.py | 2 +- src/nebari/schema.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 06c6f1702..7d7a97899 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -18,7 +18,7 @@ ) from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import NebariTerraformState -from _nebari.utils import escape_string, modified_environ +from _nebari.utils import escape_string, modified_environ, random_secure_string from nebari import schema from nebari.hookspecs import NebariStage, hookimpl diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 3d17912da..597d7927f 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -1,7 +1,6 @@ import enum import pydantic -from pydantic import validator from ruamel.yaml import yaml_object from _nebari.utils import yaml @@ -51,7 +50,7 @@ class Main(Base): # If the nebari_version in the schema is old # we must tell the user to first run nebari upgrade - @validator("nebari_version", pre=True, always=True) + @pydantic.validator("nebari_version", pre=True, always=True) def check_default(cls, v): """ Always called even if nebari_version is not supplied at all (so defaults to ''). That way we can give a more helpful error message. @@ -77,6 +76,3 @@ def is_version_accepted(v): for deployment with the current Nebari package. """ return Main.is_version_accepted(v) - - -# from _nebari.config import read_configuration, write_configuration From 608c16f208083cf855da17f08686bbaae2587985 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 29 Jun 2023 19:35:43 -0400 Subject: [PATCH 096/147] Fixing path --- src/nebari/plugins.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index ef74d1827..9099a74e0 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -115,7 +115,9 @@ def load_config(self, config_path: typing.Union[str, Path]): raise FileNotFoundError(f"Config file {config_path} not found") self.config_path = config_path - self.config = schema.read_configuration(config_path, self.config_schema) + + from _nebari.config import read_configuration + self.config = read_configuration(config_path, self.config_schema) @property def ordered_stages(self): From bd527d923562e9fcf9d5f9eef6cb9d4a5f5ee3f8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Jun 2023 23:35:58 +0000 Subject: [PATCH 097/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/nebari/plugins.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index 9099a74e0..827d9d4f3 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -117,6 +117,7 @@ def load_config(self, config_path: typing.Union[str, Path]): self.config_path = config_path from _nebari.config import read_configuration + self.config = read_configuration(config_path, self.config_schema) @property From 840364a10cfd66bb11ba6a6fbe84a83ee02a58fd Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 29 Jun 2023 21:41:18 -0400 Subject: [PATCH 098/147] Adding digital ocean validation --- src/_nebari/initialize.py | 5 +- src/_nebari/provider/cloud/digital_ocean.py | 14 ----- src/_nebari/stages/infrastructure/__init__.py | 53 ++++++++++--------- .../stages/terraform_state/__init__.py | 34 ++++++++++++ src/_nebari/stages/tf_objects.py | 30 +++++------ src/_nebari/subcommands/validate.py | 12 +++-- src/nebari/plugins.py | 10 +--- src/nebari/schema.py | 19 ++++++- 8 files changed, 109 insertions(+), 68 deletions(-) diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index 1b33bfa69..ae7f5d780 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -131,7 +131,10 @@ def render_config( # validate configuration and convert to model from nebari.plugins import nebari_plugin_manager - config_model = nebari_plugin_manager.config_schema.parse_obj(config) + try: + config_model = nebari_plugin_manager.config_schema.parse_obj(config) + except pydantic.ValidationError as e: + print(str(e)) if repository_auto_provision: GITHUB_REGEX = "(https://)?github.com/([^/]+)/([^/]+)/?" diff --git a/src/_nebari/provider/cloud/digital_ocean.py b/src/_nebari/provider/cloud/digital_ocean.py index 984dc219e..e000ab764 100644 --- a/src/_nebari/provider/cloud/digital_ocean.py +++ b/src/_nebari/provider/cloud/digital_ocean.py @@ -10,8 +10,6 @@ def check_credentials(): DO_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-do" for variable in { - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", "SPACES_ACCESS_KEY_ID", "SPACES_SECRET_ACCESS_KEY", "DIGITALOCEAN_TOKEN", @@ -22,18 +20,6 @@ def check_credentials(): Please see the documentation for more information: {DO_ENV_DOCS}""" ) - if os.environ["AWS_ACCESS_KEY_ID"] != os.environ["SPACES_ACCESS_KEY_ID"]: - raise ValueError( - f"""The environment variables AWS_ACCESS_KEY_ID and SPACES_ACCESS_KEY_ID must be equal\n - See {DO_ENV_DOCS} for more information""" - ) - - if os.environ["AWS_SECRET_ACCESS_KEY"] != os.environ["SPACES_SECRET_ACCESS_KEY"]: - raise ValueError( - f"""The environment variables AWS_SECRET_ACCESS_KEY and SPACES_SECRET_ACCESS_KEY must be equal\n - See {DO_ENV_DOCS} for more information""" - ) - def digital_ocean_request(url, method="GET", json=None): BASE_DIGITALOCEAN_URL = "https://api.digitalocean.com/v2/" diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 7d7a97899..2acc4f75a 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -1,3 +1,4 @@ +import os import contextlib import inspect import pathlib @@ -49,7 +50,7 @@ class DigitalOceanInputVars(schema.Base): tags: typing.List[str] kubernetes_version: str node_groups: typing.Dict[str, DigitalOceanNodeGroup] - kubeconfig_filename: str + kubeconfig_filename: str = get_kubeconfig_filename() class GCPGuestAccelerators(schema.Base): @@ -200,6 +201,16 @@ class DigitalOceanNodeGroup(schema.Base): min_nodes: pydantic.conint(ge=1) = 1 max_nodes: pydantic.conint(ge=1) = 1 + @pydantic.validator('instance') + def _validate_instance(cls, value): + digital_ocean.check_credentials() + + available_instances = {_['slug'] for _ in digital_ocean.instances()} + if value not in available_instances: + raise ValueError(f'Digital Ocean instance size={instance} not one of available instance types={available_instances}') + + return value + class DigitalOceanProvider(schema.Base): region: str = "nyc3" @@ -218,10 +229,22 @@ class DigitalOceanProvider(schema.Base): } tags: typing.Optional[typing.List[str]] = [] + @pydantic.validator('region') + def _validate_region(cls, value): + digital_ocean.check_credentials() + + available_regions = set(_['slug'] for _ in digital_ocean.regions()) + if value not in available_regions: + raise ValueError(f'Digital Ocean region={value} is not one of {available_regions}') + return value + @pydantic.root_validator def _validate_kubernetes_version(cls, values): digital_ocean.check_credentials() + if "region" not in values: + raise ValueError("region required in order to set kubernetes version") + available_kubernetes_versions = digital_ocean.kubernetes_versions( values["region"] ) @@ -569,18 +592,6 @@ def tf_objects(self) -> List[Dict]: return [] def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): - def escape_project_name(project_name: str, provider: schema.ProviderEnum): - if provider == schema.ProviderEnum.azure and "-" in project_name: - project_name = escape_string(project_name, escape_char="a") - - if provider == schema.ProviderEnum.aws and project_name.startswith("aws"): - project_name = "a" + project_name - - if len(project_name) > 16: - project_name = project_name[:16] - - return project_name - if self.config.provider == schema.ProviderEnum.local: return LocalInputVars(kube_context=self.config.local.kube_context).dict() elif self.config.provider == schema.ProviderEnum.existing: @@ -589,9 +600,7 @@ def escape_project_name(project_name: str, provider: schema.ProviderEnum): ).dict() elif self.config.provider == schema.ProviderEnum.do: return DigitalOceanInputVars( - name=escape_project_name( - self.config.project_name, self.config.provider - ), + name=self.config.escaped_project_name, environment=self.config.namespace, region=self.config.digital_ocean.region, tags=self.config.digital_ocean.tags, @@ -600,9 +609,7 @@ def escape_project_name(project_name: str, provider: schema.ProviderEnum): ).dict() elif self.config.provider == schema.ProviderEnum.gcp: return GCPInputVars( - name=escape_project_name( - self.config.project_name, self.config.provider - ), + name=self.config.escaped_project_name, environment=self.config.namespace, region=self.config.google_cloud_platform.region, project_id=self.config.google_cloud_platform.project, @@ -631,9 +638,7 @@ def escape_project_name(project_name: str, provider: schema.ProviderEnum): ).dict() elif self.config.provider == schema.ProviderEnum.azure: return AzureInputVars( - name=escape_project_name( - self.config.project_name, self.config.provider - ), + name=self.config.escaped_project_name, environment=self.config.namespace, region=self.config.azure.region, kubernetes_version=self.config.azure.kubernetes_version, @@ -652,9 +657,7 @@ def escape_project_name(project_name: str, provider: schema.ProviderEnum): ).dict() elif self.config.provider == schema.ProviderEnum.aws: return AWSInputVars( - name=escape_project_name( - self.config.project_name, self.config.provider - ), + name=self.config.escaped_project_name, environment=self.config.namespace, existing_subnet_ids=self.config.amazon_web_services.existing_subnet_ids, existing_security_group_id=self.config.amazon_web_services.existing_security_group_ids, diff --git a/src/_nebari/stages/terraform_state/__init__.py b/src/_nebari/stages/terraform_state/__init__.py index a7944d8b1..8eba019a9 100644 --- a/src/_nebari/stages/terraform_state/__init__.py +++ b/src/_nebari/stages/terraform_state/__init__.py @@ -4,8 +4,10 @@ import pathlib import typing from typing import Any, Dict, List, Tuple +import contextlib from _nebari.stages.base import NebariTerraformStage +from _nebari.utils import modified_environ from nebari import schema from nebari.hookspecs import NebariStage, hookimpl @@ -166,6 +168,38 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): else: ValueError(f"Unknown provider: {self.config.provider}") + @contextlib.contextmanager + def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): + with super().deploy(stage_outputs): + env_mapping = {} + # DigitalOcean terraform remote state using Spaces Bucket + # assumes aws credentials thus we set them to match spaces credentials + if self.config.provider == schema.ProviderEnum.do: + env_mapping.update({ + "AWS_ACCESS_KEY_ID": os.environ["SPACES_ACCESS_KEY_ID"], + "AWS_SECRET_ACCESS_KEY": os.environ["SPACES_SECRET_ACCESS_KEY"], + }) + + with modified_environ(**env_mapping): + yield + + @contextlib.contextmanager + def destroy( + self, stage_outputs: Dict[str, Dict[str, Any]], status: Dict[str, bool] + ): + with super().destroy(stage_outputs, status): + env_mapping = {} + # DigitalOcean terraform remote state using Spaces Bucket + # assumes aws credentials thus we set them to match spaces credentials + if self.config.provider == schema.ProviderEnum.do: + env_mapping.update({ + "AWS_ACCESS_KEY_ID": os.environ["SPACES_ACCESS_KEY_ID"], + "AWS_SECRET_ACCESS_KEY": os.environ["SPACES_SECRET_ACCESS_KEY"], + }) + + with modified_environ(**env_mapping): + yield + @hookimpl def nebari_stage() -> List[NebariStage]: diff --git a/src/_nebari/stages/tf_objects.py b/src/_nebari/stages/tf_objects.py index c4e619613..f35b6aed1 100644 --- a/src/_nebari/stages/tf_objects.py +++ b/src/_nebari/stages/tf_objects.py @@ -5,7 +5,7 @@ def NebariKubernetesProvider(nebari_config: schema.Main): if nebari_config.provider == "aws": - cluster_name = f"{nebari_config.project_name}-{nebari_config.namespace}" + cluster_name = f"{nebari_config.escaped_project_name}-{nebari_config.namespace}" # The AWS provider needs to be added, as we are using aws related resources #1254 return deep_merge( Data("aws_eks_cluster", "default", name=cluster_name), @@ -27,7 +27,7 @@ def NebariKubernetesProvider(nebari_config: schema.Main): def NebariHelmProvider(nebari_config: schema.Main): if nebari_config.provider == "aws": - cluster_name = f"{nebari_config.project_name}-{nebari_config.namespace}" + cluster_name = f"{nebari_config.escaped_project_name}-{nebari_config.namespace}" return deep_merge( Data("aws_eks_cluster", "default", name=cluster_name), @@ -55,36 +55,36 @@ def NebariTerraformState(directory: str, nebari_config: schema.Main): elif nebari_config.provider == "aws": return TerraformBackend( "s3", - bucket=f"{nebari_config.project_name}-{nebari_config.namespace}-terraform-state", - key=f"terraform/{nebari_config.project_name}-{nebari_config.namespace}/{directory}.tfstate", + bucket=f"{nebari_config.escaped_project_name}-{nebari_config.namespace}-terraform-state", + key=f"terraform/{nebari_config.escaped_project_name}-{nebari_config.namespace}/{directory}.tfstate", region=nebari_config.amazon_web_services.region, encrypt=True, - dynamodb_table=f"{nebari_config.project_name}-{nebari_config.namespace}-terraform-state-lock", + dynamodb_table=f"{nebari_config.escaped_project_name}-{nebari_config.namespace}-terraform-state-lock", ) elif nebari_config.provider == "gcp": return TerraformBackend( "gcs", - bucket=f"{nebari_config.project_name}-{nebari_config.namespace}-terraform-state", - prefix=f"terraform/{nebari_config.project_name}/{directory}", + bucket=f"{nebari_config.escaped_project_name}-{nebari_config.namespace}-terraform-state", + prefix=f"terraform/{nebari_config.escaped_project_name}/{directory}", ) elif nebari_config.provider == "do": return TerraformBackend( "s3", endpoint=f"{nebari_config.digital_ocean.region}.digitaloceanspaces.com", region="us-west-1", # fake aws region required by terraform - bucket=f"{nebari_config.project_name}-{nebari_config.namespace}-terraform-state", - key=f"terraform/{nebari_config.project_name}-{nebari_config.namespace}/{directory}.tfstate", + bucket=f"{nebari_config.escaped_project_name}-{nebari_config.namespace}-terraform-state", + key=f"terraform/{nebari_config.escaped_project_name}-{nebari_config.namespace}/{directory}.tfstate", skip_credentials_validation=True, skip_metadata_api_check=True, ) elif nebari_config.provider == "azure": return TerraformBackend( "azurerm", - resource_group_name=f"{nebari_config.project_name}-{nebari_config.namespace}-state", + resource_group_name=f"{nebari_config.escaped_project_name}-{nebari_config.namespace}-state", # storage account must be globally unique - storage_account_name=f"{nebari_config.project_name}{nebari_config.namespace}{nebari_config.azure.storage_account_postfix}", - container_name=f"{nebari_config.project_name}-{nebari_config.namespace}-state", - key=f"terraform/{nebari_config.project_name}-{nebari_config.namespace}/{directory}", + storage_account_name=f"{nebari_config.escaped_project_name}{nebari_config.namespace}{nebari_config.azure.storage_account_postfix}", + container_name=f"{nebari_config.escaped_project_name}-{nebari_config.namespace}-state", + key=f"terraform/{nebari_config.escaped_project_name}-{nebari_config.namespace}/{directory}", ) elif nebari_config.provider == "existing": optional_kwargs = {} @@ -92,7 +92,7 @@ def NebariTerraformState(directory: str, nebari_config: schema.Main): optional_kwargs["config_context"] = nebari_config.existing.kube_context return TerraformBackend( "kubernetes", - secret_suffix=f"{nebari_config.project_name}-{nebari_config.namespace}-{directory}", + secret_suffix=f"{nebari_config.escaped_project_name}-{nebari_config.namespace}-{directory}", load_config_file=True, **optional_kwargs, ) @@ -102,7 +102,7 @@ def NebariTerraformState(directory: str, nebari_config: schema.Main): optional_kwargs["config_context"] = nebari_config.local.kube_context return TerraformBackend( "kubernetes", - secret_suffix=f"{nebari_config.project_name}-{nebari_config.namespace}-{directory}", + secret_suffix=f"{nebari_config.escaped_project_name}-{nebari_config.namespace}-{directory}", load_config_file=True, **optional_kwargs, ) diff --git a/src/_nebari/subcommands/validate.py b/src/_nebari/subcommands/validate.py index 37747269f..5bd8be2c3 100644 --- a/src/_nebari/subcommands/validate.py +++ b/src/_nebari/subcommands/validate.py @@ -2,6 +2,7 @@ import typer from rich import print +import pydantic from _nebari.config import read_configuration from nebari.hookspecs import hookimpl @@ -31,7 +32,10 @@ def validate( else: from nebari.plugins import nebari_plugin_manager - read_configuration( - config_filename, config_schema=nebari_plugin_manager.config_schema - ) - print("[bold purple]Successfully validated configuration.[/bold purple]") + try: + nebari_plugin_manager.read_config(config_filename) + print("[bold purple]Successfully validated configuration.[/bold purple]") + except pydantic.ValidationError as e: + print(f"[bold red]ERROR validating configuration {config_filename.absolute()}[/bold red]") + print(str(e)) + typer.Abort() diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index 827d9d4f3..2de05f80e 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -45,9 +45,6 @@ class NebariPluginManager: exclude_default_stages: bool = False exclude_stages: typing.List[str] = [] - config_path: typing.Union[Path, None] = None - config: typing.Union[pydantic.BaseModel, None] = None - def __init__(self) -> None: self.plugin_manager.add_hookspecs(hookspecs) @@ -107,18 +104,15 @@ def get_available_stages(self): return included_stages - def load_config(self, config_path: typing.Union[str, Path]): + def read_config(self, config_path: typing.Union[str, Path]): if isinstance(config_path, str): config_path = Path(config_path) if not config_path.exists(): raise FileNotFoundError(f"Config file {config_path} not found") - self.config_path = config_path - from _nebari.config import read_configuration - - self.config = read_configuration(config_path, self.config_schema) + return read_configuration(config_path, self.config_schema) @property def ordered_stages(self): diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 597d7927f..8d085ac84 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -3,7 +3,7 @@ import pydantic from ruamel.yaml import yaml_object -from _nebari.utils import yaml +from _nebari.utils import yaml, escape_string from _nebari.version import __version__, rounded_ver_parse # Regex for suitable project names @@ -68,6 +68,23 @@ def check_default(cls, v): def is_version_accepted(cls, v): return v != "" and rounded_ver_parse(v) == rounded_ver_parse(__version__) + @property + def escaped_project_name(self): + """Escaped project-name know to be compatible with all clouds + """ + project_name = self.project_name + + if self.provider == ProviderEnum.azure and "-" in project_name: + project_name = escape_string(project_name, escape_char="a") + + if self.provider == ProviderEnum.aws and project_name.startswith("aws"): + project_name = "a" + project_name + + if len(project_name) > 16: + project_name = project_name[:16] + + return project_name + def is_version_accepted(v): """ From 158958a7ad0db41d2e34c11cd4a6dfa6e86b9247 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Jun 2023 01:42:09 +0000 Subject: [PATCH 099/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/stages/infrastructure/__init__.py | 20 ++++++++++------- .../stages/terraform_state/__init__.py | 22 +++++++++++-------- src/_nebari/subcommands/validate.py | 11 ++++++---- src/nebari/plugins.py | 2 +- src/nebari/schema.py | 7 +++--- 5 files changed, 36 insertions(+), 26 deletions(-) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 2acc4f75a..04f331514 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -1,6 +1,6 @@ -import os import contextlib import inspect +import os import pathlib import sys import tempfile @@ -19,7 +19,7 @@ ) from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import NebariTerraformState -from _nebari.utils import escape_string, modified_environ, random_secure_string +from _nebari.utils import modified_environ, random_secure_string from nebari import schema from nebari.hookspecs import NebariStage, hookimpl @@ -201,13 +201,15 @@ class DigitalOceanNodeGroup(schema.Base): min_nodes: pydantic.conint(ge=1) = 1 max_nodes: pydantic.conint(ge=1) = 1 - @pydantic.validator('instance') + @pydantic.validator("instance") def _validate_instance(cls, value): digital_ocean.check_credentials() - available_instances = {_['slug'] for _ in digital_ocean.instances()} + available_instances = {_["slug"] for _ in digital_ocean.instances()} if value not in available_instances: - raise ValueError(f'Digital Ocean instance size={instance} not one of available instance types={available_instances}') + raise ValueError( + f"Digital Ocean instance size={instance} not one of available instance types={available_instances}" + ) return value @@ -229,13 +231,15 @@ class DigitalOceanProvider(schema.Base): } tags: typing.Optional[typing.List[str]] = [] - @pydantic.validator('region') + @pydantic.validator("region") def _validate_region(cls, value): digital_ocean.check_credentials() - available_regions = set(_['slug'] for _ in digital_ocean.regions()) + available_regions = set(_["slug"] for _ in digital_ocean.regions()) if value not in available_regions: - raise ValueError(f'Digital Ocean region={value} is not one of {available_regions}') + raise ValueError( + f"Digital Ocean region={value} is not one of {available_regions}" + ) return value @pydantic.root_validator diff --git a/src/_nebari/stages/terraform_state/__init__.py b/src/_nebari/stages/terraform_state/__init__.py index 8eba019a9..ed01f6eb5 100644 --- a/src/_nebari/stages/terraform_state/__init__.py +++ b/src/_nebari/stages/terraform_state/__init__.py @@ -1,10 +1,10 @@ +import contextlib import enum import inspect import os import pathlib import typing from typing import Any, Dict, List, Tuple -import contextlib from _nebari.stages.base import NebariTerraformStage from _nebari.utils import modified_environ @@ -175,10 +175,12 @@ def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): # DigitalOcean terraform remote state using Spaces Bucket # assumes aws credentials thus we set them to match spaces credentials if self.config.provider == schema.ProviderEnum.do: - env_mapping.update({ - "AWS_ACCESS_KEY_ID": os.environ["SPACES_ACCESS_KEY_ID"], - "AWS_SECRET_ACCESS_KEY": os.environ["SPACES_SECRET_ACCESS_KEY"], - }) + env_mapping.update( + { + "AWS_ACCESS_KEY_ID": os.environ["SPACES_ACCESS_KEY_ID"], + "AWS_SECRET_ACCESS_KEY": os.environ["SPACES_SECRET_ACCESS_KEY"], + } + ) with modified_environ(**env_mapping): yield @@ -192,10 +194,12 @@ def destroy( # DigitalOcean terraform remote state using Spaces Bucket # assumes aws credentials thus we set them to match spaces credentials if self.config.provider == schema.ProviderEnum.do: - env_mapping.update({ - "AWS_ACCESS_KEY_ID": os.environ["SPACES_ACCESS_KEY_ID"], - "AWS_SECRET_ACCESS_KEY": os.environ["SPACES_SECRET_ACCESS_KEY"], - }) + env_mapping.update( + { + "AWS_ACCESS_KEY_ID": os.environ["SPACES_ACCESS_KEY_ID"], + "AWS_SECRET_ACCESS_KEY": os.environ["SPACES_SECRET_ACCESS_KEY"], + } + ) with modified_environ(**env_mapping): yield diff --git a/src/_nebari/subcommands/validate.py b/src/_nebari/subcommands/validate.py index 5bd8be2c3..9cf7448f2 100644 --- a/src/_nebari/subcommands/validate.py +++ b/src/_nebari/subcommands/validate.py @@ -1,10 +1,9 @@ import pathlib +import pydantic import typer from rich import print -import pydantic -from _nebari.config import read_configuration from nebari.hookspecs import hookimpl @@ -34,8 +33,12 @@ def validate( try: nebari_plugin_manager.read_config(config_filename) - print("[bold purple]Successfully validated configuration.[/bold purple]") + print( + "[bold purple]Successfully validated configuration.[/bold purple]" + ) except pydantic.ValidationError as e: - print(f"[bold red]ERROR validating configuration {config_filename.absolute()}[/bold red]") + print( + f"[bold red]ERROR validating configuration {config_filename.absolute()}[/bold red]" + ) print(str(e)) typer.Abort() diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index 2de05f80e..5fd523ac9 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -7,7 +7,6 @@ from pathlib import Path import pluggy -import pydantic from nebari import hookspecs, schema @@ -112,6 +111,7 @@ def read_config(self, config_path: typing.Union[str, Path]): raise FileNotFoundError(f"Config file {config_path} not found") from _nebari.config import read_configuration + return read_configuration(config_path, self.config_schema) @property diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 8d085ac84..b3a5c169a 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -3,7 +3,7 @@ import pydantic from ruamel.yaml import yaml_object -from _nebari.utils import yaml, escape_string +from _nebari.utils import escape_string, yaml from _nebari.version import __version__, rounded_ver_parse # Regex for suitable project names @@ -70,15 +70,14 @@ def is_version_accepted(cls, v): @property def escaped_project_name(self): - """Escaped project-name know to be compatible with all clouds - """ + """Escaped project-name know to be compatible with all clouds""" project_name = self.project_name if self.provider == ProviderEnum.azure and "-" in project_name: project_name = escape_string(project_name, escape_char="a") if self.provider == ProviderEnum.aws and project_name.startswith("aws"): - project_name = "a" + project_name + project_name = "a" + project_name if len(project_name) > 16: project_name = project_name[:16] From fa398e44c1216999a801fb6d47ba278b9313b119 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 29 Jun 2023 23:00:27 -0400 Subject: [PATCH 100/147] Adding aws checks for region, zone, and instances --- .../provider/cloud/amazon_web_services.py | 8 ++++--- src/_nebari/stages/infrastructure/__init__.py | 23 +++++++++++++++++-- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/_nebari/provider/cloud/amazon_web_services.py b/src/_nebari/provider/cloud/amazon_web_services.py index a18a64ca2..c31b5a476 100644 --- a/src/_nebari/provider/cloud/amazon_web_services.py +++ b/src/_nebari/provider/cloud/amazon_web_services.py @@ -12,6 +12,7 @@ def check_credentials(): AWS_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-aws" for variable in { + "AWS_DEFAULT_REGION", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", }: @@ -30,7 +31,7 @@ def regions(): @functools.lru_cache() -def zones(region): +def zones(region: str = "us-west-2"): output = subprocess.check_output( ["aws", "ec2", "describe-availability-zones", "--region", region] ) @@ -39,11 +40,12 @@ def zones(region): @functools.lru_cache() -def kubernetes_versions(region="us-west-2"): +def kubernetes_versions(region = "us-west-2"): """Return list of available kubernetes supported by cloud provider. Sorted from oldest to latest.""" # AWS SDK (boto3) currently doesn't offer an intuitive way to list available kubernetes version. This implementation grabs kubernetes versions for specific EKS addons. It will therefore always be (at the very least) a subset of all kubernetes versions still supported by AWS. if not os.getenv("AWS_DEFAULT_REGION"): os.environ["AWS_DEFAULT_REGION"] = region + client = boto3.client("eks") supported_kubernetes_versions = list() available_addons = client.describe_addon_versions() @@ -59,7 +61,7 @@ def kubernetes_versions(region="us-west-2"): @functools.lru_cache() -def instances(region): +def instances(region: str = "us-west-2"): output = subprocess.check_output( ["aws", "ec2", "describe-instance-types", "--region", region] ) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 04f331514..de7f263fa 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -247,7 +247,7 @@ def _validate_kubernetes_version(cls, values): digital_ocean.check_credentials() if "region" not in values: - raise ValueError("region required in order to set kubernetes version") + raise ValueError("Region required in order to set kubernetes_version") available_kubernetes_versions = digital_ocean.kubernetes_versions( values["region"] @@ -391,6 +391,14 @@ class AWSNodeGroup(schema.Base): gpu: bool = False single_subnet: bool = False + @pydantic.validator('instance') + def _validate_instance(cls, value): + amazon_web_services.check_credentials() + + available_instances = amazon_web_services.instances() + if value not in available_instances: + raise ValueError(f"Instance {value} not available out of available instances {available_instances.keys()}") + return value class AmazonWebServicesProvider(schema.Base): region: str = pydantic.Field( @@ -424,8 +432,19 @@ def _validate_kubernetes_version(cls, values): ) return values + @pydantic.validator('region') + def _validate_region(cls, value): + amazon_web_services.check_credentials() + + available_regions = amazon_web_services.regions() + if value not in available_regions: + raise ValueError(f"Region {region} is not one of available regions {available_regions}") + return value + @pydantic.root_validator def _validate_availability_zones(cls, values): + amazon_web_services.check_credentials() + if values["availability_zones"] is None: zones = amazon_web_services.zones(values["region"]) values["availability_zones"] = list(sorted(zones))[:2] @@ -588,7 +607,7 @@ def tf_objects(self) -> List[Dict]: elif self.config.provider == schema.ProviderEnum.aws: return [ terraform.Provider( - "aws", region=nebari_config.amazon_web_services.region + "aws", region=self.config.amazon_web_services.region ), NebariTerraformState(self.name, self.config), ] From 237a78012e466513abb6dd57a9e931331086e0e8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Jun 2023 03:00:49 +0000 Subject: [PATCH 101/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/provider/cloud/amazon_web_services.py | 2 +- src/_nebari/stages/infrastructure/__init__.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/_nebari/provider/cloud/amazon_web_services.py b/src/_nebari/provider/cloud/amazon_web_services.py index c31b5a476..937b55b76 100644 --- a/src/_nebari/provider/cloud/amazon_web_services.py +++ b/src/_nebari/provider/cloud/amazon_web_services.py @@ -40,7 +40,7 @@ def zones(region: str = "us-west-2"): @functools.lru_cache() -def kubernetes_versions(region = "us-west-2"): +def kubernetes_versions(region="us-west-2"): """Return list of available kubernetes supported by cloud provider. Sorted from oldest to latest.""" # AWS SDK (boto3) currently doesn't offer an intuitive way to list available kubernetes version. This implementation grabs kubernetes versions for specific EKS addons. It will therefore always be (at the very least) a subset of all kubernetes versions still supported by AWS. if not os.getenv("AWS_DEFAULT_REGION"): diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index de7f263fa..33e6fead2 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -391,15 +391,18 @@ class AWSNodeGroup(schema.Base): gpu: bool = False single_subnet: bool = False - @pydantic.validator('instance') + @pydantic.validator("instance") def _validate_instance(cls, value): amazon_web_services.check_credentials() available_instances = amazon_web_services.instances() if value not in available_instances: - raise ValueError(f"Instance {value} not available out of available instances {available_instances.keys()}") + raise ValueError( + f"Instance {value} not available out of available instances {available_instances.keys()}" + ) return value + class AmazonWebServicesProvider(schema.Base): region: str = pydantic.Field( default_factory=lambda: os.environ.get("AWS_DEFAULT_REGION", "us-west-2") @@ -432,13 +435,15 @@ def _validate_kubernetes_version(cls, values): ) return values - @pydantic.validator('region') + @pydantic.validator("region") def _validate_region(cls, value): amazon_web_services.check_credentials() available_regions = amazon_web_services.regions() if value not in available_regions: - raise ValueError(f"Region {region} is not one of available regions {available_regions}") + raise ValueError( + f"Region {region} is not one of available regions {available_regions}" + ) return value @pydantic.root_validator From 33bcdfca6e9b6dde0dc1c4b354be1dba1620e0ed Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 29 Jun 2023 23:30:59 -0400 Subject: [PATCH 102/147] Typo and apply validation at different level --- src/_nebari/stages/infrastructure/__init__.py | 48 ++++++++++--------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 33e6fead2..0e10b0805 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -201,18 +201,6 @@ class DigitalOceanNodeGroup(schema.Base): min_nodes: pydantic.conint(ge=1) = 1 max_nodes: pydantic.conint(ge=1) = 1 - @pydantic.validator("instance") - def _validate_instance(cls, value): - digital_ocean.check_credentials() - - available_instances = {_["slug"] for _ in digital_ocean.instances()} - if value not in available_instances: - raise ValueError( - f"Digital Ocean instance size={instance} not one of available instance types={available_instances}" - ) - - return value - class DigitalOceanProvider(schema.Base): region: str = "nyc3" @@ -242,6 +230,19 @@ def _validate_region(cls, value): ) return value + @pydantic.validator("node_groups") + def _validate_node_group(cls, value): + digital_ocean.check_credentials() + + available_instances = {_["slug"] for _ in digital_ocean.instances()} + for name, node_group in value.items(): + if node_group.instance not in available_instances: + raise ValueError( + f"Digital Ocean instance {node_group.instance} not one of available instance types={available_instances}" + ) + + return value + @pydantic.root_validator def _validate_kubernetes_version(cls, values): digital_ocean.check_credentials() @@ -391,17 +392,6 @@ class AWSNodeGroup(schema.Base): gpu: bool = False single_subnet: bool = False - @pydantic.validator("instance") - def _validate_instance(cls, value): - amazon_web_services.check_credentials() - - available_instances = amazon_web_services.instances() - if value not in available_instances: - raise ValueError( - f"Instance {value} not available out of available instances {available_instances.keys()}" - ) - return value - class AmazonWebServicesProvider(schema.Base): region: str = pydantic.Field( @@ -435,6 +425,18 @@ def _validate_kubernetes_version(cls, values): ) return values + @pydantic.validator("node_groups") + def _validate_node_group(cls, value): + amazon_web_services.check_credentials() + + available_instances = amazon_web_services.instances() + for name, node_group in value.items(): + if node_group.instance not in available_instances: + raise ValueError( + f"Instance {node_group.instance} not available out of available instances {available_instances.keys()}" + ) + return value + @pydantic.validator("region") def _validate_region(cls, value): amazon_web_services.check_credentials() From da12f6e7934c50a6f22228d93a3f18b23eb8da79 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 29 Jun 2023 23:33:33 -0400 Subject: [PATCH 103/147] load_config -> read_config for naming --- src/_nebari/subcommands/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index 32551c9f4..b0aa3c6e7 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -434,7 +434,7 @@ def init( handle_init(inputs, config_schema=nebari_plugin_manager.config_schema) - nebari_plugin_manager.load_config(output) + nebari_plugin_manager.read_config(output) def guided_init_wizard(ctx: typer.Context, guided_init: str): From 00ae443c892f70173758f43e8c9de5c92f315e69 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 29 Jun 2023 23:44:27 -0400 Subject: [PATCH 104/147] Adding digital ocean spaces access keys --- .github/workflows/test-provider.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-provider.yaml b/.github/workflows/test-provider.yaml index 60649f79e..3c0a3fa89 100644 --- a/.github/workflows/test-provider.yaml +++ b/.github/workflows/test-provider.yaml @@ -85,6 +85,8 @@ jobs: kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci tenant_id | ARM_TENANT_ID; kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci subscription_id | ARM_SUBSCRIPTION_ID; kv/data/repository/nebari-dev/nebari/shared_secrets DIGITALOCEAN_TOKEN | DIGITALOCEAN_TOKEN; + kv/data/repository/nebari-dev/nebari/shared_secrets SPACES_ACCESS_KEY_ID | SPACES_ACCESS_KEY_ID; + kv/data/repository/nebari-dev/nebari/shared_secrets SPACES_SECRET_ACCESS_KEY | SPACES_SECRET_ACCESS_KEY; - name: 'Authenticate to GCP' if: ${{ matrix.provider == 'gcp' }} From da2ce8be361d71aad1af3ad43ada108598ea35ab Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Thu, 29 Jun 2023 23:54:11 -0400 Subject: [PATCH 105/147] Wrong reference to config --- src/_nebari/stages/infrastructure/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 0e10b0805..8deae4722 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -598,8 +598,8 @@ def tf_objects(self) -> List[Dict]: return [ terraform.Provider( "google", - project=nebari_config.google_cloud_platform.project, - region=nebari_config.google_cloud_platform.region, + project=self.config.google_cloud_platform.project, + region=self.config.google_cloud_platform.region, ), NebariTerraformState(self.name, self.config), ] From 542aaebea5638b9600bd01c7107549168eeeafa2 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 30 Jun 2023 00:05:17 -0400 Subject: [PATCH 106/147] Fixing conftest --- tests/tests_unit/test_render.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests_unit/test_render.py b/tests/tests_unit/test_render.py index edf48b305..02aa21e63 100644 --- a/tests/tests_unit/test_render.py +++ b/tests/tests_unit/test_render.py @@ -39,9 +39,9 @@ def test_render_config(nebari_render): assert (output_directory / "stages" / "01-terraform-state/azure").is_dir() assert (output_directory / "stages" / "02-infrastructure/azure").is_dir() - if config.ci_cd.type == schema.CiEnum.github_actions: + if config.ci_cd.type == CiEnum.github_actions: assert (output_directory / ".github/workflows/").is_dir() - elif config.ci_cd.type == schema.CiEnum.gitlab_ci: + elif config.ci_cd.type == CiEnum.gitlab_ci: assert (output_directory / ".gitlab-ci.yml").is_file() From 1febe5fe19ed0717bee16c766a85997ca47956e5 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 30 Jun 2023 00:18:06 -0400 Subject: [PATCH 107/147] Making docs links in constants module --- src/_nebari/constants.py | 7 +++++++ src/_nebari/provider/cloud/amazon_web_services.py | 5 ++--- src/_nebari/provider/cloud/azure_cloud.py | 5 ++--- src/_nebari/provider/cloud/digital_ocean.py | 5 ++--- src/_nebari/provider/cloud/google_cloud.py | 3 +-- tests/tests_unit/test_links.py | 2 +- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py index aedaeee68..7380c6a8f 100644 --- a/src/_nebari/constants.py +++ b/src/_nebari/constants.py @@ -15,3 +15,10 @@ DEFAULT_CONDA_STORE_IMAGE_TAG = "v0.4.14" LATEST_SUPPORTED_PYTHON_VERSION = "3.10" + + +# DOCS +DO_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-do" +AZURE_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-azure" +AWS_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-aws" +GCP_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-gcp" diff --git a/src/_nebari/provider/cloud/amazon_web_services.py b/src/_nebari/provider/cloud/amazon_web_services.py index 937b55b76..8f358e1a5 100644 --- a/src/_nebari/provider/cloud/amazon_web_services.py +++ b/src/_nebari/provider/cloud/amazon_web_services.py @@ -6,11 +6,10 @@ import boto3 from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version +from _nebari import constants def check_credentials(): - AWS_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-aws" - for variable in { "AWS_DEFAULT_REGION", "AWS_ACCESS_KEY_ID", @@ -19,7 +18,7 @@ def check_credentials(): if variable not in os.environ: raise ValueError( f"""Missing the following required environment variable: {variable}\n - Please see the documentation for more information: {AWS_ENV_DOCS}""" + Please see the documentation for more information: {constants.AWS_ENV_DOCS}""" ) diff --git a/src/_nebari/provider/cloud/azure_cloud.py b/src/_nebari/provider/cloud/azure_cloud.py index f344dadcc..fd1e91e8c 100644 --- a/src/_nebari/provider/cloud/azure_cloud.py +++ b/src/_nebari/provider/cloud/azure_cloud.py @@ -6,14 +6,13 @@ from azure.mgmt.containerservice import ContainerServiceClient from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version +from _nebari import constants logger = logging.getLogger("azure") logger.setLevel(logging.ERROR) def check_credentials(): - AZURE_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-azure" - for variable in { "ARM_CLIENT_ID", "ARM_CLIENT_SECRET", @@ -23,7 +22,7 @@ def check_credentials(): if variable not in os.environ: raise ValueError( f"""Missing the following required environment variable: {variable}\n - Please see the documentation for more information: {AZURE_ENV_DOCS}""" + Please see the documentation for more information: {constants.AZURE_ENV_DOCS}""" ) diff --git a/src/_nebari/provider/cloud/digital_ocean.py b/src/_nebari/provider/cloud/digital_ocean.py index e000ab764..cff34239f 100644 --- a/src/_nebari/provider/cloud/digital_ocean.py +++ b/src/_nebari/provider/cloud/digital_ocean.py @@ -4,11 +4,10 @@ import requests from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version +from _nebari import constants def check_credentials(): - DO_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-do" - for variable in { "SPACES_ACCESS_KEY_ID", "SPACES_SECRET_ACCESS_KEY", @@ -17,7 +16,7 @@ def check_credentials(): if variable not in os.environ: raise ValueError( f"""Missing the following required environment variable: {variable}\n - Please see the documentation for more information: {DO_ENV_DOCS}""" + Please see the documentation for more information: {constants.DO_ENV_DOCS}""" ) diff --git a/src/_nebari/provider/cloud/google_cloud.py b/src/_nebari/provider/cloud/google_cloud.py index 3ad2ce50c..c83e37a06 100644 --- a/src/_nebari/provider/cloud/google_cloud.py +++ b/src/_nebari/provider/cloud/google_cloud.py @@ -7,12 +7,11 @@ def check_credentials(): - GCP_ENV_DOCS = "https://www.nebari.dev/docs/how-tos/nebari-gcp" for variable in {"GOOGLE_CREDENTIALS"}: if variable not in os.environ: raise ValueError( f"""Missing the following required environment variable: {variable}\n - Please see the documentation for more information: {GCP_ENV_DOCS}""" + Please see the documentation for more information: {constants.GCP_ENV_DOCS}""" ) diff --git a/tests/tests_unit/test_links.py b/tests/tests_unit/test_links.py index 8df7ee1e6..a393391ce 100644 --- a/tests/tests_unit/test_links.py +++ b/tests/tests_unit/test_links.py @@ -1,7 +1,7 @@ import pytest import requests -from _nebari.utils import AWS_ENV_DOCS, AZURE_ENV_DOCS, DO_ENV_DOCS, GCP_ENV_DOCS +from _nebari.constants import AWS_ENV_DOCS, AZURE_ENV_DOCS, DO_ENV_DOCS, GCP_ENV_DOCS LINKS_TO_TEST = [ DO_ENV_DOCS, From 1eb5c05db69ccab4206a6931ab2d77e8f81ab17a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Jun 2023 04:18:33 +0000 Subject: [PATCH 108/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/provider/cloud/amazon_web_services.py | 2 +- src/_nebari/provider/cloud/azure_cloud.py | 2 +- src/_nebari/provider/cloud/digital_ocean.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/_nebari/provider/cloud/amazon_web_services.py b/src/_nebari/provider/cloud/amazon_web_services.py index 8f358e1a5..943f9d3e8 100644 --- a/src/_nebari/provider/cloud/amazon_web_services.py +++ b/src/_nebari/provider/cloud/amazon_web_services.py @@ -5,8 +5,8 @@ import boto3 -from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version from _nebari import constants +from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version def check_credentials(): diff --git a/src/_nebari/provider/cloud/azure_cloud.py b/src/_nebari/provider/cloud/azure_cloud.py index fd1e91e8c..170a301b8 100644 --- a/src/_nebari/provider/cloud/azure_cloud.py +++ b/src/_nebari/provider/cloud/azure_cloud.py @@ -5,8 +5,8 @@ from azure.identity import DefaultAzureCredential from azure.mgmt.containerservice import ContainerServiceClient -from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version from _nebari import constants +from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version logger = logging.getLogger("azure") logger.setLevel(logging.ERROR) diff --git a/src/_nebari/provider/cloud/digital_ocean.py b/src/_nebari/provider/cloud/digital_ocean.py index cff34239f..519491f25 100644 --- a/src/_nebari/provider/cloud/digital_ocean.py +++ b/src/_nebari/provider/cloud/digital_ocean.py @@ -3,8 +3,8 @@ import requests -from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version from _nebari import constants +from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version def check_credentials(): From 77fc3278775fb572e389dbdc644f41d1181fd9e5 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 30 Jun 2023 00:42:03 -0400 Subject: [PATCH 109/147] Fixing tests --- src/nebari/plugins.py | 4 ++-- tests/tests_unit/test_schema.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nebari/plugins.py b/src/nebari/plugins.py index 5fd523ac9..ca593347e 100644 --- a/src/nebari/plugins.py +++ b/src/nebari/plugins.py @@ -103,7 +103,7 @@ def get_available_stages(self): return included_stages - def read_config(self, config_path: typing.Union[str, Path]): + def read_config(self, config_path: typing.Union[str, Path], **kwargs): if isinstance(config_path, str): config_path = Path(config_path) @@ -112,7 +112,7 @@ def read_config(self, config_path: typing.Union[str, Path]): from _nebari.config import read_configuration - return read_configuration(config_path, self.config_schema) + return read_configuration(config_path, self.config_schema, **kwargs) @property def ordered_stages(self): diff --git a/tests/tests_unit/test_schema.py b/tests/tests_unit/test_schema.py index 44d0b6d2f..3e1c5cad0 100644 --- a/tests/tests_unit/test_schema.py +++ b/tests/tests_unit/test_schema.py @@ -14,7 +14,7 @@ def test_minimal_schema_from_file(tmp_path): with filename.open("w") as f: f.write("project_name: test\n") - config = schema.read_configuration(filename) + config = nebari_plugin_manager.read_config(filename) assert config.project_name == "test" assert config.storage.conda_store == "200Gi" @@ -27,7 +27,7 @@ def test_minimal_schema_from_file_with_env(tmp_path, monkeypatch): monkeypatch.setenv("NEBARI_SECRET__project_name", "env") monkeypatch.setenv("NEBARI_SECRET__storage__conda_store", "1000Gi") - config = schema.read_configuration(filename) + config = nebari_plugin_manager.read_config(filename) assert config.project_name == "env" assert config.storage.conda_store == "1000Gi" @@ -40,7 +40,7 @@ def test_minimal_schema_from_file_without_env(tmp_path, monkeypatch): monkeypatch.setenv("NEBARI_SECRET__project_name", "env") monkeypatch.setenv("NEBARI_SECRET__storage__conda_store", "1000Gi") - config = schema.read_configuration(filename, read_environment=False) + config = nebari_plugin_manager.read_config(filename, read_environment=False) assert config.project_name == "test" assert config.storage.conda_store == "200Gi" From 9a1191be6c51d9186b56f08f625b8697d1277373 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 30 Jun 2023 00:44:24 -0400 Subject: [PATCH 110/147] Missing import --- src/_nebari/keycloak.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/_nebari/keycloak.py b/src/_nebari/keycloak.py index 77b0ae46e..8839688fd 100644 --- a/src/_nebari/keycloak.py +++ b/src/_nebari/keycloak.py @@ -8,6 +8,7 @@ import rich from nebari import schema +from _nebari.stages.kubernetes_keycloak import CertificateEnum logger = logging.getLogger(__name__) @@ -90,7 +91,7 @@ def get_keycloak_admin_from_config(config: schema.Main): "KEYCLOAK_ADMIN_PASSWORD", config.security.keycloak.initial_root_password ) - should_verify_tls = config.certificate.type != schema.CertificateEnum.selfsigned + should_verify_tls = config.certificate.type != CertificateEnum.selfsigned try: keycloak_admin = keycloak.KeycloakAdmin( From 1db152394974f94460a6b20ee22a7fd511323448 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Jun 2023 04:44:45 +0000 Subject: [PATCH 111/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/keycloak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/keycloak.py b/src/_nebari/keycloak.py index 8839688fd..54fe6cabd 100644 --- a/src/_nebari/keycloak.py +++ b/src/_nebari/keycloak.py @@ -7,8 +7,8 @@ import requests import rich -from nebari import schema from _nebari.stages.kubernetes_keycloak import CertificateEnum +from nebari import schema logger = logging.getLogger(__name__) From 394e608eb2ff3a1a10eae2974f438ab3812a2547 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 30 Jun 2023 00:50:41 -0400 Subject: [PATCH 112/147] Wrong import module --- src/_nebari/keycloak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/keycloak.py b/src/_nebari/keycloak.py index 54fe6cabd..18a896224 100644 --- a/src/_nebari/keycloak.py +++ b/src/_nebari/keycloak.py @@ -7,8 +7,8 @@ import requests import rich -from _nebari.stages.kubernetes_keycloak import CertificateEnum from nebari import schema +from _nebari.stages.kubernetes_ingress import CertificateEnum logger = logging.getLogger(__name__) From ae0b5fe40797e137d1ab031fe3d59631c26641db Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Jun 2023 04:51:22 +0000 Subject: [PATCH 113/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/keycloak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/keycloak.py b/src/_nebari/keycloak.py index 18a896224..674b7c8ca 100644 --- a/src/_nebari/keycloak.py +++ b/src/_nebari/keycloak.py @@ -7,8 +7,8 @@ import requests import rich -from nebari import schema from _nebari.stages.kubernetes_ingress import CertificateEnum +from nebari import schema logger = logging.getLogger(__name__) From 1e47558d6c0e575ac5aaa5beee1b2bbf220d5c36 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 30 Jun 2023 01:36:48 -0400 Subject: [PATCH 114/147] Fixing tests --- src/_nebari/provider/cloud/google_cloud.py | 1 + src/_nebari/upgrade.py | 3 ++- tests/tests_unit/test_cli.py | 7 ++++--- tests/tests_unit/test_render.py | 2 +- tests/tests_unit/test_schema.py | 2 +- tests/tests_unit/test_upgrade.py | 4 ++-- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/_nebari/provider/cloud/google_cloud.py b/src/_nebari/provider/cloud/google_cloud.py index c83e37a06..810011ff5 100644 --- a/src/_nebari/provider/cloud/google_cloud.py +++ b/src/_nebari/provider/cloud/google_cloud.py @@ -3,6 +3,7 @@ import os import subprocess +from _nebari import constants from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 81ac57bde..dabe1691e 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -32,7 +32,8 @@ def do_upgrade(config_filename, attempt_fixes=False): return try: - schema.read_configuration(config_filename) + from nebari.plugins import nebari_plugin_manager + nebari_plugin_manager.read_config(config_filename) rich.print( f"Your config file [purple]{config_filename}[/purple] appears to be already up-to-date for Nebari version [green]{__version__}[/green]" ) diff --git a/tests/tests_unit/test_cli.py b/tests/tests_unit/test_cli.py index 0de08b706..e6c423b72 100644 --- a/tests/tests_unit/test_cli.py +++ b/tests/tests_unit/test_cli.py @@ -2,7 +2,8 @@ import pytest -from nebari import schema +from nebari.plugins import nebari_plugin_manager +from _nebari.subcommands.init import InitInputs PROJECT_NAME = "clitest" DOMAIN_NAME = "clitest.dev" @@ -26,7 +27,7 @@ def test_nebari_init(tmp_path, namespace, auth_provider, ci_provider, ssl_cert_e "--disable-prompt", ] - default_values = schema.InitInputs() + default_values = InitInputs() if namespace: command.append(f"--namespace={namespace}") @@ -47,7 +48,7 @@ def test_nebari_init(tmp_path, namespace, auth_provider, ci_provider, ssl_cert_e subprocess.run(command, cwd=tmp_path, check=True) - config = schema.read_configuration(tmp_path / "nebari-config.yaml") + config = nebari_plugin_manager.read_config(tmp_path / "nebari-config.yaml") assert config.namespace == namespace assert config.security.authentication.type.lower() == auth_provider diff --git a/tests/tests_unit/test_render.py b/tests/tests_unit/test_render.py index 02aa21e63..9e56fede2 100644 --- a/tests/tests_unit/test_render.py +++ b/tests/tests_unit/test_render.py @@ -11,7 +11,7 @@ def test_render_config(nebari_render): output_directory, config_filename = nebari_render - config = schema.read_configuration(config_filename) + config = nebari_plugin_manager.read_config(config_filename) assert {"nebari-config.yaml", "stages", ".gitignore"} <= set( os.listdir(output_directory) ) diff --git a/tests/tests_unit/test_schema.py b/tests/tests_unit/test_schema.py index 3e1c5cad0..dea5b96e5 100644 --- a/tests/tests_unit/test_schema.py +++ b/tests/tests_unit/test_schema.py @@ -4,7 +4,7 @@ def test_minimal_schema(): - config = schema.Main(project_name="test") + config = nebari_plugin_manager.config_schema(project_name="test") assert config.project_name == "test" assert config.storage.conda_store == "200Gi" diff --git a/tests/tests_unit/test_upgrade.py b/tests/tests_unit/test_upgrade.py index 8c86e89c6..0946dcd99 100644 --- a/tests/tests_unit/test_upgrade.py +++ b/tests/tests_unit/test_upgrade.py @@ -4,7 +4,7 @@ from _nebari.upgrade import do_upgrade from _nebari.version import __version__, rounded_ver_parse -from nebari import schema +from nebari.plugins import nebari_plugin_manager @pytest.fixture @@ -70,7 +70,7 @@ def test_upgrade_4_0( return # Check the resulting YAML - config = schema.read_configuration(tmp_qhub_config) + config = nebari_plugin_manager.read_config(tmp_qhub_config) assert len(config.security.keycloak.initial_root_password) == 16 assert not hasattr(config.security, "users") From f2d69720ae145a1a15e901380b49bf3e4eff2f20 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Jun 2023 05:37:02 +0000 Subject: [PATCH 115/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/upgrade.py | 1 + tests/tests_unit/test_cli.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index dabe1691e..6cb5b098a 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -33,6 +33,7 @@ def do_upgrade(config_filename, attempt_fixes=False): try: from nebari.plugins import nebari_plugin_manager + nebari_plugin_manager.read_config(config_filename) rich.print( f"Your config file [purple]{config_filename}[/purple] appears to be already up-to-date for Nebari version [green]{__version__}[/green]" diff --git a/tests/tests_unit/test_cli.py b/tests/tests_unit/test_cli.py index e6c423b72..d8a4e423b 100644 --- a/tests/tests_unit/test_cli.py +++ b/tests/tests_unit/test_cli.py @@ -2,8 +2,8 @@ import pytest -from nebari.plugins import nebari_plugin_manager from _nebari.subcommands.init import InitInputs +from nebari.plugins import nebari_plugin_manager PROJECT_NAME = "clitest" DOMAIN_NAME = "clitest.dev" From 949461482178e22fe0d3ea56cd134aabda30e7b7 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 30 Jun 2023 10:59:04 -0400 Subject: [PATCH 116/147] Mocking more methods in tests --- src/_nebari/provider/cloud/digital_ocean.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/_nebari/provider/cloud/digital_ocean.py b/src/_nebari/provider/cloud/digital_ocean.py index 519491f25..309714228 100644 --- a/src/_nebari/provider/cloud/digital_ocean.py +++ b/src/_nebari/provider/cloud/digital_ocean.py @@ -1,3 +1,4 @@ +import typing import functools import os @@ -55,7 +56,7 @@ def regions(): return _kubernetes_options()["options"]["regions"] -def kubernetes_versions(region): +def kubernetes_versions(region) -> typing.List[str]: """Return list of available kubernetes supported by cloud provider. Sorted from oldest to latest.""" supported_kubernetes_versions = sorted( [_["slug"].split("-")[0] for _ in _kubernetes_options()["options"]["versions"]] From 3f153362738b3f0909d1f1bfbf2ec0e8bc93ccda Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 30 Jun 2023 11:24:30 -0400 Subject: [PATCH 117/147] Adding more concise data --- src/_nebari/stages/infrastructure/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 8deae4722..38312b9ab 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -444,7 +444,7 @@ def _validate_region(cls, value): available_regions = amazon_web_services.regions() if value not in available_regions: raise ValueError( - f"Region {region} is not one of available regions {available_regions}" + f"Region {value} is not one of available regions {available_regions}" ) return value From 026172db3cc5509d18757f74f9e75ac66bfaa786 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Jun 2023 15:25:18 +0000 Subject: [PATCH 118/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/provider/cloud/digital_ocean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/provider/cloud/digital_ocean.py b/src/_nebari/provider/cloud/digital_ocean.py index 309714228..7998bb1af 100644 --- a/src/_nebari/provider/cloud/digital_ocean.py +++ b/src/_nebari/provider/cloud/digital_ocean.py @@ -1,6 +1,6 @@ -import typing import functools import os +import typing import requests From 963cf85b43bd5752b2591d73d6dd344351386c90 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 30 Jun 2023 12:05:23 -0400 Subject: [PATCH 119/147] Properly escaping strings --- src/_nebari/utils.py | 71 ++++++++++++++++++++++++++++++--- tests/tests_deployment/utils.py | 8 ---- 2 files changed, 66 insertions(+), 13 deletions(-) diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 0781f951c..ee37c3adb 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -9,9 +9,9 @@ import sys import threading import time +import warnings from typing import Dict, List -import escapism from ruamel.yaml import YAML # environment variable overrides @@ -182,10 +182,71 @@ def deep_merge(*args): return d1 -def escape_string( - s: str, safe_chars=string.ascii_letters + string.digits, escape_char="-" -): - return escapism.escape(s, safe=safe_chars, escape_char=escape_char) + + +# https://github.com/minrk/escapism/blob/master/escapism.py +def escape_string(to_escape, safe=set(string.ascii_letters + string.digits), escape_char='_', allow_collisions=False): + """Escape a string so that it only contains characters in a safe set. + + Characters outside the safe list will be escaped with _%x_, + where %x is the hex value of the character. + + If `allow_collisions` is True, occurrences of `escape_char` + in the input will not be escaped. + + In this case, `unescape` cannot be used to reverse the transform + because occurrences of the escape char in the resulting string are ambiguous. + Only use this mode when: + + 1. collisions cannot occur or do not matter, and + 2. unescape will never be called. + + .. versionadded: 1.0 + allow_collisions argument. + Prior to 1.0, behavior was the same as allow_collisions=False (default). + + """ + if sys.version_info >= (3,): + _ord = lambda byte: byte + _bchr = lambda n: bytes([n]) + else: + _ord = ord + _bchr = chr + + def _escape_char(c, escape_char=escape_char): + """Escape a single character""" + buf = [] + for byte in c.encode('utf8'): + buf.append(escape_char) + buf.append('%X' % _ord(byte)) + return ''.join(buf) + + if isinstance(to_escape, bytes): + # always work on text + to_escape = to_escape.decode('utf8') + + if not isinstance(safe, set): + safe = set(safe) + + if allow_collisions: + safe.add(escape_char) + elif escape_char in safe: + warnings.warn( + "Escape character %r cannot be a safe character." + " Set allow_collisions=True if you want to allow ambiguous escaped strings." + % escape_char, + RuntimeWarning, + stacklevel=2, + ) + safe.remove(escape_char) + + chars = [] + for c in to_escape: + if c in safe: + chars.append(c) + else: + chars.append(_escape_char(c, escape_char)) + return u''.join(chars) def random_secure_string( diff --git a/tests/tests_deployment/utils.py b/tests/tests_deployment/utils.py index 5be0f948d..f53b9c02c 100644 --- a/tests/tests_deployment/utils.py +++ b/tests/tests_deployment/utils.py @@ -61,11 +61,3 @@ def _inner(*args, **kwargs): sslcontext = ssl.create_default_context() ssl.create_default_context = create_default_context(sslcontext) - - -def escape_string(s): - # https://github.com/jupyterhub/kubespawner/blob/main/kubespawner/spawner.py#L1681 - # Make sure username and servername match the restrictions for DNS labels - # Note: '-' is not in safe_chars, as it is being used as escape character - safe_chars = set(string.ascii_lowercase + string.digits) - return escapism.escape(s, safe=safe_chars, escape_char="-").lower() From 6fdaeaf4db9f9d417fbd860f8e6c6505169aaa3c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Jun 2023 16:06:58 +0000 Subject: [PATCH 120/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/utils.py | 25 +++++++++++++++---------- tests/tests_deployment/utils.py | 2 -- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index ee37c3adb..6167ab80a 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -182,10 +182,13 @@ def deep_merge(*args): return d1 - - # https://github.com/minrk/escapism/blob/master/escapism.py -def escape_string(to_escape, safe=set(string.ascii_letters + string.digits), escape_char='_', allow_collisions=False): +def escape_string( + to_escape, + safe=set(string.ascii_letters + string.digits), + escape_char="_", + allow_collisions=False, +): """Escape a string so that it only contains characters in a safe set. Characters outside the safe list will be escaped with _%x_, @@ -207,8 +210,10 @@ def escape_string(to_escape, safe=set(string.ascii_letters + string.digits), esc """ if sys.version_info >= (3,): - _ord = lambda byte: byte - _bchr = lambda n: bytes([n]) + def _ord(byte): + return byte + def _bchr(n): + return bytes([n]) else: _ord = ord _bchr = chr @@ -216,14 +221,14 @@ def escape_string(to_escape, safe=set(string.ascii_letters + string.digits), esc def _escape_char(c, escape_char=escape_char): """Escape a single character""" buf = [] - for byte in c.encode('utf8'): + for byte in c.encode("utf8"): buf.append(escape_char) - buf.append('%X' % _ord(byte)) - return ''.join(buf) + buf.append("%X" % _ord(byte)) + return "".join(buf) if isinstance(to_escape, bytes): # always work on text - to_escape = to_escape.decode('utf8') + to_escape = to_escape.decode("utf8") if not isinstance(safe, set): safe = set(safe) @@ -246,7 +251,7 @@ def _escape_char(c, escape_char=escape_char): chars.append(c) else: chars.append(_escape_char(c, escape_char)) - return u''.join(chars) + return "".join(chars) def random_secure_string( diff --git a/tests/tests_deployment/utils.py b/tests/tests_deployment/utils.py index f53b9c02c..327de5330 100644 --- a/tests/tests_deployment/utils.py +++ b/tests/tests_deployment/utils.py @@ -1,8 +1,6 @@ import re import ssl -import string -import escapism import requests from tests.tests_deployment import constants From 5e9f0630f222bcc2a950fed78a618e7844fb99bf Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 30 Jun 2023 12:43:24 -0400 Subject: [PATCH 121/147] Trigger CI again --- src/_nebari/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 6167ab80a..612354a39 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -251,6 +251,7 @@ def _escape_char(c, escape_char=escape_char): chars.append(c) else: chars.append(_escape_char(c, escape_char)) + return "".join(chars) From 01003459eac1af44785279b63f023d9cb5f870a2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Jun 2023 16:43:39 +0000 Subject: [PATCH 122/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 612354a39..65205f89d 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -210,10 +210,13 @@ def escape_string( """ if sys.version_info >= (3,): + def _ord(byte): return byte + def _bchr(n): return bytes([n]) + else: _ord = ord _bchr = chr From 5592821df15006bae1fa52598011538440810d3c Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 30 Jun 2023 16:51:27 -0400 Subject: [PATCH 123/147] Ensure that username matches before --- src/_nebari/utils.py | 2 +- tests/tests_deployment/test_jupyterhub_ssh.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 65205f89d..144425ae9 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -221,7 +221,7 @@ def _bchr(n): _ord = ord _bchr = chr - def _escape_char(c, escape_char=escape_char): + def _escape_char(c, escape_char): """Escape a single character""" buf = [] for byte in c.encode("utf8"): diff --git a/tests/tests_deployment/test_jupyterhub_ssh.py b/tests/tests_deployment/test_jupyterhub_ssh.py index 8ecfa4f40..07be3ea9b 100644 --- a/tests/tests_deployment/test_jupyterhub_ssh.py +++ b/tests/tests_deployment/test_jupyterhub_ssh.py @@ -3,6 +3,7 @@ import paramiko import pytest +import string from tests.tests_deployment import constants from tests.tests_deployment.utils import ( @@ -101,7 +102,7 @@ def test_exact_jupyterhub_ssh(paramiko_object): ("pwd", f"/home/{constants.KEYCLOAK_USERNAME}"), ("echo $HOME", f"/home/{constants.KEYCLOAK_USERNAME}"), ("conda activate default && echo $CONDA_PREFIX", "/opt/conda/envs/default"), - ("hostname", f"jupyter-{escape_string(constants.KEYCLOAK_USERNAME)}"), + ("hostname", f"jupyter-{escape_string(constants.KEYCLOAK_USERNAME, safe=set(string.ascii_lowercase + string.digits), escape_char='-').lower()}"), ] for command, output in commands_exact: From a43dadf2cb0848f6a2c2e34cbb509969fc35d404 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Jun 2023 20:51:50 +0000 Subject: [PATCH 124/147] [pre-commit.ci] Apply automatic pre-commit fixes --- tests/tests_deployment/test_jupyterhub_ssh.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/tests_deployment/test_jupyterhub_ssh.py b/tests/tests_deployment/test_jupyterhub_ssh.py index 07be3ea9b..f03079fb3 100644 --- a/tests/tests_deployment/test_jupyterhub_ssh.py +++ b/tests/tests_deployment/test_jupyterhub_ssh.py @@ -1,9 +1,9 @@ import re +import string import uuid import paramiko import pytest -import string from tests.tests_deployment import constants from tests.tests_deployment.utils import ( @@ -102,7 +102,10 @@ def test_exact_jupyterhub_ssh(paramiko_object): ("pwd", f"/home/{constants.KEYCLOAK_USERNAME}"), ("echo $HOME", f"/home/{constants.KEYCLOAK_USERNAME}"), ("conda activate default && echo $CONDA_PREFIX", "/opt/conda/envs/default"), - ("hostname", f"jupyter-{escape_string(constants.KEYCLOAK_USERNAME, safe=set(string.ascii_lowercase + string.digits), escape_char='-').lower()}"), + ( + "hostname", + f"jupyter-{escape_string(constants.KEYCLOAK_USERNAME, safe=set(string.ascii_lowercase + string.digits), escape_char='-').lower()}", + ), ] for command, output in commands_exact: From fe55fa9d2843a692f41c8582b297dd8f26793bfa Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 2 Aug 2023 16:56:57 -0400 Subject: [PATCH 125/147] Missing import after rebase --- src/_nebari/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 144425ae9..a495452ef 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -10,6 +10,7 @@ import threading import time import warnings +import pathlib from typing import Dict, List from ruamel.yaml import YAML From 2670a2083da8981437f5b76031e44f8781b42603 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 2 Aug 2023 20:57:18 +0000 Subject: [PATCH 126/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index a495452ef..9a5fc2599 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -1,6 +1,7 @@ import contextlib import functools import os +import pathlib import re import secrets import signal @@ -10,7 +11,6 @@ import threading import time import warnings -import pathlib from typing import Dict, List from ruamel.yaml import YAML From b48eef61302c9c190e12a671e2a29888a558d49d Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 2 Aug 2023 17:02:08 -0400 Subject: [PATCH 127/147] Remove os.path.join error in ruff --- pyproject.toml | 1 + src/_nebari/render.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index eebff1089..192c3c715 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,6 +108,7 @@ select = [ ignore = [ "E501", # Line too long "F821", # Undefined name + "PTH118", # os.path.join() should be replaced by Path() "PTH123", # open() should be replaced by Path.open() ] extend-exclude = [ diff --git a/src/_nebari/render.py b/src/_nebari/render.py index ba8138633..164c6f21b 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -102,7 +102,6 @@ def render_template( def inspect_files( output_base_dir: str, ->>>>>>> c9abe710 (Working render!) ignore_filenames: List[str] = None, ignore_directories: List[str] = None, deleted_paths: List[Path] = None, From e08b1f4f67076d2f18c34cd5c48b348c08835ec6 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 2 Aug 2023 17:05:34 -0400 Subject: [PATCH 128/147] Fixing import --- src/_nebari/render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/render.py b/src/_nebari/render.py index 164c6f21b..c854f556a 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -2,7 +2,7 @@ import os import shutil import sys -from pathlib import Path +import pathlib from typing import Dict, List from rich import print From 27cd898a67d799eb7bd3f92c3ee439f58d8485b3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 2 Aug 2023 21:06:16 +0000 Subject: [PATCH 129/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/render.py b/src/_nebari/render.py index c854f556a..1e1c6d85b 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -1,8 +1,8 @@ import hashlib import os +import pathlib import shutil import sys -import pathlib from typing import Dict, List from rich import print From b7eebd2fe4b4133246b75db4de73aa87afbc0a32 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 2 Aug 2023 17:09:40 -0400 Subject: [PATCH 130/147] Ignore certain pathlib warings --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 192c3c715..44a2c3298 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,7 +108,11 @@ select = [ ignore = [ "E501", # Line too long "F821", # Undefined name + "PTH101", # `os.chmod()` should be replaced by `Path.chmod()` + "PTH103", # `os.makedirs()` should be replaced by `Path.mkdir(parents=True)` + "PTH113", # os.path.isfile() -> Path.is_file() "PTH118", # os.path.join() should be replaced by Path() + "PTH120", # `os.path.dirname()` should be replaced by `Path.parent` "PTH123", # open() should be replaced by Path.open() ] extend-exclude = [ From 1f9338e0d0e84b21eb436956057fbdeb5dd7a1cb Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 2 Aug 2023 17:16:53 -0400 Subject: [PATCH 131/147] Missing import --- src/_nebari/render.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_nebari/render.py b/src/_nebari/render.py index 1e1c6d85b..96a32b2e8 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -104,7 +104,7 @@ def inspect_files( output_base_dir: str, ignore_filenames: List[str] = None, ignore_directories: List[str] = None, - deleted_paths: List[Path] = None, + deleted_paths: List[pathlib.Path] = None, contents: Dict[str, str] = None, ): """Return created, updated and untracked files by computing a checksum over the provided directory. @@ -124,7 +124,7 @@ def inspect_files( output_files = {} def list_files( - directory: Path, ignore_filenames: List[str], ignore_directories: List[str] + directory: pathlib.Path, ignore_filenames: List[str], ignore_directories: List[str] ): for path in directory.rglob("*"): if not path.is_file(): From 2bdb9d756dd93fab790bc8481e69fd04a47657ae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 2 Aug 2023 21:25:01 +0000 Subject: [PATCH 132/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/render.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/_nebari/render.py b/src/_nebari/render.py index 96a32b2e8..bb04e351c 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -124,7 +124,9 @@ def inspect_files( output_files = {} def list_files( - directory: pathlib.Path, ignore_filenames: List[str], ignore_directories: List[str] + directory: pathlib.Path, + ignore_filenames: List[str], + ignore_directories: List[str], ): for path in directory.rglob("*"): if not path.is_file(): From c045be466ffca87a8eaedc6400149059ed270531 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 2 Aug 2023 17:16:53 -0400 Subject: [PATCH 133/147] Missing import --- src/_nebari/render.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_nebari/render.py b/src/_nebari/render.py index bb04e351c..64c96f7d2 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -34,7 +34,7 @@ def render_template( ) new, untracked, updated, deleted = inspect_files( - output_base_dir=str(output_directory), + output_base_dir=output_directory, ignore_filenames=[ "terraform.tfstate", ".terraform.lock.hcl", @@ -101,7 +101,7 @@ def render_template( def inspect_files( - output_base_dir: str, + output_base_dir: pathlib.Path, ignore_filenames: List[str] = None, ignore_directories: List[str] = None, deleted_paths: List[pathlib.Path] = None, From 142182c8b40f957cfac3b73c61bd64a6183310cd Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 2 Aug 2023 20:21:01 -0400 Subject: [PATCH 134/147] Changes to account for #1832 and #1868 --- pyproject.toml | 5 ----- src/_nebari/deploy.py | 2 +- src/_nebari/destroy.py | 2 +- src/_nebari/initialize.py | 8 +++----- src/_nebari/render.py | 12 +++++++----- src/_nebari/stages/base.py | 17 ++++++++--------- .../stages/kubernetes_services/__init__.py | 6 ++++++ src/_nebari/utils.py | 12 ++++++++++++ 8 files changed, 38 insertions(+), 26 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 44a2c3298..eebff1089 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,11 +108,6 @@ select = [ ignore = [ "E501", # Line too long "F821", # Undefined name - "PTH101", # `os.chmod()` should be replaced by `Path.chmod()` - "PTH103", # `os.makedirs()` should be replaced by `Path.mkdir(parents=True)` - "PTH113", # os.path.isfile() -> Path.is_file() - "PTH118", # os.path.join() should be replaced by Path() - "PTH120", # `os.path.dirname()` should be replaced by `Path.parent` "PTH123", # open() should be replaced by Path.open() ] extend-exclude = [ diff --git a/src/_nebari/deploy.py b/src/_nebari/deploy.py index 0b669931e..394fe0cce 100644 --- a/src/_nebari/deploy.py +++ b/src/_nebari/deploy.py @@ -52,7 +52,7 @@ def deploy_configuration( stage_outputs = {} with contextlib.ExitStack() as stack: for stage in stages: - s = stage(output_directory=pathlib.Path("."), config=config) + s = stage(output_directory=pathlib.Path.cwd(), config=config) stack.enter_context(s.deploy(stage_outputs)) if not disable_checks: diff --git a/src/_nebari/destroy.py b/src/_nebari/destroy.py index ecea5874a..900ad8acf 100644 --- a/src/_nebari/destroy.py +++ b/src/_nebari/destroy.py @@ -22,7 +22,7 @@ def destroy_configuration(config: schema.Main, stages: List[hookspecs.NebariStag with contextlib.ExitStack() as stack: for stage in stages: try: - s = stage(output_directory=pathlib.Path("."), config=config) + s = stage(output_directory=pathlib.Path.cwd(), config=config) stack.enter_context(s.destroy(stage_outputs, status)) except Exception as e: status[s.name] = False diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index ae7f5d780..569d6b353 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -54,15 +54,13 @@ def render_config( config["terraform_state"] = {"type": terraform_state.value} # Save default password to file - default_password_filename = os.path.join( - tempfile.gettempdir(), "NEBARI_DEFAULT_PASSWORD" - ) + default_password_filename = pathlib.Path(tempfile.gettempdir()) / "NEBARI_DEFAULT_PASSWORD" config["security"] = { "keycloak": {"initial_root_password": random_secure_string(length=32)} } - with open(default_password_filename, "w") as f: + with default_password_filename.open("w") as f: f.write(config["security"]["keycloak"]["initial_root_password"]) - os.chmod(default_password_filename, 0o700) + default_password_filename.chmod(0o700) config["theme"] = {"jupyterhub": {"hub_title": f"Nebari - { project_name }"}} config["theme"]["jupyterhub"][ diff --git a/src/_nebari/render.py b/src/_nebari/render.py index 64c96f7d2..f77e03a16 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -10,6 +10,7 @@ from _nebari.deprecate import DEPRECATED_FILE_PATHS from nebari import hookspecs, schema +from _nebari.utils import is_relative_to def render_template( @@ -76,8 +77,8 @@ def render_template( print("dry-run enabled no files will be created, updated, or deleted") else: for filename in new | updated: - output_filename = os.path.join(str(output_directory), filename) - os.makedirs(os.path.dirname(output_filename), exist_ok=True) + output_filename = output_directory / filename + output_filename.parent.mkdir(parents=True, exist_ok=True) if isinstance(contents[filename], str): with open(output_filename, "w") as f: @@ -131,6 +132,7 @@ def list_files( for path in directory.rglob("*"): if not path.is_file(): continue + yield path for filename in contents: if isinstance(contents[filename], str): @@ -140,8 +142,8 @@ def list_files( else: source_files[filename] = hashlib.sha256(contents[filename]).hexdigest() - output_filename = os.path.join(output_base_dir, filename) - if os.path.isfile(output_filename): + output_filename = pathlib.Path(output_base_dir) / filename + if output_filename.is_file(): output_files[filename] = hash_file(filename) deleted_files = set() @@ -152,7 +154,7 @@ def list_files( for filename in list_files(output_base_dir, ignore_filenames, ignore_directories): relative_path = os.path.relpath(filename, output_base_dir) - if os.path.isfile(filename): + if filename.is_file(): output_files[relative_path] = hash_file(filename) new_files = source_files.keys() - output_files.keys() diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index 8f603250d..0462eb792 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -32,15 +32,14 @@ def render(self) -> Dict[str, str]: } for root, dirs, filenames in os.walk(self.template_directory): for filename in filenames: - with open(os.path.join(root, filename), "rb") as f: - contents[ - os.path.join( - self.stage_prefix, - os.path.relpath( - os.path.join(root, filename), self.template_directory - ), - ) - ] = f.read() + root_filename = pathlib.Path(root) / filename + with root_filename.open("rb") as f: + contents[pathlib.Path( + self.stage_prefix, + os.path.relpath( + root_filename, self.template_directory + ), + )] = f.read() return contents def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index ca74cee8c..d6e26a59c 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -460,6 +460,12 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): "*/*": ["viewer"], }, }, + "argo-workflows-jupyter-scheduler": { + "primary_namespace": "", + "role_bindings": { + "*/*": ["viewer"], + }, + }, } # Compound any logout URLs from extensions so they are are logged out in succession diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 9a5fc2599..d729bd913 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -263,3 +263,15 @@ def random_secure_string( length: int = 16, chars: str = string.ascii_lowercase + string.digits ): return "".join(secrets.choice(chars) for i in range(length)) + + +def is_relative_to(self: pathlib.Path, other: pathlib.Path, /) -> bool: + """Compatibility function to bring ``Path.is_relative_to`` to Python 3.8""" + if sys.version_info[:2] >= (3, 9): + return self.is_relative_to(other) + + try: + self.relative_to(other) + return True + except ValueError: + return False From 371483b07d99eb8f594857aaab04cadb288939be Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 3 Aug 2023 00:22:39 +0000 Subject: [PATCH 135/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/initialize.py | 4 +++- src/_nebari/render.py | 2 +- src/_nebari/stages/base.py | 12 ++++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index 569d6b353..661dca9e5 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -54,7 +54,9 @@ def render_config( config["terraform_state"] = {"type": terraform_state.value} # Save default password to file - default_password_filename = pathlib.Path(tempfile.gettempdir()) / "NEBARI_DEFAULT_PASSWORD" + default_password_filename = ( + pathlib.Path(tempfile.gettempdir()) / "NEBARI_DEFAULT_PASSWORD" + ) config["security"] = { "keycloak": {"initial_root_password": random_secure_string(length=32)} } diff --git a/src/_nebari/render.py b/src/_nebari/render.py index f77e03a16..f5d4a94ab 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -9,8 +9,8 @@ from rich.table import Table from _nebari.deprecate import DEPRECATED_FILE_PATHS -from nebari import hookspecs, schema from _nebari.utils import is_relative_to +from nebari import hookspecs, schema def render_template( diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index 0462eb792..60bd27ac6 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -34,12 +34,12 @@ def render(self) -> Dict[str, str]: for filename in filenames: root_filename = pathlib.Path(root) / filename with root_filename.open("rb") as f: - contents[pathlib.Path( - self.stage_prefix, - os.path.relpath( - root_filename, self.template_directory - ), - )] = f.read() + contents[ + pathlib.Path( + self.stage_prefix, + os.path.relpath(root_filename, self.template_directory), + ) + ] = f.read() return contents def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): From f36c41fb9b2acbc051c78d71a4b774c41b785921 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 2 Aug 2023 20:25:41 -0400 Subject: [PATCH 136/147] Missing pathlib import --- src/_nebari/initialize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index 661dca9e5..3b71e335c 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -2,6 +2,7 @@ import os import re import tempfile +import pathlib import pydantic import requests From beac07aa1b4f1a1a26d3ef22468f523ca50c5e63 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 3 Aug 2023 00:25:59 +0000 Subject: [PATCH 137/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/initialize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index 3b71e335c..d1d3cd1b3 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -1,8 +1,8 @@ import logging import os +import pathlib import re import tempfile -import pathlib import pydantic import requests From cc253933c6e917fad2b25c3f85f49f08873597f4 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 2 Aug 2023 20:39:11 -0400 Subject: [PATCH 138/147] Ensure pathlib everywhere --- src/_nebari/render.py | 2 +- src/_nebari/stages/base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_nebari/render.py b/src/_nebari/render.py index f5d4a94ab..4d3b685e7 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -153,7 +153,7 @@ def list_files( deleted_files.add(path) for filename in list_files(output_base_dir, ignore_filenames, ignore_directories): - relative_path = os.path.relpath(filename, output_base_dir) + relative_path = pathlib.Path(os.path.relpath(filename, output_base_dir)) if filename.is_file(): output_files[relative_path] = hash_file(filename) diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index 60bd27ac6..8d57b9819 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -26,7 +26,7 @@ def tf_objects(self) -> List[Dict]: def render(self) -> Dict[str, str]: contents = { - str(self.stage_prefix / "_nebari.tf.json"): terraform.tf_render_objects( + (self.stage_prefix / "_nebari.tf.json"): terraform.tf_render_objects( self.tf_objects() ) } From 05fe1d87c04a7b393eacf70cf6bae86fb400fa33 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 2 Aug 2023 21:01:21 -0400 Subject: [PATCH 139/147] Ensure using pathlib type --- src/_nebari/stages/base.py | 2 +- src/_nebari/stages/bootstrap/__init__.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index 8d57b9819..05776f9e3 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -24,7 +24,7 @@ def state_imports(self) -> List[Tuple[str, str]]: def tf_objects(self) -> List[Dict]: return [NebariTerraformState(self.name, self.config)] - def render(self) -> Dict[str, str]: + def render(self) -> Dict[pathlib.Path, str]: contents = { (self.stage_prefix / "_nebari.tf.json"): terraform.tf_render_objects( self.tf_objects() diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index 6229417bd..2ff7c9d4f 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -3,6 +3,7 @@ import typing from inspect import cleandoc from typing import Dict, List +import pathlib from _nebari.provider.cicd.github import gen_nebari_linter, gen_nebari_ops from _nebari.provider.cicd.gitlab import gen_gitlab_ci @@ -25,7 +26,7 @@ def gen_gitignore(): # python __pycache__ """ - return {".gitignore": cleandoc(filestoignore)} + return {pathlib.Path(".gitignore"): cleandoc(filestoignore)} def gen_cicd(config: schema.Main): @@ -40,12 +41,12 @@ def gen_cicd(config: schema.Main): cicd_files = {} if config.ci_cd.type == CiEnum.github_actions: - gha_dir = ".github/workflows/" - cicd_files[gha_dir + "nebari-ops.yaml"] = gen_nebari_ops(config) - cicd_files[gha_dir + "nebari-linter.yaml"] = gen_nebari_linter(config) + gha_dir = pathlib.Path(".github/workflows/") + cicd_files[gha_dir / "nebari-ops.yaml"] = gen_nebari_ops(config) + cicd_files[gha_dir / "nebari-linter.yaml"] = gen_nebari_linter(config) elif config.ci_cd.type == CiEnum.gitlab_ci: - cicd_files[".gitlab-ci.yml"] = gen_gitlab_ci(config) + cicd_files[pathlib.Path(".gitlab-ci.yml")] = gen_gitlab_ci(config) else: raise ValueError( From ec78e2cba7679b41a243d2d073dca808a95a8b46 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 3 Aug 2023 01:01:37 +0000 Subject: [PATCH 140/147] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/stages/bootstrap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/stages/bootstrap/__init__.py b/src/_nebari/stages/bootstrap/__init__.py index 2ff7c9d4f..873ab33de 100644 --- a/src/_nebari/stages/bootstrap/__init__.py +++ b/src/_nebari/stages/bootstrap/__init__.py @@ -1,9 +1,9 @@ import enum import io +import pathlib import typing from inspect import cleandoc from typing import Dict, List -import pathlib from _nebari.provider.cicd.github import gen_nebari_linter, gen_nebari_ops from _nebari.provider.cicd.gitlab import gen_gitlab_ci From e27c83e730be650bfe71dca1589ae46d02860f39 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Wed, 2 Aug 2023 21:48:42 -0400 Subject: [PATCH 141/147] Bump the highest supported kubernetes version --- src/_nebari/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py index 7380c6a8f..f4c1edcbf 100644 --- a/src/_nebari/constants.py +++ b/src/_nebari/constants.py @@ -5,7 +5,7 @@ # 04-kubernetes-ingress DEFAULT_TRAEFIK_IMAGE_TAG = "2.9.1" -HIGHEST_SUPPORTED_K8S_VERSION = ("1", "24", "13") +HIGHEST_SUPPORTED_K8S_VERSION = ("1", "24", "16") DEFAULT_GKE_RELEASE_CHANNEL = "UNSPECIFIED" DEFAULT_NEBARI_DASK_VERSION = CURRENT_RELEASE From 107d882ace50914c5add0181587125a0724403fe Mon Sep 17 00:00:00 2001 From: iameskild Date: Tue, 8 Aug 2023 16:00:59 -0700 Subject: [PATCH 142/147] Clean up tests --- tests/tests_unit/test_cli.py | 48 ++++--- tests/tests_unit/test_init.py | 2 +- tests/tests_unit/test_links.py | 2 +- tests/tests_unit/test_render.py | 219 +++++++++++++++---------------- tests/tests_unit/test_schema.py | 68 ++++------ tests/tests_unit/test_upgrade.py | 25 ++-- 6 files changed, 180 insertions(+), 184 deletions(-) diff --git a/tests/tests_unit/test_cli.py b/tests/tests_unit/test_cli.py index d8a4e423b..a45072d70 100644 --- a/tests/tests_unit/test_cli.py +++ b/tests/tests_unit/test_cli.py @@ -2,8 +2,8 @@ import pytest -from _nebari.subcommands.init import InitInputs -from nebari.plugins import nebari_plugin_manager +from _nebari.schema import InitInputs +from _nebari.utils import load_yaml PROJECT_NAME = "clitest" DOMAIN_NAME = "clitest.dev" @@ -13,7 +13,7 @@ "namespace, auth_provider, ci_provider, ssl_cert_email", ( [None, None, None, None], - ["prod", "password", "github-actions", "it@acme.org"], + ["prod", "github", "github-actions", "it@acme.org"], ), ) def test_nebari_init(tmp_path, namespace, auth_provider, ci_provider, ssl_cert_email): @@ -48,20 +48,34 @@ def test_nebari_init(tmp_path, namespace, auth_provider, ci_provider, ssl_cert_e subprocess.run(command, cwd=tmp_path, check=True) - config = nebari_plugin_manager.read_config(tmp_path / "nebari-config.yaml") + config = load_yaml(tmp_path / "nebari-config.yaml") - assert config.namespace == namespace - assert config.security.authentication.type.lower() == auth_provider - assert config.ci_cd.type == ci_provider - assert config.certificate.acme_email == ssl_cert_email + assert config.get("namespace") == namespace + assert ( + config.get("security", {}).get("authentication", {}).get("type").lower() + == auth_provider + ) + ci_cd = config.get("ci_cd", None) + if ci_cd: + assert ci_cd.get("type", {}) == ci_provider + else: + assert ci_cd == ci_provider + acme_email = config.get("certificate", None) + if acme_email: + assert acme_email.get("acme_email") == ssl_cert_email + else: + assert acme_email == ssl_cert_email -@pytest.mark.parametrize( - "command", - ( - ["nebari", "--version"], - ["nebari", "info"], - ), -) -def test_nebari_commands_no_args(command): - subprocess.run(command, check=True, capture_output=True, text=True).stdout.strip() +def test_python_invocation(): + def run(command): + return subprocess.run( + command, check=True, capture_output=True, text=True + ).stdout.strip() + + command = ["nebari", "--version"] + + actual = run(["python", "-m", *command]) + expected = run(command) + + assert actual == expected diff --git a/tests/tests_unit/test_init.py b/tests/tests_unit/test_init.py index 897839cbd..a64d511fc 100644 --- a/tests/tests_unit/test_init.py +++ b/tests/tests_unit/test_init.py @@ -6,7 +6,7 @@ @pytest.mark.parametrize( "k8s_version, expected", [(None, True), ("1.19", True), (1000, ValueError)] ) -def test_init(setup_fixture, k8s_version, expected, render_config_partial): +def test_init(setup_fixture, k8s_version, expected): (nebari_config_loc, render_config_inputs) = setup_fixture ( project, diff --git a/tests/tests_unit/test_links.py b/tests/tests_unit/test_links.py index a393391ce..8df7ee1e6 100644 --- a/tests/tests_unit/test_links.py +++ b/tests/tests_unit/test_links.py @@ -1,7 +1,7 @@ import pytest import requests -from _nebari.constants import AWS_ENV_DOCS, AZURE_ENV_DOCS, DO_ENV_DOCS, GCP_ENV_DOCS +from _nebari.utils import AWS_ENV_DOCS, AZURE_ENV_DOCS, DO_ENV_DOCS, GCP_ENV_DOCS LINKS_TO_TEST = [ DO_ENV_DOCS, diff --git a/tests/tests_unit/test_render.py b/tests/tests_unit/test_render.py index 9e56fede2..2ec7f407a 100644 --- a/tests/tests_unit/test_render.py +++ b/tests/tests_unit/test_render.py @@ -1,120 +1,119 @@ import os +from pathlib import Path import pytest from ruamel.yaml import YAML -from _nebari.render import render_template -from _nebari.stages.base import get_available_stages +from _nebari.render import render_template, set_env_vars_in_config from .utils import PRESERVED_DIR, render_config_partial -def test_render_config(nebari_render): - output_directory, config_filename = nebari_render - config = nebari_plugin_manager.read_config(config_filename) - assert {"nebari-config.yaml", "stages", ".gitignore"} <= set( - os.listdir(output_directory) +@pytest.fixture +def write_nebari_config_to_file(setup_fixture): + nebari_config_loc, render_config_inputs = setup_fixture + ( + project, + namespace, + domain, + cloud_provider, + ci_provider, + auth_provider, + ) = render_config_inputs + + config = render_config_partial( + project_name=project, + namespace=namespace, + nebari_domain=domain, + cloud_provider=cloud_provider, + ci_provider=ci_provider, + auth_provider=auth_provider, + kubernetes_version=None, ) - assert { - "07-kubernetes-services", - "02-infrastructure", - "01-terraform-state", - "05-kubernetes-keycloak", - "08-nebari-tf-extensions", - "06-kubernetes-keycloak-configuration", - "04-kubernetes-ingress", - "03-kubernetes-initialize", - } == set(os.listdir(output_directory / "stages")) - - if config.provider == schema.ProviderEnum.do: - assert (output_directory / "stages" / "01-terraform-state/do").is_dir() - assert (output_directory / "stages" / "02-infrastructure/do").is_dir() - elif config.provider == schema.ProviderEnum.aws: - assert (output_directory / "stages" / "01-terraform-state/aws").is_dir() - assert (output_directory / "stages" / "02-infrastructure/aws").is_dir() - elif config.provider == schema.ProviderEnum.gcp: - assert (output_directory / "stages" / "01-terraform-state/gcp").is_dir() - assert (output_directory / "stages" / "02-infrastructure/gcp").is_dir() - elif config.provider == schema.ProviderEnum.azure: - assert (output_directory / "stages" / "01-terraform-state/azure").is_dir() - assert (output_directory / "stages" / "02-infrastructure/azure").is_dir() - - if config.ci_cd.type == CiEnum.github_actions: - assert (output_directory / ".github/workflows/").is_dir() - elif config.ci_cd.type == CiEnum.gitlab_ci: - assert (output_directory / ".gitlab-ci.yml").is_file() - - -# @pytest.fixture -# def write_nebari_config_to_file(setup_fixture, render_config_partial): -# nebari_config_loc, render_config_inputs = setup_fixture -# ( -# project, -# namespace, -# domain, -# cloud_provider, -# ci_provider, -# auth_provider, -# ) = render_config_inputs - -# config = render_config_partial( -# project_name=project, -# namespace=namespace, -# nebari_domain=domain, -# cloud_provider=cloud_provider, -# ci_provider=ci_provider, -# auth_provider=auth_provider, -# kubernetes_version=None, -# ) - -# stages = get_available_stages() -# render_template(str(nebari_config_loc.parent), config, stages) - -# yield setup_fixture - - -# def test_render_template(write_nebari_config_to_file): -# nebari_config_loc, render_config_inputs = write_nebari_config_to_file -# ( -# project, -# namespace, -# domain, -# cloud_provider, -# ci_provider, -# auth_provider, -# ) = render_config_inputs - -# yaml = YAML() -# nebari_config_json = yaml.load(nebari_config_loc.read_text()) - -# assert nebari_config_json["project_name"] == project -# assert nebari_config_json["namespace"] == namespace -# assert nebari_config_json["domain"] == domain -# assert nebari_config_json["provider"] == cloud_provider - - -# def test_exists_after_render(write_nebari_config_to_file): -# items_to_check = [ -# ".gitignore", -# "stages", -# "nebari-config.yaml", -# PRESERVED_DIR, -# ] - -# nebari_config_loc, _ = write_nebari_config_to_file - -# yaml = YAML() -# nebari_config_json = yaml.load(nebari_config_loc.read_text()) - -# # list of files/dirs available after `nebari render` command -# ls = os.listdir(Path(nebari_config_loc).parent.resolve()) - -# cicd = nebari_config_json.get("ci_cd", {}).get("type", None) - -# if cicd == "github-actions": -# items_to_check.append(".github") -# elif cicd == "gitlab-ci": -# items_to_check.append(".gitlab-ci.yml") - -# for i in items_to_check: -# assert i in ls + + # write to nebari_config.yaml + yaml = YAML(typ="unsafe", pure=True) + yaml.dump(config, nebari_config_loc) + + render_template(nebari_config_loc.parent, nebari_config_loc) + + yield setup_fixture + + +def test_get_secret_config_entries(monkeypatch): + sec1 = "secret1" + sec2 = "nestedsecret1" + config_orig = { + "key1": "value1", + "key2": "NEBARI_SECRET_secret_val", + "key3": { + "nested_key1": "nested_value1", + "nested_key2": "NEBARI_SECRET_nested_secret_val", + }, + } + expected = { + "key1": "value1", + "key2": sec1, + "key3": { + "nested_key1": "nested_value1", + "nested_key2": sec2, + }, + } + + # should raise error if implied env var is not set + with pytest.raises(EnvironmentError): + config = config_orig.copy() + set_env_vars_in_config(config) + + monkeypatch.setenv("secret_val", sec1, prepend=False) + monkeypatch.setenv("nested_secret_val", sec2, prepend=False) + config = config_orig.copy() + set_env_vars_in_config(config) + assert config == expected + + +def test_render_template(write_nebari_config_to_file): + nebari_config_loc, render_config_inputs = write_nebari_config_to_file + ( + project, + namespace, + domain, + cloud_provider, + ci_provider, + auth_provider, + ) = render_config_inputs + + yaml = YAML() + nebari_config_json = yaml.load(nebari_config_loc.read_text()) + + assert nebari_config_json["project_name"] == project + assert nebari_config_json["namespace"] == namespace + assert nebari_config_json["domain"] == domain + assert nebari_config_json["provider"] == cloud_provider + + +def test_exists_after_render(write_nebari_config_to_file): + items_to_check = [ + ".gitignore", + "stages", + "nebari-config.yaml", + PRESERVED_DIR, + ] + + nebari_config_loc, _ = write_nebari_config_to_file + + yaml = YAML() + nebari_config_json = yaml.load(nebari_config_loc.read_text()) + + # list of files/dirs available after `nebari render` command + ls = os.listdir(Path(nebari_config_loc).parent.resolve()) + + cicd = nebari_config_json.get("ci_cd", {}).get("type", None) + + if cicd == "github-actions": + items_to_check.append(".github") + elif cicd == "gitlab-ci": + items_to_check.append(".gitlab-ci.yml") + + for i in items_to_check: + assert i in ls diff --git a/tests/tests_unit/test_schema.py b/tests/tests_unit/test_schema.py index dea5b96e5..d4d8cf878 100644 --- a/tests/tests_unit/test_schema.py +++ b/tests/tests_unit/test_schema.py @@ -3,49 +3,25 @@ from .utils import render_config_partial -def test_minimal_schema(): - config = nebari_plugin_manager.config_schema(project_name="test") - assert config.project_name == "test" - assert config.storage.conda_store == "200Gi" - - -def test_minimal_schema_from_file(tmp_path): - filename = tmp_path / "nebari-config.yaml" - with filename.open("w") as f: - f.write("project_name: test\n") - - config = nebari_plugin_manager.read_config(filename) - assert config.project_name == "test" - assert config.storage.conda_store == "200Gi" - - -def test_minimal_schema_from_file_with_env(tmp_path, monkeypatch): - filename = tmp_path / "nebari-config.yaml" - with filename.open("w") as f: - f.write("project_name: test\n") - - monkeypatch.setenv("NEBARI_SECRET__project_name", "env") - monkeypatch.setenv("NEBARI_SECRET__storage__conda_store", "1000Gi") - - config = nebari_plugin_manager.read_config(filename) - assert config.project_name == "env" - assert config.storage.conda_store == "1000Gi" - - -def test_minimal_schema_from_file_without_env(tmp_path, monkeypatch): - filename = tmp_path / "nebari-config.yaml" - with filename.open("w") as f: - f.write("project_name: test\n") - - monkeypatch.setenv("NEBARI_SECRET__project_name", "env") - monkeypatch.setenv("NEBARI_SECRET__storage__conda_store", "1000Gi") - - config = nebari_plugin_manager.read_config(filename, read_environment=False) - assert config.project_name == "test" - assert config.storage.conda_store == "200Gi" - - -def test_render_schema(nebari_config): - assert isinstance(nebari_config, schema.Main) - assert nebari_config.project_name == f"pytest{nebari_config.provider.value}" - assert nebari_config.namespace == "dev" +def test_schema(setup_fixture): + (nebari_config_loc, render_config_inputs) = setup_fixture + ( + project, + namespace, + domain, + cloud_provider, + ci_provider, + auth_provider, + ) = render_config_inputs + + config = render_config_partial( + project_name=project, + namespace=namespace, + nebari_domain=domain, + cloud_provider=cloud_provider, + ci_provider=ci_provider, + auth_provider=auth_provider, + kubernetes_version=None, + ) + + _nebari.schema.verify(config) diff --git a/tests/tests_unit/test_upgrade.py b/tests/tests_unit/test_upgrade.py index 0946dcd99..f53d980f4 100644 --- a/tests/tests_unit/test_upgrade.py +++ b/tests/tests_unit/test_upgrade.py @@ -2,9 +2,8 @@ import pytest -from _nebari.upgrade import do_upgrade +from _nebari.upgrade import do_upgrade, load_yaml, verify from _nebari.version import __version__, rounded_ver_parse -from nebari.plugins import nebari_plugin_manager @pytest.fixture @@ -70,24 +69,32 @@ def test_upgrade_4_0( return # Check the resulting YAML - config = nebari_plugin_manager.read_config(tmp_qhub_config) + config = load_yaml(tmp_qhub_config) - assert len(config.security.keycloak.initial_root_password) == 16 - assert not hasattr(config.security, "users") - assert not hasattr(config.security, "groups") + verify( + config + ) # Would raise an error if invalid by current Nebari version's standards + + assert len(config["security"]["keycloak"]["initial_root_password"]) == 16 + + assert "users" not in config["security"] + assert "groups" not in config["security"] __rounded_version__ = ".".join([str(c) for c in rounded_ver_parse(__version__)]) # Check image versions have been bumped up assert ( - config.default_images.jupyterhub + config["default_images"]["jupyterhub"] == f"quansight/nebari-jupyterhub:v{__rounded_version__}" ) assert ( - config.profiles.jupyterlab[0].kubespawner_override.image + config["profiles"]["jupyterlab"][0]["kubespawner_override"]["image"] == f"quansight/nebari-jupyterlab:v{__rounded_version__}" ) - assert config.security.authentication.type != "custom" + + assert ( + config.get("security", {}).get("authentication", {}).get("type", "") != "custom" + ) # Keycloak import users json assert ( From 3e1380d592e5473c447287828ed858593d0179c1 Mon Sep 17 00:00:00 2001 From: iameskild Date: Tue, 8 Aug 2023 16:58:43 -0700 Subject: [PATCH 143/147] Minor clean, path updates --- src/_nebari/initialize.py | 6 ++---- src/_nebari/render.py | 5 +++-- src/_nebari/stages/base.py | 4 +++- src/_nebari/subcommands/init.py | 5 ++++- src/_nebari/utils.py | 6 +++--- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/_nebari/initialize.py b/src/_nebari/initialize.py index d1d3cd1b3..559ea5ae3 100644 --- a/src/_nebari/initialize.py +++ b/src/_nebari/initialize.py @@ -1,8 +1,8 @@ import logging import os -import pathlib import re import tempfile +from pathlib import Path import pydantic import requests @@ -55,9 +55,7 @@ def render_config( config["terraform_state"] = {"type": terraform_state.value} # Save default password to file - default_password_filename = ( - pathlib.Path(tempfile.gettempdir()) / "NEBARI_DEFAULT_PASSWORD" - ) + default_password_filename = Path(tempfile.gettempdir()) / "NEBARI_DEFAULT_PASSWORD" config["security"] = { "keycloak": {"initial_root_password": random_secure_string(length=32)} } diff --git a/src/_nebari/render.py b/src/_nebari/render.py index 4d3b685e7..fa7033f60 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -1,5 +1,4 @@ import hashlib -import os import pathlib import shutil import sys @@ -153,7 +152,9 @@ def list_files( deleted_files.add(path) for filename in list_files(output_base_dir, ignore_filenames, ignore_directories): - relative_path = pathlib.Path(os.path.relpath(filename, output_base_dir)) + relative_path = pathlib.Path.relative_to( + pathlib.Path(filename), output_base_dir + ) if filename.is_file(): output_files[relative_path] = hash_file(filename) diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index 05776f9e3..d6e91ddbc 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -37,7 +37,9 @@ def render(self) -> Dict[pathlib.Path, str]: contents[ pathlib.Path( self.stage_prefix, - os.path.relpath(root_filename, self.template_directory), + pathlib.Path.relative_to( + pathlib.Path(root_filename), self.template_directory + ), ) ] = f.read() return contents diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index b0aa3c6e7..4ec23adec 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -639,7 +639,10 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): ).unsafe_ask() if not disable_checks: - check_ssl_cert_email(ctx, ssl_cert_email=inputs.ssl_cert_email) + typer_validate_regex( + schema.email_regex, + f"Email must be valid and match the regex {schema.email_regex}", + ) # ADVANCED FEATURES rich.print( diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index d729bd913..154f2faf7 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -1,7 +1,6 @@ import contextlib import functools import os -import pathlib import re import secrets import signal @@ -11,6 +10,7 @@ import threading import time import warnings +from pathlib import Path from typing import Dict, List from ruamel.yaml import YAML @@ -96,7 +96,7 @@ def kill_process(): ) # Should already have finished because we have drained stdout -def load_yaml(config_filename: pathlib.Path): +def load_yaml(config_filename: Path): """ Return yaml dict containing config loaded from config_filename. """ @@ -265,7 +265,7 @@ def random_secure_string( return "".join(secrets.choice(chars) for i in range(length)) -def is_relative_to(self: pathlib.Path, other: pathlib.Path, /) -> bool: +def is_relative_to(self: Path, other: Path, /) -> bool: """Compatibility function to bring ``Path.is_relative_to`` to Python 3.8""" if sys.version_info[:2] >= (3, 9): return self.is_relative_to(other) From 79b9066b4c7d3ddb0871c8fe1823e8f392b33208 Mon Sep 17 00:00:00 2001 From: iameskild Date: Tue, 8 Aug 2023 18:21:37 -0700 Subject: [PATCH 144/147] Remove linger checks.py --- src/_nebari/stages/checks.py | 315 ----------------------------------- 1 file changed, 315 deletions(-) delete mode 100644 src/_nebari/stages/checks.py diff --git a/src/_nebari/stages/checks.py b/src/_nebari/stages/checks.py deleted file mode 100644 index 86259846a..000000000 --- a/src/_nebari/stages/checks.py +++ /dev/null @@ -1,315 +0,0 @@ -import socket -import sys -import time - -# check and retry settings -NUM_ATTEMPTS = 10 -TIMEOUT = 10 # seconds - - -def stage_02_infrastructure(stage_outputs, nebari_config): - from kubernetes import client, config - from kubernetes.client.rest import ApiException - - directory = "stages/02-infrastructure" - config.load_kube_config( - config_file=stage_outputs["stages/02-infrastructure"]["kubeconfig_filename"][ - "value" - ] - ) - - try: - api_instance = client.CoreV1Api() - result = api_instance.list_namespace() - except ApiException: - print( - f"ERROR: After stage directory={directory} unable to connect to kubernetes cluster" - ) - sys.exit(1) - - if len(result.items) < 1: - print( - f"ERROR: After stage directory={directory} no nodes provisioned within kubernetes cluster" - ) - sys.exit(1) - - print( - f"After stage directory={directory} kubernetes cluster successfully provisioned" - ) - - -def stage_03_kubernetes_initialize(stage_outputs, nebari_config): - from kubernetes import client, config - from kubernetes.client.rest import ApiException - - directory = "stages/03-kubernetes-initialize" - config.load_kube_config( - config_file=stage_outputs["stages/02-infrastructure"]["kubeconfig_filename"][ - "value" - ] - ) - - try: - api_instance = client.CoreV1Api() - result = api_instance.list_namespace() - except ApiException: - print( - f"ERROR: After stage directory={directory} unable to connect to kubernetes cluster" - ) - sys.exit(1) - - namespaces = {_.metadata.name for _ in result.items} - if nebari_config["namespace"] not in namespaces: - print( - f"ERROR: After stage directory={directory} namespace={config['namespace']} not provisioned within kubernetes cluster" - ) - sys.exit(1) - - print(f"After stage directory={directory} kubernetes initialized successfully") - - -def stage_04_kubernetes_ingress(stage_outputs, nebari_config): - directory = "stages/04-kubernetes-ingress" - - def _attempt_tcp_connect(host, port, num_attempts=NUM_ATTEMPTS, timeout=TIMEOUT): - for i in range(num_attempts): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - # normalize hostname to ip address - ip = socket.gethostbyname(host) - s.settimeout(5) - result = s.connect_ex((ip, port)) - if result == 0: - print(f"Attempt {i+1} succeeded to connect to tcp://{ip}:{port}") - return True - print(f"Attempt {i+1} failed to connect to tcp tcp://{ip}:{port}") - except socket.gaierror: - print(f"Attempt {i+1} failed to get IP for {host}...") - finally: - s.close() - - time.sleep(timeout) - - return False - - tcp_ports = { - 80, # http - 443, # https - 8022, # jupyterhub-ssh ssh - 8023, # jupyterhub-ssh sftp - 9080, # minio - 8786, # dask-scheduler - } - ip_or_name = stage_outputs[directory]["load_balancer_address"]["value"] - host = ip_or_name["hostname"] or ip_or_name["ip"] - host = host.strip("\n") - - for port in tcp_ports: - if not _attempt_tcp_connect(host, port): - print( - f"ERROR: After stage directory={directory} unable to connect to ingress host={host} port={port}" - ) - sys.exit(1) - - print( - f"After stage directory={directory} kubernetes ingress available on tcp ports={tcp_ports}" - ) - - -def check_ingress_dns(stage_outputs, config, disable_prompt): - directory = "stages/04-kubernetes-ingress" - - ip_or_name = stage_outputs[directory]["load_balancer_address"]["value"] - ip = socket.gethostbyname(ip_or_name["hostname"] or ip_or_name["ip"]) - domain_name = config["domain"] - - def _attempt_dns_lookup( - domain_name, ip, num_attempts=NUM_ATTEMPTS, timeout=TIMEOUT - ): - for i in range(num_attempts): - try: - resolved_ip = socket.gethostbyname(domain_name) - if resolved_ip == ip: - print( - f"DNS configured domain={domain_name} matches ingress ip={ip}" - ) - return True - else: - print( - f"Attempt {i+1} polling DNS domain={domain_name} does not match ip={ip} instead got {resolved_ip}" - ) - except socket.gaierror: - print( - f"Attempt {i+1} polling DNS domain={domain_name} record does not exist" - ) - time.sleep(timeout) - return False - - attempt = 0 - while not _attempt_dns_lookup(domain_name, ip): - sleeptime = 60 * (2**attempt) - if not disable_prompt: - input( - f"After attempting to poll the DNS, the record for domain={domain_name} appears not to exist, " - f"has recently been updated, or has yet to fully propagate. This non-deterministic behavior is likely due to " - f"DNS caching and will likely resolve itself in a few minutes.\n\n\tTo poll the DNS again in {sleeptime} seconds " - f"[Press Enter].\n\n...otherwise kill the process and run the deployment again later..." - ) - - print(f"Will attempt to poll DNS again in {sleeptime} seconds...") - time.sleep(sleeptime) - attempt += 1 - if attempt == 5: - print( - f"ERROR: After stage directory={directory} DNS domain={domain_name} does not point to ip={ip}" - ) - sys.exit(1) - - -def stage_05_kubernetes_keycloak(stage_outputs, config): - directory = "stages/05-kubernetes-keycloak" - - from keycloak import KeycloakAdmin - from keycloak.exceptions import KeycloakError - - keycloak_url = ( - f"{stage_outputs[directory]['keycloak_credentials']['value']['url']}/auth/" - ) - - def _attempt_keycloak_connection( - keycloak_url, - username, - password, - realm_name, - client_id, - verify=False, - num_attempts=NUM_ATTEMPTS, - timeout=TIMEOUT, - ): - for i in range(num_attempts): - try: - KeycloakAdmin( - keycloak_url, - username=username, - password=password, - realm_name=realm_name, - client_id=client_id, - verify=verify, - ) - print(f"Attempt {i+1} succeeded connecting to keycloak master realm") - return True - except KeycloakError as e: - print(e) - print(f"Attempt {i+1} failed connecting to keycloak master realm") - time.sleep(timeout) - return False - - if not _attempt_keycloak_connection( - keycloak_url, - stage_outputs[directory]["keycloak_credentials"]["value"]["username"], - stage_outputs[directory]["keycloak_credentials"]["value"]["password"], - stage_outputs[directory]["keycloak_credentials"]["value"]["realm"], - stage_outputs[directory]["keycloak_credentials"]["value"]["client_id"], - verify=False, - ): - print( - f"ERROR: unable to connect to keycloak master realm at url={keycloak_url} with root credentials" - ) - sys.exit(1) - - print("Keycloak service successfully started") - - -def stage_06_kubernetes_keycloak_configuration(stage_outputs, config): - directory = "stages/05-kubernetes-keycloak" - - from keycloak import KeycloakAdmin - from keycloak.exceptions import KeycloakError - - keycloak_url = ( - f"{stage_outputs[directory]['keycloak_credentials']['value']['url']}/auth/" - ) - - def _attempt_keycloak_connection( - keycloak_url, - username, - password, - realm_name, - client_id, - nebari_realm, - verify=False, - num_attempts=NUM_ATTEMPTS, - timeout=TIMEOUT, - ): - for i in range(num_attempts): - try: - realm_admin = KeycloakAdmin( - keycloak_url, - username=username, - password=password, - realm_name=realm_name, - client_id=client_id, - verify=verify, - ) - existing_realms = {_["id"] for _ in realm_admin.get_realms()} - if nebari_realm in existing_realms: - print( - f"Attempt {i+1} succeeded connecting to keycloak and nebari realm={nebari_realm} exists" - ) - return True - else: - print( - f"Attempt {i+1} succeeded connecting to keycloak but nebari realm did not exist" - ) - except KeycloakError: - print(f"Attempt {i+1} failed connecting to keycloak master realm") - time.sleep(timeout) - return False - - if not _attempt_keycloak_connection( - keycloak_url, - stage_outputs[directory]["keycloak_credentials"]["value"]["username"], - stage_outputs[directory]["keycloak_credentials"]["value"]["password"], - stage_outputs[directory]["keycloak_credentials"]["value"]["realm"], - stage_outputs[directory]["keycloak_credentials"]["value"]["client_id"], - nebari_realm=stage_outputs["stages/06-kubernetes-keycloak-configuration"][ - "realm_id" - ]["value"], - verify=False, - ): - print( - "ERROR: unable to connect to keycloak master realm and ensure that nebari realm exists" - ) - sys.exit(1) - - print("Keycloak service successfully started with nebari realm") - - -def stage_07_kubernetes_services(stage_outputs, config): - directory = "stages/07-kubernetes-services" - import requests - - # suppress insecure warnings - import urllib3 - - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - - def _attempt_connect_url( - url, verify=False, num_attempts=NUM_ATTEMPTS, timeout=TIMEOUT - ): - for i in range(num_attempts): - response = requests.get(url, verify=verify, timeout=timeout) - if response.status_code < 400: - print(f"Attempt {i+1} health check succeeded for url={url}") - return True - else: - print(f"Attempt {i+1} health check failed for url={url}") - time.sleep(timeout) - return False - - services = stage_outputs[directory]["service_urls"]["value"] - for service_name, service in services.items(): - service_url = service["health_url"] - if service_url and not _attempt_connect_url(service_url): - print(f"ERROR: Service {service_name} DOWN when checking url={service_url}") - sys.exit(1) From 7167b5134180ba0804e04f457aa0a8813cc7ee91 Mon Sep 17 00:00:00 2001 From: iameskild Date: Wed, 9 Aug 2023 08:54:16 -0700 Subject: [PATCH 145/147] Fix tests --- src/_nebari/render.py | 66 ++++++++++ tests/tests_unit/conftest.py | 220 +++++++++++++++++++++++-------- tests/tests_unit/test_cli.py | 48 +++---- tests/tests_unit/test_init.py | 57 ++++---- tests/tests_unit/test_links.py | 2 +- tests/tests_unit/test_render.py | 150 +++++---------------- tests/tests_unit/test_schema.py | 77 +++++++---- tests/tests_unit/test_upgrade.py | 25 ++-- 8 files changed, 377 insertions(+), 268 deletions(-) diff --git a/src/_nebari/render.py b/src/_nebari/render.py index fa7033f60..8679a80cd 100644 --- a/src/_nebari/render.py +++ b/src/_nebari/render.py @@ -1,4 +1,6 @@ +import functools import hashlib +import os import pathlib import shutil import sys @@ -177,3 +179,67 @@ def hash_file(file_path: str): """ with open(file_path, "rb") as f: return hashlib.sha256(f.read()).hexdigest() + + +def set_env_vars_in_config(config): + """ + + For values in the config starting with 'NEBARI_SECRET_XXX' the environment + variables are searched for the pattern XXX and the config value is + modified. This enables setting secret values that should not be directly + stored in the config file. + + NOTE: variables are most likely written to a file somewhere upon render. In + order to further reduce risk of exposure of any of these variables you might + consider preventing storage of the terraform render output. + """ + private_entries = get_secret_config_entries(config) + for idx in private_entries: + set_nebari_secret(config, idx) + + +def get_secret_config_entries(config, config_idx=None, private_entries=None): + output = private_entries or [] + if config_idx is None: + sub_dict = config + config_idx = [] + else: + sub_dict = get_sub_config(config, config_idx) + + for key, value in sub_dict.items(): + if type(value) is dict: + sub_dict_outputs = get_secret_config_entries( + config, [*config_idx, key], private_entries + ) + output = [*output, *sub_dict_outputs] + else: + if "NEBARI_SECRET_" in str(value): + output = [*output, [*config_idx, key]] + return output + + +def get_sub_config(conf, conf_idx): + sub_config = functools.reduce(dict.__getitem__, conf_idx, conf) + return sub_config + + +def set_sub_config(conf, conf_idx, value): + get_sub_config(conf, conf_idx[:-1])[conf_idx[-1]] = value + + +def set_nebari_secret(config, idx): + placeholder = get_sub_config(config, idx) + secret_var = get_nebari_secret(placeholder) + set_sub_config(config, idx, secret_var) + + +def get_nebari_secret(secret_var): + env_var = secret_var.lstrip("NEBARI_SECRET_") + val = os.environ.get(env_var) + if not val: + raise EnvironmentError( + f"Since '{secret_var}' was found in the" + " Nebari config, the environment variable" + f" '{env_var}' must be set." + ) + return val diff --git a/tests/tests_unit/conftest.py b/tests/tests_unit/conftest.py index ba588d1b1..72b5b18b6 100644 --- a/tests/tests_unit/conftest.py +++ b/tests/tests_unit/conftest.py @@ -1,18 +1,142 @@ +import typing from unittest.mock import Mock import pytest -from tests.tests_unit.utils import INIT_INPUTS, NEBARI_CONFIG_FN, PRESERVED_DIR +from _nebari.config import write_configuration +from _nebari.initialize import render_config +from _nebari.render import render_template +from _nebari.stages.bootstrap import CiEnum +from _nebari.stages.kubernetes_keycloak import AuthenticationEnum +from _nebari.stages.terraform_state import TerraformStateEnum +from nebari import schema +from nebari.plugins import nebari_plugin_manager -@pytest.fixture(params=INIT_INPUTS) -def setup_fixture(request, monkeypatch, tmp_path): - """This fixture helps simplify writing tests by: - - parametrizing the different cloud provider inputs in a single place - - creating a tmp directory (and file) for the `nebari-config.yaml` to be save to - - monkeypatching functions that call out to external APIs. - """ - render_config_inputs = request.param +@pytest.fixture(autouse=True) +def mock_all_cloud_methods(monkeypatch): + def _mock_kubernetes_versions( + k8s_versions: typing.List[str] = ["1.18", "1.19", "1.20"], + grab_latest_version=False, + ): + # template for all `kubernetes_versions` calls + # monkeypatched to avoid making outbound API calls in CI + m = Mock() + m.return_value = k8s_versions + if grab_latest_version: + m.return_value = k8s_versions[-1] + return m + + def _mock_return_value(return_value): + m = Mock() + m.return_value = return_value + return m + + def _mock_aws_availability_zones(region="us-west-2"): + m = Mock() + m.return_value = ["us-west-2a", "us-west-2b"] + return m + + MOCK_VALUES = { + # AWS + "_nebari.provider.cloud.amazon_web_services.kubernetes_versions": [ + "1.18", + "1.19", + "1.20", + ], + "_nebari.provider.cloud.amazon_web_services.check_credentials": None, + "_nebari.provider.cloud.amazon_web_services.regions": [ + "us-east-1", + "us-west-2", + ], + "_nebari.provider.cloud.amazon_web_services.zones": [ + "us-west-2a", + "us-west-2b", + ], + "_nebari.provider.cloud.amazon_web_services.instances": { + "m5.xlarge": "m5.xlarge", + "m5.2xlarge": "m5.2xlarge", + }, + # Azure + "_nebari.provider.cloud.azure_cloud.kubernetes_versions": [ + "1.18", + "1.19", + "1.20", + ], + "_nebari.provider.cloud.azure_cloud.check_credentials": None, + # Digital Ocean + "_nebari.provider.cloud.digital_ocean.kubernetes_versions": [ + "1.19.2-do.3", + "1.20.2-do.0", + "1.21.5-do.0", + ], + "_nebari.provider.cloud.digital_ocean.check_credentials": None, + "_nebari.provider.cloud.digital_ocean.regions": [ + {"name": "New York 3", "slug": "nyc3"}, + ], + "_nebari.provider.cloud.digital_ocean.instances": [ + {"name": "s-2vcpu-4gb", "slug": "s-2vcpu-4gb"}, + {"name": "g-2vcpu-8gb", "slug": "g-2vcpu-8gb"}, + {"name": "g-8vcpu-32gb", "slug": "g-8vcpu-32gb"}, + {"name": "g-4vcpu-16gb", "slug": "g-4vcpu-16gb"}, + ], + # Google Cloud + "_nebari.provider.cloud.google_cloud.kubernetes_versions": [ + "1.18", + "1.19", + "1.20", + ], + "_nebari.provider.cloud.google_cloud.check_credentials": None, + } + + for attribute_path, return_value in MOCK_VALUES.items(): + monkeypatch.setattr(attribute_path, _mock_return_value(return_value)) + + monkeypatch.setenv("PROJECT_ID", "pytest-project") + + +@pytest.fixture( + params=[ + # project, namespace, domain, cloud_provider, ci_provider, auth_provider + ( + "pytestdo", + "dev", + "do.nebari.dev", + schema.ProviderEnum.do, + CiEnum.github_actions, + AuthenticationEnum.password, + ), + ( + "pytestaws", + "dev", + "aws.nebari.dev", + schema.ProviderEnum.aws, + CiEnum.github_actions, + AuthenticationEnum.password, + ), + ( + "pytestgcp", + "dev", + "gcp.nebari.dev", + schema.ProviderEnum.gcp, + CiEnum.github_actions, + AuthenticationEnum.password, + ), + ( + "pytestazure", + "dev", + "azure.nebari.dev", + schema.ProviderEnum.azure, + CiEnum.github_actions, + AuthenticationEnum.password, + ), + ] +) +def nebari_config_options(request) -> schema.Main: + """This fixtures creates a set of nebari configurations for tests""" + DEFAULT_GH_REPO = "github.com/test/test" + DEFAULT_TERRAFORM_STATE = TerraformStateEnum.remote + ( project, namespace, @@ -20,48 +144,40 @@ def setup_fixture(request, monkeypatch, tmp_path): cloud_provider, ci_provider, auth_provider, - ) = render_config_inputs + ) = request.param - def _mock_kubernetes_versions(grab_latest_version=False): - # template for all `kubernetes_versions` calls - # monkeypatched to avoid making outbound API calls in CI - k8s_versions = ["1.18", "1.19", "1.20"] - m = Mock() - m.return_value = k8s_versions - if grab_latest_version: - m.return_value = k8s_versions[-1] - return m + return dict( + project_name=project, + namespace=namespace, + nebari_domain=domain, + cloud_provider=cloud_provider, + ci_provider=ci_provider, + auth_provider=auth_provider, + repository=DEFAULT_GH_REPO, + repository_auto_provision=False, + auth_auto_provision=False, + terraform_state=DEFAULT_TERRAFORM_STATE, + disable_prompt=True, + ) + + +@pytest.fixture +def nebari_config(nebari_config_options): + return nebari_plugin_manager.config_schema.parse_obj( + render_config(**nebari_config_options) + ) + + +@pytest.fixture +def nebari_stages(): + return nebari_plugin_manager.ordered_stages + + +@pytest.fixture +def nebari_render(nebari_config, nebari_stages, tmp_path): + NEBARI_CONFIG_FN = "nebari-config.yaml" - if cloud_provider == "aws": - monkeypatch.setattr( - "_nebari.utils.amazon_web_services.kubernetes_versions", - _mock_kubernetes_versions(), - ) - elif cloud_provider == "azure": - monkeypatch.setattr( - "_nebari.utils.azure_cloud.kubernetes_versions", - _mock_kubernetes_versions(), - ) - elif cloud_provider == "do": - monkeypatch.setattr( - "_nebari.utils.digital_ocean.kubernetes_versions", - _mock_kubernetes_versions(), - ) - elif cloud_provider == "gcp": - monkeypatch.setattr( - "_nebari.utils.google_cloud.kubernetes_versions", - _mock_kubernetes_versions(), - ) - - output_directory = tmp_path / f"{cloud_provider}_output_dir" - output_directory.mkdir() - nebari_config_loc = output_directory / NEBARI_CONFIG_FN - - # data that should NOT be deleted when `nebari render` is called - # see test_render.py::test_remove_existing_renders - preserved_directory = output_directory / PRESERVED_DIR - preserved_directory.mkdir() - preserved_filename = preserved_directory / "file.txt" - preserved_filename.write_text("This is a test...") - - yield (nebari_config_loc, render_config_inputs) + config_filename = tmp_path / NEBARI_CONFIG_FN + write_configuration(config_filename, nebari_config) + render_template(tmp_path, nebari_config, nebari_stages) + return tmp_path, config_filename diff --git a/tests/tests_unit/test_cli.py b/tests/tests_unit/test_cli.py index a45072d70..d8a4e423b 100644 --- a/tests/tests_unit/test_cli.py +++ b/tests/tests_unit/test_cli.py @@ -2,8 +2,8 @@ import pytest -from _nebari.schema import InitInputs -from _nebari.utils import load_yaml +from _nebari.subcommands.init import InitInputs +from nebari.plugins import nebari_plugin_manager PROJECT_NAME = "clitest" DOMAIN_NAME = "clitest.dev" @@ -13,7 +13,7 @@ "namespace, auth_provider, ci_provider, ssl_cert_email", ( [None, None, None, None], - ["prod", "github", "github-actions", "it@acme.org"], + ["prod", "password", "github-actions", "it@acme.org"], ), ) def test_nebari_init(tmp_path, namespace, auth_provider, ci_provider, ssl_cert_email): @@ -48,34 +48,20 @@ def test_nebari_init(tmp_path, namespace, auth_provider, ci_provider, ssl_cert_e subprocess.run(command, cwd=tmp_path, check=True) - config = load_yaml(tmp_path / "nebari-config.yaml") + config = nebari_plugin_manager.read_config(tmp_path / "nebari-config.yaml") - assert config.get("namespace") == namespace - assert ( - config.get("security", {}).get("authentication", {}).get("type").lower() - == auth_provider - ) - ci_cd = config.get("ci_cd", None) - if ci_cd: - assert ci_cd.get("type", {}) == ci_provider - else: - assert ci_cd == ci_provider - acme_email = config.get("certificate", None) - if acme_email: - assert acme_email.get("acme_email") == ssl_cert_email - else: - assert acme_email == ssl_cert_email - - -def test_python_invocation(): - def run(command): - return subprocess.run( - command, check=True, capture_output=True, text=True - ).stdout.strip() - - command = ["nebari", "--version"] + assert config.namespace == namespace + assert config.security.authentication.type.lower() == auth_provider + assert config.ci_cd.type == ci_provider + assert config.certificate.acme_email == ssl_cert_email - actual = run(["python", "-m", *command]) - expected = run(command) - assert actual == expected +@pytest.mark.parametrize( + "command", + ( + ["nebari", "--version"], + ["nebari", "info"], + ), +) +def test_nebari_commands_no_args(command): + subprocess.run(command, check=True, capture_output=True, text=True).stdout.strip() diff --git a/tests/tests_unit/test_init.py b/tests/tests_unit/test_init.py index a64d511fc..4ad980a23 100644 --- a/tests/tests_unit/test_init.py +++ b/tests/tests_unit/test_init.py @@ -1,42 +1,45 @@ import pytest -from .utils import render_config_partial +from _nebari.initialize import render_config +from _nebari.stages.bootstrap import CiEnum +from _nebari.stages.kubernetes_keycloak import AuthenticationEnum +from nebari.schema import ProviderEnum @pytest.mark.parametrize( - "k8s_version, expected", [(None, True), ("1.19", True), (1000, ValueError)] + "k8s_version, cloud_provider, expected", + [ + (None, ProviderEnum.aws, None), + ("1.19", ProviderEnum.aws, "1.19"), + # (1000, ProviderEnum.aws, ValueError), # TODO: fix this + ], ) -def test_init(setup_fixture, k8s_version, expected): - (nebari_config_loc, render_config_inputs) = setup_fixture - ( - project, - namespace, - domain, - cloud_provider, - ci_provider, - auth_provider, - ) = render_config_inputs - - # pass "unsupported" kubernetes version to `render_config` - # resulting in a `ValueError` +def test_render_config(mock_all_cloud_methods, k8s_version, cloud_provider, expected): if type(expected) == type and issubclass(expected, Exception): with pytest.raises(expected): - render_config_partial( - project_name=project, - namespace=namespace, - nebari_domain=domain, + config = render_config( + project_name="test", + namespace="dev", + nebari_domain="test.dev", cloud_provider=cloud_provider, - ci_provider=ci_provider, - auth_provider=auth_provider, + ci_provider=CiEnum.none, + auth_provider=AuthenticationEnum.password, kubernetes_version=k8s_version, ) + assert config else: - render_config_partial( - project_name=project, - namespace=namespace, - nebari_domain=domain, + config = render_config( + project_name="test", + namespace="dev", + nebari_domain="test.dev", cloud_provider=cloud_provider, - ci_provider=ci_provider, - auth_provider=auth_provider, + ci_provider=CiEnum.none, + auth_provider=AuthenticationEnum.password, kubernetes_version=k8s_version, ) + + assert ( + config.get("amazon_web_services", {}).get("kubernetes_version") == expected + ) + + assert config["project_name"] == "test" diff --git a/tests/tests_unit/test_links.py b/tests/tests_unit/test_links.py index 8df7ee1e6..a393391ce 100644 --- a/tests/tests_unit/test_links.py +++ b/tests/tests_unit/test_links.py @@ -1,7 +1,7 @@ import pytest import requests -from _nebari.utils import AWS_ENV_DOCS, AZURE_ENV_DOCS, DO_ENV_DOCS, GCP_ENV_DOCS +from _nebari.constants import AWS_ENV_DOCS, AZURE_ENV_DOCS, DO_ENV_DOCS, GCP_ENV_DOCS LINKS_TO_TEST = [ DO_ENV_DOCS, diff --git a/tests/tests_unit/test_render.py b/tests/tests_unit/test_render.py index 2ec7f407a..06b5e5a1f 100644 --- a/tests/tests_unit/test_render.py +++ b/tests/tests_unit/test_render.py @@ -1,119 +1,41 @@ import os -from pathlib import Path -import pytest -from ruamel.yaml import YAML +from _nebari.stages.bootstrap import CiEnum +from nebari import schema +from nebari.plugins import nebari_plugin_manager -from _nebari.render import render_template, set_env_vars_in_config -from .utils import PRESERVED_DIR, render_config_partial - - -@pytest.fixture -def write_nebari_config_to_file(setup_fixture): - nebari_config_loc, render_config_inputs = setup_fixture - ( - project, - namespace, - domain, - cloud_provider, - ci_provider, - auth_provider, - ) = render_config_inputs - - config = render_config_partial( - project_name=project, - namespace=namespace, - nebari_domain=domain, - cloud_provider=cloud_provider, - ci_provider=ci_provider, - auth_provider=auth_provider, - kubernetes_version=None, +def test_render_config(nebari_render): + output_directory, config_filename = nebari_render + config = nebari_plugin_manager.read_config(config_filename) + assert {"nebari-config.yaml", "stages", ".gitignore"} <= set( + os.listdir(output_directory) ) - - # write to nebari_config.yaml - yaml = YAML(typ="unsafe", pure=True) - yaml.dump(config, nebari_config_loc) - - render_template(nebari_config_loc.parent, nebari_config_loc) - - yield setup_fixture - - -def test_get_secret_config_entries(monkeypatch): - sec1 = "secret1" - sec2 = "nestedsecret1" - config_orig = { - "key1": "value1", - "key2": "NEBARI_SECRET_secret_val", - "key3": { - "nested_key1": "nested_value1", - "nested_key2": "NEBARI_SECRET_nested_secret_val", - }, - } - expected = { - "key1": "value1", - "key2": sec1, - "key3": { - "nested_key1": "nested_value1", - "nested_key2": sec2, - }, - } - - # should raise error if implied env var is not set - with pytest.raises(EnvironmentError): - config = config_orig.copy() - set_env_vars_in_config(config) - - monkeypatch.setenv("secret_val", sec1, prepend=False) - monkeypatch.setenv("nested_secret_val", sec2, prepend=False) - config = config_orig.copy() - set_env_vars_in_config(config) - assert config == expected - - -def test_render_template(write_nebari_config_to_file): - nebari_config_loc, render_config_inputs = write_nebari_config_to_file - ( - project, - namespace, - domain, - cloud_provider, - ci_provider, - auth_provider, - ) = render_config_inputs - - yaml = YAML() - nebari_config_json = yaml.load(nebari_config_loc.read_text()) - - assert nebari_config_json["project_name"] == project - assert nebari_config_json["namespace"] == namespace - assert nebari_config_json["domain"] == domain - assert nebari_config_json["provider"] == cloud_provider - - -def test_exists_after_render(write_nebari_config_to_file): - items_to_check = [ - ".gitignore", - "stages", - "nebari-config.yaml", - PRESERVED_DIR, - ] - - nebari_config_loc, _ = write_nebari_config_to_file - - yaml = YAML() - nebari_config_json = yaml.load(nebari_config_loc.read_text()) - - # list of files/dirs available after `nebari render` command - ls = os.listdir(Path(nebari_config_loc).parent.resolve()) - - cicd = nebari_config_json.get("ci_cd", {}).get("type", None) - - if cicd == "github-actions": - items_to_check.append(".github") - elif cicd == "gitlab-ci": - items_to_check.append(".gitlab-ci.yml") - - for i in items_to_check: - assert i in ls + assert { + "07-kubernetes-services", + "02-infrastructure", + "01-terraform-state", + "05-kubernetes-keycloak", + "08-nebari-tf-extensions", + "06-kubernetes-keycloak-configuration", + "04-kubernetes-ingress", + "03-kubernetes-initialize", + } == set(os.listdir(output_directory / "stages")) + + if config.provider == schema.ProviderEnum.do: + assert (output_directory / "stages" / "01-terraform-state/do").is_dir() + assert (output_directory / "stages" / "02-infrastructure/do").is_dir() + elif config.provider == schema.ProviderEnum.aws: + assert (output_directory / "stages" / "01-terraform-state/aws").is_dir() + assert (output_directory / "stages" / "02-infrastructure/aws").is_dir() + elif config.provider == schema.ProviderEnum.gcp: + assert (output_directory / "stages" / "01-terraform-state/gcp").is_dir() + assert (output_directory / "stages" / "02-infrastructure/gcp").is_dir() + elif config.provider == schema.ProviderEnum.azure: + assert (output_directory / "stages" / "01-terraform-state/azure").is_dir() + assert (output_directory / "stages" / "02-infrastructure/azure").is_dir() + + if config.ci_cd.type == CiEnum.github_actions: + assert (output_directory / ".github/workflows/").is_dir() + elif config.ci_cd.type == CiEnum.gitlab_ci: + assert (output_directory / ".gitlab-ci.yml").is_file() diff --git a/tests/tests_unit/test_schema.py b/tests/tests_unit/test_schema.py index d4d8cf878..2e733a302 100644 --- a/tests/tests_unit/test_schema.py +++ b/tests/tests_unit/test_schema.py @@ -1,27 +1,50 @@ -import _nebari.schema - -from .utils import render_config_partial - - -def test_schema(setup_fixture): - (nebari_config_loc, render_config_inputs) = setup_fixture - ( - project, - namespace, - domain, - cloud_provider, - ci_provider, - auth_provider, - ) = render_config_inputs - - config = render_config_partial( - project_name=project, - namespace=namespace, - nebari_domain=domain, - cloud_provider=cloud_provider, - ci_provider=ci_provider, - auth_provider=auth_provider, - kubernetes_version=None, - ) - - _nebari.schema.verify(config) +from nebari import schema +from nebari.plugins import nebari_plugin_manager + + +def test_minimal_schema(): + config = nebari_plugin_manager.config_schema(project_name="test") + assert config.project_name == "test" + assert config.storage.conda_store == "200Gi" + + +def test_minimal_schema_from_file(tmp_path): + filename = tmp_path / "nebari-config.yaml" + with filename.open("w") as f: + f.write("project_name: test\n") + + config = nebari_plugin_manager.read_config(filename) + assert config.project_name == "test" + assert config.storage.conda_store == "200Gi" + + +def test_minimal_schema_from_file_with_env(tmp_path, monkeypatch): + filename = tmp_path / "nebari-config.yaml" + with filename.open("w") as f: + f.write("project_name: test\n") + + monkeypatch.setenv("NEBARI_SECRET__project_name", "env") + monkeypatch.setenv("NEBARI_SECRET__storage__conda_store", "1000Gi") + + config = nebari_plugin_manager.read_config(filename) + assert config.project_name == "env" + assert config.storage.conda_store == "1000Gi" + + +def test_minimal_schema_from_file_without_env(tmp_path, monkeypatch): + filename = tmp_path / "nebari-config.yaml" + with filename.open("w") as f: + f.write("project_name: test\n") + + monkeypatch.setenv("NEBARI_SECRET__project_name", "env") + monkeypatch.setenv("NEBARI_SECRET__storage__conda_store", "1000Gi") + + config = nebari_plugin_manager.read_config(filename, read_environment=False) + assert config.project_name == "test" + assert config.storage.conda_store == "200Gi" + + +def test_render_schema(nebari_config): + assert isinstance(nebari_config, schema.Main) + assert nebari_config.project_name == f"pytest{nebari_config.provider.value}" + assert nebari_config.namespace == "dev" diff --git a/tests/tests_unit/test_upgrade.py b/tests/tests_unit/test_upgrade.py index f53d980f4..0946dcd99 100644 --- a/tests/tests_unit/test_upgrade.py +++ b/tests/tests_unit/test_upgrade.py @@ -2,8 +2,9 @@ import pytest -from _nebari.upgrade import do_upgrade, load_yaml, verify +from _nebari.upgrade import do_upgrade from _nebari.version import __version__, rounded_ver_parse +from nebari.plugins import nebari_plugin_manager @pytest.fixture @@ -69,32 +70,24 @@ def test_upgrade_4_0( return # Check the resulting YAML - config = load_yaml(tmp_qhub_config) + config = nebari_plugin_manager.read_config(tmp_qhub_config) - verify( - config - ) # Would raise an error if invalid by current Nebari version's standards - - assert len(config["security"]["keycloak"]["initial_root_password"]) == 16 - - assert "users" not in config["security"] - assert "groups" not in config["security"] + assert len(config.security.keycloak.initial_root_password) == 16 + assert not hasattr(config.security, "users") + assert not hasattr(config.security, "groups") __rounded_version__ = ".".join([str(c) for c in rounded_ver_parse(__version__)]) # Check image versions have been bumped up assert ( - config["default_images"]["jupyterhub"] + config.default_images.jupyterhub == f"quansight/nebari-jupyterhub:v{__rounded_version__}" ) assert ( - config["profiles"]["jupyterlab"][0]["kubespawner_override"]["image"] + config.profiles.jupyterlab[0].kubespawner_override.image == f"quansight/nebari-jupyterlab:v{__rounded_version__}" ) - - assert ( - config.get("security", {}).get("authentication", {}).get("type", "") != "custom" - ) + assert config.security.authentication.type != "custom" # Keycloak import users json assert ( From 5577d373f0a55b0f7a67b9479b3234e58c24216f Mon Sep 17 00:00:00 2001 From: iameskild Date: Wed, 9 Aug 2023 10:39:19 -0700 Subject: [PATCH 146/147] fix jhub_ssh test --- tests/tests_deployment/test_jupyterhub_ssh.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/tests_deployment/test_jupyterhub_ssh.py b/tests/tests_deployment/test_jupyterhub_ssh.py index f03079fb3..0a310f7fa 100644 --- a/tests/tests_deployment/test_jupyterhub_ssh.py +++ b/tests/tests_deployment/test_jupyterhub_ssh.py @@ -4,13 +4,10 @@ import paramiko import pytest +from tests_deployment import constants +from tests_deployment.utils import get_jupyterhub_token, monkeypatch_ssl_context -from tests.tests_deployment import constants -from tests.tests_deployment.utils import ( - escape_string, - get_jupyterhub_token, - monkeypatch_ssl_context, -) +from _nebari.utils import escape_string monkeypatch_ssl_context() From 16f6ba47d17da3801befe7b59e2d3def0fbe35f2 Mon Sep 17 00:00:00 2001 From: sblair-metrostar <70233904+sblair-metrostar@users.noreply.github.com> Date: Wed, 9 Aug 2023 15:28:02 -0500 Subject: [PATCH 147/147] Extension mechanism AWS testing (#1864) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Chris Ostrouchov Co-authored-by: eskild <42120229+iameskild@users.noreply.github.com> Co-authored-by: Scott Blair --- src/_nebari/stages/base.py | 4 ++++ src/_nebari/stages/infrastructure/__init__.py | 8 ++++--- .../stages/kubernetes_ingress/__init__.py | 22 ++++++++++++++++++- .../stages/kubernetes_services/__init__.py | 3 ++- .../dask-gateway/files/gateway_config.py | 3 +++ 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index d6e91ddbc..d15e67d21 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -68,8 +68,12 @@ def deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): deploy_config["state_imports"] = state_imports self.set_outputs(stage_outputs, terraform.deploy(**deploy_config)) + self.post_deploy(stage_outputs) yield + def post_deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): + pass + def check(self, stage_outputs: Dict[str, Dict[str, Any]]): pass diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 38312b9ab..70ad37153 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -416,7 +416,9 @@ class AmazonWebServicesProvider(schema.Base): def _validate_kubernetes_version(cls, values): amazon_web_services.check_credentials() - available_kubernetes_versions = amazon_web_services.kubernetes_versions() + available_kubernetes_versions = amazon_web_services.kubernetes_versions( + values["region"] + ) if values["kubernetes_version"] is None: values["kubernetes_version"] = available_kubernetes_versions[-1] elif values["kubernetes_version"] not in available_kubernetes_versions: @@ -426,10 +428,10 @@ def _validate_kubernetes_version(cls, values): return values @pydantic.validator("node_groups") - def _validate_node_group(cls, value): + def _validate_node_group(cls, value, values): amazon_web_services.check_credentials() - available_instances = amazon_web_services.instances() + available_instances = amazon_web_services.instances(values["region"]) for name, node_group in value.items(): if node_group.instance not in available_instances: raise ValueError( diff --git a/src/_nebari/stages/kubernetes_ingress/__init__.py b/src/_nebari/stages/kubernetes_ingress/__init__.py index 0697cd2b0..28e5679c6 100644 --- a/src/_nebari/stages/kubernetes_ingress/__init__.py +++ b/src/_nebari/stages/kubernetes_ingress/__init__.py @@ -1,4 +1,5 @@ import enum +import logging import socket import sys import time @@ -6,6 +7,7 @@ from typing import Any, Dict, List from _nebari import constants +from _nebari.provider.dns.cloudflare import update_record from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import ( NebariHelmProvider, @@ -15,6 +17,8 @@ from nebari import schema from nebari.hookspecs import NebariStage, hookimpl +logger = logging.getLogger(__name__) + # check and retry settings NUM_ATTEMPTS = 10 TIMEOUT = 10 # seconds @@ -75,7 +79,7 @@ def provision_ingress_dns( ) if not disable_checks: - checks.check_ingress_dns(stage_outputs, config, disable_prompt) + check_ingress_dns(stage_outputs, config, disable_prompt) def check_ingress_dns(stage_outputs, config, disable_prompt): @@ -149,6 +153,10 @@ class Certificate(schema.Base): acme_server: str = "https://acme-v02.api.letsencrypt.org/directory" +class DnsProvider(schema.Base): + provider: typing.Optional[str] + + class Ingress(schema.Base): terraform_overrides: typing.Dict = {} @@ -157,6 +165,7 @@ class InputSchema(schema.Base): domain: typing.Optional[str] certificate: Certificate = Certificate() ingress: Ingress = Ingress() + dns: DnsProvider = DnsProvider() class IngressEndpoint(schema.Base): @@ -224,6 +233,17 @@ def set_outputs( super().set_outputs(stage_outputs, outputs) + def post_deploy(self, stage_outputs: Dict[str, Dict[str, Any]]): + if self.config.dns and self.config.dns.provider: + provision_ingress_dns( + stage_outputs, + self.config, + dns_provider=self.config.dns.provider, + dns_auto_provision=True, + disable_prompt=True, + disable_checks=False, + ) + def check(self, stage_outputs: Dict[str, Dict[str, Any]]): def _attempt_tcp_connect( host, port, num_attempts=NUM_ATTEMPTS, timeout=TIMEOUT diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index d6e26a59c..087bac464 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -127,7 +127,8 @@ class DaskWorkerProfile(schema.Base): worker_cores: int worker_memory_limit: str worker_memory: str - image: typing.Optional[str] + worker_threads: int = 1 + image: str = f"quay.io/nebari/nebari-dask-worker:{set_docker_image_tag()}" class Config: extra = "allow" diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py index ae2532d55..ccca6ba39 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py @@ -48,6 +48,9 @@ def dask_gateway_config(path="/var/lib/dask-gateway/config.json"): c.KubeClusterConfig.worker_cores_limit = config["cluster"]["worker_cores_limit"] c.KubeClusterConfig.worker_memory = config["cluster"]["worker_memory"] c.KubeClusterConfig.worker_memory_limit = config["cluster"]["worker_memory_limit"] +c.KubeClusterConfig.worker_threads = config["cluster"].get( + "worker_threads", config["cluster"]["worker_cores"] +) c.KubeClusterConfig.worker_extra_container_config = config["cluster"][ "worker_extra_container_config" ]