K8s 1.25 + More Improvements

src/_nebari/stages/infrastructure/template/gcp/main.tf

@@ -34,4 +34,6 @@ module "kubernetes" {
   private_cluster_config            = var.private_cluster_config
   kubernetes_version                = var.kubernetes_version
   release_channel                   = var.release_channel
+  tags                              = var.tags
+  labels                            = var.labels
 }

src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/main.tf

@@ -57,6 +57,9 @@ resource "google_container_cluster" "main" {
     }
   }
 
+  cost_management_config {
+    enabled = true
+  }
 
   lifecycle {
     ignore_changes = [
@@ -96,7 +99,7 @@ resource "google_container_node_pool" "main" {
     metadata = {
       disable-legacy-endpoints = "true"
     }
-    labels = local.merged_node_groups[count.index].labels
+    labels = merge(local.merged_node_groups[count.index].labels, var.labels)
     dynamic "guest_accelerator" {
       for_each = local.merged_node_groups[count.index].guest_accelerators
 
@@ -105,5 +108,12 @@ resource "google_container_node_pool" "main" {
         count = guest_accelerator.value.count
       }
     }
+    tags = var.tags
+  }
+
+  lifecycle {
+    ignore_changes = [
+      node_config[0].taint
+    ]
   }
 }

src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/variables.tf

@@ -144,3 +144,15 @@ variable "private_cluster_config" {
   }))
   default = null
 }
+
+variable "tags" {
+  description = "Google Cloud Platform tags to assign to resources"
+  type        = list(string)
+  default     = []
+}
+
+variable "labels" {
+  description = "Google Cloud Platform labels to assign to resources"
+  type        = map(string)
+  default     = {}
+}

src/_nebari/stages/infrastructure/template/gcp/variables.tf

@@ -36,8 +36,16 @@ variable "kubeconfig_filename" {
 variable "tags" {
   description = "Google Cloud Platform tags to assign to resources"
   type        = list(string)
+  default     = []
 }
 
+variable "labels" {
+  description = "Google Cloud Platform labels to assign to resources"
+  type        = map(string)
+  default     = {}
+}
+
+
 variable "kubernetes_version" {
   description = "Kubernetes version for GKE cluster"
   type        = string

src/_nebari/stages/infrastructure/template/gcp/versions.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "4.8.0"
+      version = "4.72.0"
     }
   }
   required_version = ">= 1.0"

src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/gcp-nvidia-installer.tf

@@ -74,8 +74,9 @@ resource "kubernetes_daemonset" "gcp_nvidia_installer" {
           }
         }
         init_container {
-          image = "cos-nvidia-installer:fixed"
-          name  = "nvidia-driver-installer"
+          image   = "cos-nvidia-installer:fixed"
+          name    = "nvidia-driver-installer"
+          command = ["/cos-gpu-installer", "install", "--version=latest"]
           resources {
             requests = {
               cpu = 0.15

src/_nebari/stages/kubernetes_services/__init__.py

@@ -54,12 +54,6 @@ class Prefect(schema.Base):
     token: typing.Optional[str]
 
 
-class CDSDashboards(schema.Base):
-    enabled: bool = True
-    cds_hide_user_named_servers: bool = True
-    cds_hide_user_dashboard_servers: bool = False
-
-
 class DefaultImages(schema.Base):
     jupyterhub: str = f"quay.io/nebari/nebari-jupyterhub:{set_docker_image_tag()}"
     jupyterlab: str = f"quay.io/nebari/nebari-jupyterlab:{set_docker_image_tag()}"
@@ -246,7 +240,6 @@ class JupyterLab(schema.Base):
 
 class InputSchema(schema.Base):
     prefect: Prefect = Prefect()
-    cdsdashboards: CDSDashboards = CDSDashboards()
     default_images: DefaultImages = DefaultImages()
     storage: Storage = Storage()
     theme: Theme = Theme()
@@ -279,7 +272,6 @@ class InputSchema(schema.Base):
             channels=["conda-forge"],
             dependencies=[
                 "python=3.10",
-                "cdsdashboards-singleuser=0.6.3",
                 "cufflinks-py=0.17.3",
                 "dash=2.8.1",
                 "geopandas=0.12.2",
@@ -370,7 +362,6 @@ class CondaStoreInputVars(schema.Base):
 
 
 class JupyterhubInputVars(schema.Base):
-    cdsdashboards: Dict[str, Any]
     jupyterhub_theme: Dict[str, Any] = Field(alias="jupyterhub-theme")
     jupyterlab_image: ImageNameTag = Field(alias="jupyterlab-image")
     jupyterhub_overrides: List[str] = Field(alias="jupyterhub-overrides")
@@ -449,12 +440,6 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
         ]["keycloak-read-only-user-credentials"]["value"]
 
         conda_store_token_scopes = {
-            "cdsdashboards": {
-                "primary_namespace": "cdsdashboards",
-                "role_bindings": {
-                    "*/*": ["viewer"],
-                },
-            },
             "dask-gateway": {
                 "primary_namespace": "",
                 "role_bindings": {
@@ -508,7 +493,6 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
         )
 
         jupyterhub_vars = JupyterhubInputVars(
-            cdsdashboards=self.config.cdsdashboards.dict(),
             jupyterhub_theme=jupyterhub_theme.dict(),
             jupyterlab_image=_split_docker_image_name(
                 self.config.default_images.jupyterlab

src/_nebari/stages/kubernetes_services/template/jupyterhub.tf

@@ -1,12 +1,3 @@
-variable "cdsdashboards" {
-  description = "Enable CDS Dashboards"
-  type = object({
-    enabled                         = bool
-    cds_hide_user_named_servers     = bool
-    cds_hide_user_dashboard_servers = bool
-  })
-}
-
 variable "jupyterhub-theme" {
   description = "JupyterHub theme"
   type        = map(any)
@@ -105,7 +96,6 @@ module "jupyterhub" {
   conda-store-mount                                  = "/home/conda"
   conda-store-environments                           = var.conda-store-environments
   default-conda-store-namespace                      = var.conda-store-default-namespace
-  conda-store-cdsdashboard-token                     = module.kubernetes-conda-store-server.service-tokens.cdsdashboards
   conda-store-argo-workflows-jupyter-scheduler-token = module.kubernetes-conda-store-server.service-tokens.argo-workflows-jupyter-scheduler
   conda-store-service-name                           = module.kubernetes-conda-store-server.service_name
 
@@ -131,8 +121,6 @@ module "jupyterhub" {
   jupyterhub-image = var.jupyterhub-image
   jupyterlab-image = var.jupyterlab-image
 
-  cdsdashboards = var.cdsdashboards
-
   theme    = var.jupyterhub-theme
   profiles = var.jupyterlab-profiles
 

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/01-theme.py

@@ -6,11 +6,7 @@
 import z2jh  # noqa: E402
 
 jupyterhub_theme = z2jh.get_config("custom.theme")
-cdsdashboards = z2jh.get_config("custom.cdsdashboards")
 
 c.JupyterHub.template_vars = {
     **jupyterhub_theme,
-    "cdsdashboards_enabled": cdsdashboards["enabled"],
-    "cds_hide_user_named_servers": cdsdashboards["cds_hide_user_named_servers"],
-    "cds_hide_user_dashboard_servers": cdsdashboards["cds_hide_user_dashboard_servers"],
 }

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py

@@ -1,21 +1,12 @@
-import json
-import os
-from functools import partial
-
 import kubernetes.client.models
-import urllib3
-import yarl
 from tornado import gen
 
 kubernetes.client.models.V1EndpointPort = (
     kubernetes.client.models.CoreV1EndpointPort
 )  # noqa: E402
 
-import z2jh  # noqa: E402
 from kubespawner import KubeSpawner  # noqa: E402
 
-cdsdashboards = z2jh.get_config("custom.cdsdashboards")
-
 
 @gen.coroutine
 def get_username_hook(spawner):
@@ -31,73 +22,5 @@ def get_username_hook(spawner):
 
 c.Spawner.pre_spawn_hook = get_username_hook
 
-if cdsdashboards["enabled"]:
-    from cdsdashboards.app import CDS_TEMPLATE_PATHS
-    from cdsdashboards.builder.kubebuilder import KubeBuilder
-    from cdsdashboards.hubextension import cds_extra_handlers
-    from cdsdashboards.hubextension.spawners.variablekube import VariableKubeSpawner
-
-    c.JupyterHub.allow_named_servers = True
-    c.JupyterHub.extra_handlers.extend(cds_extra_handlers)
-    c.JupyterHub.template_paths.extend(CDS_TEMPLATE_PATHS)
-    c.JupyterHub.spawner_class = VariableKubeSpawner
-    c.CDSDashboardsConfig.builder_class = KubeBuilder
-    c.VariableMixin.default_presentation_cmd = [
-        "python3",
-        "-m",
-        "jhsingle_native_proxy.main",
-    ]
-    c.JupyterHub.default_url = "/hub/home"
-
-    # Force dashboard creator to select an instance size
-    c.CDSDashboardsConfig.spawn_default_options = False
-
-    def get_conda_store_environments(query_package: str = ""):
-        external_url = z2jh.get_config("custom.conda-store-service-name")
-        token = z2jh.get_config("custom.conda-store-cdsdashboards")
-        endpoint = "conda-store/api/v1/environment"
-
-        url = yarl.URL(f"http://{external_url}/{endpoint}/")
-
-        if query_package:
-            url = url % {"packages": query_package}
-
-        http = urllib3.PoolManager()
-        response = http.request(
-            "GET", str(url), headers={"Authorization": f"Bearer {token}"}
-        )
-
-        # parse response
-        j = json.loads(response.data.decode("UTF-8"))
-        return [
-            f"{env['namespace']['name']}-{env['name']}" for env in j.get("data", [])
-        ]
-
-    c.CDSDashboardsConfig.conda_envs = partial(
-        get_conda_store_environments, query_package="cdsdashboards-singleuser"
-    )
-
-    # TODO: make timeouts configurable
-    c.VariableMixin.proxy_ready_timeout = 300
-    c.VariableMixin.proxy_request_timeout = 300
-
-    home_username = f"/home/{os.getenv('PREFERRED_USERNAME')}"
-    c.CDSDashboardsConfig.extra_presentation_types = ["panel-serve"]
-    c.VariableMixin.extra_presentation_launchers = {
-        "panel-serve": {
-            "args": [
-                "python3",
-                "{presentation_path}",
-                "{--}port",
-                "{port}",
-                "{--}address",
-                "{origin_host}",
-            ],
-            "debug_args": [],
-            "env": {"PYTHONPATH": f"{home_username}/{{presentation_dirname}}"},
-        }
-    }
-
-else:
-    c.JupyterHub.allow_named_servers = False
-    c.JupyterHub.spawner_class = KubeSpawner
+c.JupyterHub.allow_named_servers = False
+c.JupyterHub.spawner_class = KubeSpawner

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf

@@ -17,7 +17,7 @@ resource "helm_release" "jupyterhub" {
 
   repository = "https://jupyterhub.github.io/helm-chart/"
   chart      = "jupyterhub"
-  version    = "1.2.0"
+  version    = "2.0.0"
 
   values = concat([
     file("${path.module}/values.yaml"),
@@ -28,15 +28,12 @@ resource "helm_release" "jupyterhub" {
         external-url                  = var.external-url
         theme                         = var.theme
         profiles                      = var.profiles
-        cdsdashboards                 = var.cdsdashboards
         home-pvc                      = var.home-pvc
         shared-pvc                    = var.shared-pvc
         conda-store-pvc               = var.conda-store-pvc
         conda-store-mount             = var.conda-store-mount
         default-conda-store-namespace = var.default-conda-store-namespace
         conda-store-service-name      = var.conda-store-service-name
-        conda-store-cdsdashboards     = var.conda-store-cdsdashboard-token
-        conda-store-jupyter-scheduler = var.conda-store-argo-workflows-jupyter-scheduler-token
         skel-mount = {
           name      = kubernetes_config_map.etc-skel.metadata.0.name
           namespace = kubernetes_config_map.etc-skel.metadata.0.namespace

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf

@@ -97,20 +97,6 @@ variable "profiles" {
   default     = []
 }
 
-variable "cdsdashboards" {
-  description = "Enable CDS Dashboards"
-  type = object({
-    enabled                         = bool
-    cds_hide_user_named_servers     = bool
-    cds_hide_user_dashboard_servers = bool
-  })
-  default = {
-    enabled                         = true
-    cds_hide_user_named_servers     = true
-    cds_hide_user_dashboard_servers = false
-  }
-}
-
 variable "conda-store-service-name" {
   description = "Name of conda-store service"
   type        = string
@@ -122,11 +108,6 @@ variable "conda-store-environments" {
   default     = {}
 }
 
-variable "conda-store-cdsdashboard-token" {
-  description = "Token for cdsdashboards to use conda-store"
-  type        = string
-  default     = ""
-}
 
 variable "conda-store-argo-workflows-jupyter-scheduler-token" {
   description = "Token for argo-workflows-jupyter-schedule to use conda-store"

tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310-customauth.yaml

@@ -45,10 +45,10 @@ theme:
     text_color: '#111111'
     h1_color: '#652e8e'
     h2_color: '#652e8e'
-cdsdashboards:
-  enabled: true
-  cds_hide_user_named_servers: true
-  cds_hide_user_dashboard_servers: false
+# cdsdashboards:
+#   enabled: true
+#   cds_hide_user_named_servers: true
+#   cds_hide_user_dashboard_servers: false
 terraform_state:
   type: remote
 namespace: dev

tests/tests_unit/qhub-config-yaml-files-for-upgrade/qhub-config-do-310.yaml

@@ -42,10 +42,10 @@ theme:
     text_color: '#111111'
     h1_color: '#652e8e'
     h2_color: '#652e8e'
-cdsdashboards:
-  enabled: true
-  cds_hide_user_named_servers: true
-  cds_hide_user_dashboard_servers: false
+# cdsdashboards:
+#  enabled: true
+#  cds_hide_user_named_servers: true
+#  cds_hide_user_dashboard_servers: false
 terraform_state:
   type: remote
 namespace: dev