feat: add customizer as a plugin - AALGO-215

.cursor/rules/nemo-platform.mdc

@@ -37,9 +37,8 @@ User-facing skills in `packages/nemo_platform_ext/src/nemo_platform_ext/skills/`
 - `nemo-try-agent`: test a deployed agent or chat with a model.
 - `nemo-status`: read-only health dashboard. Run this before assuming the platform is up.
 - `nemo-teardown`: guided shutdown with confirmation.
-- `nemo-fine-tune`: fine-tuning. Not yet available; the skill tells the user this honestly instead of letting you improvise.
 
-Plugin-owned skills under `plugins/*/src/*/skills/` handle their own routing for guardrails, evaluations, optimization, data designer, anonymizer, and auditor.
+Plugin-owned skills under `plugins/*/src/*/skills/` handle their own routing for customization, guardrails, evaluations, optimization, data designer, anonymizer, and auditor.
 
 ## Sandboxed environments
 

.github/trufflehog-exclude.txt

@@ -0,0 +1,3 @@
+# Newline-separated regexes for paths TruffleHog should skip.
+# uv.lock contains many sha256 hex digests that false-positive as SentryToken.
+uv\.lock

.github/workflows/ci.yaml

@@ -303,6 +303,7 @@ jobs:
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
           NMP_DATA_DIR: ${{ runner.temp }}/nemo-data
+          NMP_AUTH_ENABLED: "false"
           _TYPER_FORCE_DISABLE_TERMINAL: "1"
         run: |
           set -euo pipefail

.github/workflows/security.yaml

@@ -38,6 +38,7 @@ jobs:
         with:
           path: ./
           version: 3.95.3
+          extra_args: --exclude-paths=.github/trufflehog-exclude.txt
 
       - name: Scan Results Status
         if: ${{ github.event_name != 'merge_group' && steps.trufflehog.outcome == 'failure' }}

AGENTS.md

@@ -33,9 +33,8 @@ User-facing skills in `packages/nemo_platform_ext/src/nemo_platform_ext/skills/`
 - `nemo-try-agent`: test a deployed agent or chat with a model.
 - `nemo-status`: read-only health dashboard.
 - `nemo-teardown`: guided shutdown with confirmation.
-- `nemo-fine-tune`: fine-tuning. Not yet available; the skill tells the user it's not shipped instead of improvising with another training library.
 
-Plugin-owned skills under `plugins/*/src/*/skills/` handle guardrails, evaluations, optimization, data designer, anonymizer, and auditor.
+Plugin-owned skills under `plugins/*/src/*/skills/` handle their own routing for customization, guardrails, evaluations, optimization, data designer, anonymizer, and auditor.
 
 ### Working in a sandboxed environment
 

CLAUDE.md

@@ -33,9 +33,8 @@ User-facing skills in `packages/nemo_platform_ext/src/nemo_platform_ext/skills/`
 - `nemo-try-agent`: test a deployed agent or chat with a model.
 - `nemo-status`: read-only health dashboard. Run this before assuming the platform is up.
 - `nemo-teardown`: guided shutdown with confirmation.
-- `nemo-fine-tune`: fine-tuning. Not yet available; the skill tells the user it's not shipped instead of letting the agent improvise with another training library.
 
-Plugin-owned skills live under `plugins/*/src/*/skills/` and handle their own routing for guardrails, evaluations, optimization, data designer, anonymizer, and auditor.
+Plugin-owned skills live under `plugins/*/src/*/skills/` and handle their own routing for customization, guardrails, evaluations, optimization, data designer, anonymizer, and auditor.
 
 ### Working in a sandboxed coding-agent environment
 

Makefile

@@ -228,9 +228,11 @@ check-copyright-headers:
 lint: ## Run all linters (licenses, openapi, config docs, python style/types/sdk, vendored SDK, CLI, auth config)
 	bash tools/lint/lint-all.sh
 
+LINT_FIX_VERIFY ?= 0
+
 .PHONY: lint-fix
-lint-fix: ## Auto-fix lint issues in dependency order (openapi → stainless → style → cli → vendor → licenses → config-docs)
-	bash tools/lint/lint-fix.sh
+lint-fix: ## Auto-fix lint issues (set LINT_FIX_VERIFY=1 to also run CI lint checks)
+	LINT_FIX_VERIFY=$(LINT_FIX_VERIFY) bash tools/lint/lint-fix.sh
 
 .PHONY: vendor
 vendor: ## Vendor packages into the SDK and generate wrapper metadata

conftest.py

@@ -213,8 +213,9 @@ def pytest_collection_modifyitems(config, items):
         "e2e",
         "smoke_gpu_tasks",
         "smoke_customizer_tasks",
-        "smoke_customizer_automodel",
         "smoke_customizer_rl",
+        "smoke_nmp_automodel_tasks",
+        "smoke_nmp_automodel_training",
         "integration",
         "regression",
         "canary",
@@ -304,4 +305,4 @@ def _patched_reschedule(self, node):
     _original_reschedule(self, node)
 
 
-LoadScopeScheduling._reschedule = _patched_reschedule  # type: ignore[invalid-assignment]
+LoadScopeScheduling._reschedule = _patched_reschedule

docker-bake.hcl

@@ -0,0 +1,262 @@
+# NeMo Platform GPU image bake — run from Platform repo root (context = ".").
+#
+# Groups:
+#   nmp-automodel-gpu-wheels   causal-conv1d-wheel, mamba-ssm-wheel
+#   nmp-automodel              base, tasks, training, smoke-test targets
+#   nmp-unsloth                nmp-unsloth-training
+#
+# Automodel — inspect wheels (no build):
+#   docker buildx bake --print -f docker-bake.hcl nmp-automodel-gpu-wheels
+#
+# Automodel — build and push wheels:
+#   export WHEELS_REGISTRY=my-registry/nemo-platform-dev
+#   export WHEELS_TAG=$(git rev-parse --short HEAD)
+#   docker buildx bake -f docker-bake.hcl nmp-automodel-gpu-wheels --push
+#
+# Automodel — build runtime images:
+#   docker buildx bake -f docker-bake.hcl nmp-automodel-base-builder
+#
+# Unsloth — local build (--load):
+#   docker buildx bake -f docker-bake.hcl nmp-unsloth-training --load \
+#     --set "*.platform=linux/amd64"
+#
+# Unsloth — push to registry:
+#   export IMAGE_REGISTRY=my-registry/nemo-platform-dev
+#   export BAKE_TAG=$(git rev-parse --short HEAD)
+#   docker buildx bake -f docker-bake.hcl nmp-unsloth-training --push \
+#     --set "*.platform=linux/amd64"
+#
+# Published tags:
+#   ${IMAGE_REGISTRY}/nmp-automodel-{base,tasks,training}:${BAKE_TAG}
+#   ${IMAGE_REGISTRY}/nmp-unsloth-training:${BAKE_TAG}
+
+# ---------------------------------------------------------------------------
+# Shared / automodel variables
+# ---------------------------------------------------------------------------
+
+variable "IMAGE_REGISTRY" {
+  default = "my-registry/nemo-platform-dev"
+}
+
+variable "BASE_REGISTRY" {
+  default = "my-registry/nemo-platform-dev"
+}
+
+variable "WHEELS_REGISTRY" {
+  default = "my-registry/nemo-platform-dev"
+}
+
+variable "BAKE_TAG" {
+  default = "local"
+}
+
+variable "BASE_TAG_AUTOMODEL" {
+  default = "local"
+}
+
+variable "WHEELS_TAG" {
+  default = "3fd6986ff173b598446ffac06d9be3f84b482495"
+}
+
+variable "CUDA_VERSION" {
+  default = "12.8.1"
+}
+
+variable "MAMBA_22_COMMIT" {
+  default = "6b32be06d026e170b3fdaf3ae6282c5a6ff57b06"
+}
+
+variable "MAMBA_23_COMMIT" {
+  default = "v2.3.0"
+}
+
+variable "CAUSAL_CONV1D_VERSION" {
+  default = "v1.5.3"
+}
+
+# For local builds: --set "*.platform=linux/amd64"
+variable "BUILD_PLATFORMS" {
+  default = ["linux/amd64", "linux/arm64"]
+}
+
+# ---------------------------------------------------------------------------
+# Automodel helpers
+# ---------------------------------------------------------------------------
+
+function "wheel_tags" {
+  params = [name]
+  result = ["${WHEELS_REGISTRY}/${name}:${WHEELS_TAG}"]
+}
+
+function "get_causal_conv1d_wheel_image" {
+  params = []
+  result = "${WHEELS_REGISTRY}/causal-conv1d-wheel:${WHEELS_TAG}"
+}
+
+function "get_mamba_ssm_wheel_image" {
+  params = []
+  result = "${WHEELS_REGISTRY}/mamba-ssm-wheel:${WHEELS_TAG}"
+}
+
+# ---------------------------------------------------------------------------
+# Groups
+# ---------------------------------------------------------------------------
+
+group "nmp-automodel-gpu-wheels" {
+  targets = [
+    "causal-conv1d-wheel",
+    "mamba-ssm-wheel",
+  ]
+}
+
+group "nmp-automodel" {
+  targets = [
+    "nmp-automodel-base-builder",
+    "nmp-automodel-tasks-docker",
+    "nmp-automodel-training-docker",
+    "nmp-automodel-tasks-smoke-test",
+    "nmp-automodel-training-smoke-test",
+  ]
+}
+
+group "nmp-unsloth" {
+  targets = ["nmp-unsloth-training"]
+}
+
+# ---------------------------------------------------------------------------
+# Automodel — GPU wheels
+# ---------------------------------------------------------------------------
+
+target "causal-conv1d-wheel" {
+  target     = "causal-conv1d-wheel"
+  context    = "."
+  dockerfile = "services/automodel/docker/Dockerfile.mamba-wheel"
+  tags       = wheel_tags("causal-conv1d-wheel")
+  args = {
+    CUDA_VERSION          = CUDA_VERSION
+    CAUSAL_CONV1D_VERSION = CAUSAL_CONV1D_VERSION
+  }
+  platforms = BUILD_PLATFORMS
+}
+
+target "mamba-ssm-wheel" {
+  target     = "mamba-ssm-wheel"
+  context    = "."
+  dockerfile = "services/automodel/docker/Dockerfile.mamba-wheel"
+  tags       = wheel_tags("mamba-ssm-wheel")
+  args = {
+    CUDA_VERSION    = CUDA_VERSION
+    MAMBA_22_COMMIT = MAMBA_22_COMMIT
+    MAMBA_23_COMMIT = MAMBA_23_COMMIT
+  }
+  platforms = BUILD_PLATFORMS
+}
+
+target "automodel-platform-workspace" {
+  target     = "platform-workspace"
+  context    = "."
+  dockerfile = "services/automodel/docker/Dockerfile.platform-workspace"
+  platforms = BUILD_PLATFORMS
+}
+
+target "nmp-automodel-base-builder" {
+  target          = "nmp-automodel-base"
+  context         = "."
+  dockerfile      = "services/automodel/docker/Dockerfile.nmp-automodel-base"
+  no-cache-filter = ["automodel-clone"]
+  tags            = ["${IMAGE_REGISTRY}/nmp-automodel-base:${BAKE_TAG}"]
+  args = {
+    CAUSAL_CONV1D_WHEEL_IMAGE = get_causal_conv1d_wheel_image()
+    MAMBA_SSM_WHEEL_IMAGE     = get_mamba_ssm_wheel_image()
+  }
+  platforms = BUILD_PLATFORMS
+}
+
+target "nmp-automodel-tasks-docker" {
+  target     = "runtime"
+  context    = "."
+  dockerfile = "services/automodel/docker/Dockerfile.nmp-automodel-tasks"
+  contexts = {
+    platform-workspace = "target:automodel-platform-workspace"
+    nmp-automodel-base = "target:nmp-automodel-base-builder"
+  }
+  tags = ["${IMAGE_REGISTRY}/nmp-automodel-tasks:${BAKE_TAG}"]
+  args = {
+    BASE_REGISTRY      = BASE_REGISTRY
+    BASE_TAG_AUTOMODEL = BASE_TAG_AUTOMODEL
+  }
+  platforms = BUILD_PLATFORMS
+}
+
+target "nmp-automodel-training-docker" {
+  target     = "runtime"
+  context    = "."
+  dockerfile = "services/automodel/docker/Dockerfile.nmp-automodel-training"
+  contexts = {
+    platform-workspace = "target:automodel-platform-workspace"
+    nmp-automodel-base = "target:nmp-automodel-base-builder"
+  }
+  tags = ["${IMAGE_REGISTRY}/nmp-automodel-training:${BAKE_TAG}"]
+  args = {
+    BASE_REGISTRY      = BASE_REGISTRY
+    BASE_TAG_AUTOMODEL = BASE_TAG_AUTOMODEL
+  }
+  platforms = BUILD_PLATFORMS
+}
+
+target "nmp-automodel-tasks-smoke-test" {
+  target     = "smoke-test"
+  context    = "."
+  dockerfile = "services/automodel/docker/Dockerfile.nmp-automodel-tasks"
+  contexts = {
+    platform-workspace = "target:automodel-platform-workspace"
+    nmp-automodel-base = "target:nmp-automodel-base-builder"
+  }
+  args = {
+    BASE_REGISTRY      = BASE_REGISTRY
+    BASE_TAG_AUTOMODEL = BASE_TAG_AUTOMODEL
+    SMOKE_MARKER       = "smoke_nmp_automodel_tasks"
+  }
+  output    = ["type=cacheonly"]
+  platforms = BUILD_PLATFORMS
+}
+
+target "nmp-automodel-training-smoke-test" {
+  target     = "smoke-test"
+  context    = "."
+  dockerfile = "services/automodel/docker/Dockerfile.nmp-automodel-training"
+  contexts = {
+    platform-workspace = "target:automodel-platform-workspace"
+    nmp-automodel-base = "target:nmp-automodel-base-builder"
+  }
+  args = {
+    BASE_REGISTRY      = BASE_REGISTRY
+    BASE_TAG_AUTOMODEL = BASE_TAG_AUTOMODEL
+    SMOKE_MARKER       = "smoke_nmp_automodel_training"
+  }
+  output    = ["type=cacheonly"]
+  platforms = BUILD_PLATFORMS
+}
+
+# ---------------------------------------------------------------------------
+# Unsloth
+# ---------------------------------------------------------------------------
+
+target "unsloth-platform-workspace" {
+  context    = "."
+  dockerfile = "services/unsloth/docker/Dockerfile.platform-workspace"
+  target     = "platform-workspace"
+  output     = ["type=cacheonly"]
+  platforms = BUILD_PLATFORMS
+}
+
+target "nmp-unsloth-training" {
+  context    = "."
+  dockerfile = "services/unsloth/docker/Dockerfile.nmp-unsloth-training"
+  target     = "runtime"
+  contexts = {
+    platform-workspace = "target:unsloth-platform-workspace"
+  }
+  tags = ["${IMAGE_REGISTRY}/nmp-unsloth-training:${BAKE_TAG}"]
+  platforms = BUILD_PLATFORMS
+}

docs/agents/plugins.mdx

@@ -70,7 +70,7 @@ The skills that drive the agent lifecycle are:
 | `agents-optimize` | Selects a deployed agent, establishes an evaluation baseline, and suggests Switchyard routing, model swaps, skill optimization, prompt tuning, and new-model evaluations. See [Optimize Agents](/documentation/agents/optimize-agents). |
 | `agents-secure` | Selects a deployed agent, checks guardrail coverage, and scans recent telemetry for sensitive data. See [Secure Agents](/documentation/agents/secure-agents). |
 
-Plugin-owned skills cover guardrails, evaluations, optimization, data
+Plugin-owned skills cover customization, guardrails, evaluations, optimization, data
 designer, anonymizer, and auditor. They are installed with their plugin and
 appear in `nemo skills list` once the platform restarts.
 

docs/set-up/config-reference.mdx

@@ -412,6 +412,8 @@ jobs:
   reconcile_interval_seconds: 2
   # Interval in seconds for the job scheduler to run | default: 5
   schedule_interval_seconds: 5
+  # Register the subprocess/default execution profile. When unset, defaults to true for docker/none runtimes and false for kubernetes.
+  enable_subprocess_executor:
 ```
 
 ### `models`
@@ -771,8 +773,6 @@ customizer:
   port: 8000
   # Enable debug mode | default: False
   debug: false
-  # Override container image for Automodel training. If not set, uses platform defaults.
-  training_automodel_image:
   # Override container image for DPO training. If not set, uses platform defaults.
   training_rl_image:
   # default: '1'