fix(cassandra): publish required service schemas

migrations/cassandra/keyspaces/nvcf_autoscaler/01_init_keyspace.up.sql

@@ -0,0 +1 @@
+CREATE KEYSPACE IF NOT EXISTS nvcf_autoscaler WITH replication = {'class': 'NetworkTopologyStrategy', 'ncp': '${REPLICA_COUNT}' }  AND durable_writes = true;

migrations/cassandra/keyspaces/nvcf_autoscaler/02_init_roles.up.sql

@@ -0,0 +1,8 @@
+CREATE ROLE IF NOT EXISTS nvcf_autoscaler_app_access;
+GRANT SELECT, MODIFY on keyspace nvcf_autoscaler to nvcf_autoscaler_app_access;
+GRANT SELECT on keyspace system to nvcf_autoscaler_app_access;
+
+CREATE ROLE IF NOT EXISTS nvcf_autoscaler_app_v0 with login = true and password = '${SERVICE_ROLE_PASSWORD}';
+
+INSERT INTO system_auth.role_members (role, member) VALUES ('nvcf_autoscaler_app_access', 'nvcf_autoscaler_app_v0');
+UPDATE system_auth.roles SET member_of = member_of + {'nvcf_autoscaler_app_access'} where role = 'nvcf_autoscaler_app_v0';

migrations/cassandra/keyspaces/nvcf_autoscaler/03_init_tables.up.sql

@@ -0,0 +1,76 @@
+-- Canonical schema for nvcf_autoscaler keyspace.
+-- Pinned to the upstream nvcf_autoscaler schema at migration version 9.
+-- account_id (TEXT) consolidates upstream nca_id_string from migrations 02-09. nca_id UUID dropped (NCA IDs are not UUIDs and the column was unused).
+
+-- ============================================================
+-- Tables
+-- ============================================================
+
+-- Functions invoked recently. Rows expire via TTL to track a rolling window.
+CREATE TABLE IF NOT EXISTS nvcf_autoscaler.recently_invoked_functions (
+    function_id         UUID,
+    function_version_id UUID,
+    last_updated_at     TIMESTAMP,
+    account_id          TEXT,
+    PRIMARY KEY ((function_id, function_version_id))
+) WITH default_time_to_live = 600
+  AND compaction = {'class': 'UnifiedCompactionStrategy', 'scaling_parameters': 'T4', 'target_sstable_size': '50MiB', 'base_shard_count': '4', 'expired_sstable_check_frequency_seconds': '300'}
+  AND read_repair = 'NONE';
+
+-- Historical scaling decisions for recently invoked functions, clustered by time.
+CREATE TABLE IF NOT EXISTS nvcf_autoscaler.recently_invoked_functions_history (
+    function_id                           UUID,
+    function_version_id                   UUID,
+    last_updated_at                       TIMESTAMP,
+    account_id                            TEXT STATIC,
+    num_workers                           INT STATIC,
+    last_predicted_desired_instance_count INT,
+    last_predicted_error_code             TEXT,
+    PRIMARY KEY ((function_id, function_version_id), last_updated_at)
+) WITH CLUSTERING ORDER BY (last_updated_at DESC)
+  AND default_time_to_live = 172800
+  AND compaction = {'class': 'UnifiedCompactionStrategy', 'scaling_parameters': 'T4', 'target_sstable_size': '50MiB', 'base_shard_count': '4', 'expired_sstable_check_frequency_seconds': '300'}
+  AND read_repair = 'NONE';
+
+-- Functions with running workers but no recent invocations.
+CREATE TABLE IF NOT EXISTS nvcf_autoscaler.running_functions_without_invocations (
+    function_id         UUID,
+    function_version_id UUID,
+    last_updated_at     TIMESTAMP,
+    account_id          TEXT,
+    PRIMARY KEY ((function_id, function_version_id))
+) WITH default_time_to_live = 600
+  AND compaction = {'class': 'UnifiedCompactionStrategy', 'scaling_parameters': 'T4', 'target_sstable_size': '50MiB', 'base_shard_count': '4', 'expired_sstable_check_frequency_seconds': '300'}
+  AND read_repair = 'NONE';
+
+-- Historical scaling decisions for running functions without invocations, clustered by time.
+CREATE TABLE IF NOT EXISTS nvcf_autoscaler.running_functions_without_invocations_history (
+    function_id                           UUID,
+    function_version_id                   UUID,
+    last_updated_at                       TIMESTAMP,
+    account_id                            TEXT STATIC,
+    num_workers                           INT STATIC,
+    last_predicted_desired_instance_count INT,
+    last_predicted_error_code             TEXT,
+    PRIMARY KEY ((function_id, function_version_id), last_updated_at)
+) WITH CLUSTERING ORDER BY (last_updated_at DESC)
+  AND default_time_to_live = 172800
+  AND compaction = {'class': 'UnifiedCompactionStrategy', 'scaling_parameters': 'T4', 'target_sstable_size': '50MiB', 'base_shard_count': '4', 'expired_sstable_check_frequency_seconds': '300'}
+  AND read_repair = 'NONE';
+
+-- Distributed lock table for autoscaler leader coordination.
+CREATE TABLE IF NOT EXISTS nvcf_autoscaler.locks (
+    lock_name   TEXT PRIMARY KEY,
+    node_id     TEXT,
+    acquired_at TIMESTAMP
+) WITH default_time_to_live = 3600
+  AND compaction = {'class': 'UnifiedCompactionStrategy', 'scaling_parameters': 'T4', 'target_sstable_size': '50MiB', 'base_shard_count': '4', 'expired_sstable_check_frequency_seconds': '300'}
+  AND read_repair = 'NONE';
+
+-- Liveness registry of autoscaler nodes. Rows expire quickly to reflect health.
+CREATE TABLE IF NOT EXISTS nvcf_autoscaler.healthy_nodes (
+    node_id         TEXT PRIMARY KEY,
+    last_updated_at TIMESTAMP
+) WITH default_time_to_live = 180
+  AND compaction = {'class': 'UnifiedCompactionStrategy', 'scaling_parameters': 'T4', 'target_sstable_size': '50MiB', 'base_shard_count': '4', 'expired_sstable_check_frequency_seconds': '300'}
+  AND read_repair = 'NONE';

migrations/cassandra/keyspaces/nvct_api/01_init_keyspace.up.sql

@@ -0,0 +1 @@
+CREATE KEYSPACE IF NOT EXISTS nvct_api WITH replication = {'class': 'NetworkTopologyStrategy', 'ncp': '${REPLICA_COUNT}' }  AND durable_writes = true;

migrations/cassandra/keyspaces/nvct_api/02_init_roles.up.sql

@@ -0,0 +1,8 @@
+CREATE ROLE IF NOT EXISTS nvct_api_app_access;
+GRANT SELECT, MODIFY on keyspace nvct_api to nvct_api_app_access;
+GRANT SELECT on keyspace system to nvct_api_app_access;
+
+CREATE ROLE IF NOT EXISTS nvct_api_app_v0 with login = true and password = '${SERVICE_ROLE_PASSWORD}';
+
+INSERT INTO system_auth.role_members (role, member) VALUES ('nvct_api_app_access', 'nvct_api_app_v0');
+UPDATE system_auth.roles SET member_of = member_of + {'nvct_api_app_access'} where role = 'nvct_api_app_v0';

migrations/cassandra/keyspaces/nvct_api/03_init_tables.up.sql

@@ -0,0 +1,122 @@
+-- Canonical schema for nvct_api keyspace.
+-- Reference: nvcf/nvct-api v1.5.2 local_env/cassandra/schema/0001_initial_schema.cql
+
+-- ============================================================
+-- User-Defined Types
+-- ============================================================
+
+CREATE TYPE IF NOT EXISTS nvct_api.model_udt (
+    name    TEXT,
+    version TEXT,
+    url     TEXT
+);
+
+CREATE TYPE IF NOT EXISTS nvct_api.resource_udt (
+    name    TEXT,
+    version TEXT,
+    url     TEXT
+);
+
+CREATE TYPE IF NOT EXISTS nvct_api.gpu_spec_udt (
+    instance_type           TEXT,
+    gpu                     TEXT,
+    backend                 TEXT,
+    configuration           TEXT,
+    clusters                FROZEN<SET<TEXT>>,
+    regions                 FROZEN<SET<TEXT>>,
+    attributes              FROZEN<SET<TEXT>>,
+    max_request_concurrency INT,
+    helm_validation_policy  TEXT
+);
+
+CREATE TYPE IF NOT EXISTS nvct_api.health_udt (
+    sis_request_id UUID,
+    gpu            TEXT,
+    backend        TEXT,
+    instance_type  TEXT,
+    error          TEXT
+);
+
+CREATE TYPE IF NOT EXISTS nvct_api.telemetries_udt (
+    logs_telemetry_id    UUID,
+    metrics_telemetry_id UUID,
+    traces_telemetry_id  UUID
+);
+
+-- ============================================================
+-- Tables
+-- ============================================================
+
+-- Primary task store. Each row is a unique task instance.
+CREATE TABLE IF NOT EXISTS nvct_api.tasks_v2 (
+    nca_id                         TEXT,
+    task_id                        UUID,
+    name                           TEXT,
+    description                    TEXT,
+    tags                           FROZEN<SET<TEXT>>,
+    container_image                TEXT,
+    container_args                 TEXT,
+    container_environment          TEXT,
+    models                         FROZEN<SET<model_udt>>,
+    resources                      FROZEN<SET<resource_udt>>,
+    gpu_spec                       gpu_spec_udt,
+    max_runtime_duration           DURATION,
+    max_queued_duration            DURATION,
+    terminal_grace_period_duration DURATION,
+    result_handling_strategy       TEXT,
+    helm_chart                     TEXT,
+    results_location               TEXT,
+    status                         TEXT,
+    telemetries                    FROZEN<telemetries_udt>,
+    health_info                    FROZEN<health_udt>,
+    percent_complete               INT,
+    last_updated_at                TIMESTAMP,
+    last_heartbeat_at              TIMESTAMP,
+    created_at                     TIMESTAMP,
+    has_secrets                    BOOLEAN,
+    PRIMARY KEY ((task_id))
+);
+
+CREATE CUSTOM INDEX IF NOT EXISTS tasks_v2_by_nca_id_sai_idx
+    ON nvct_api.tasks_v2 (nca_id) USING 'StorageAttachedIndex';
+
+-- Tracks active SIS requests per task.
+CREATE TABLE IF NOT EXISTS nvct_api.sis_requests_by_task (
+    task_id            UUID,
+    sis_request_id     UUID,
+    gpu_spec           gpu_spec_udt,
+    total_request_size INT,
+    created_at         TIMESTAMP,
+    PRIMARY KEY ((task_id), sis_request_id)
+);
+
+-- Events emitted during task lifecycle, keyed by task.
+CREATE TABLE IF NOT EXISTS nvct_api.events_by_task (
+    task_id    UUID,
+    event_id   UUID,
+    nca_id     TEXT,
+    message    TEXT,
+    created_at TIMESTAMP,
+    PRIMARY KEY ((task_id), event_id)
+);
+
+-- Results produced by a task, keyed by task.
+CREATE TABLE IF NOT EXISTS nvct_api.results_by_task (
+    task_id    UUID,
+    result_id  UUID,
+    nca_id     TEXT,
+    name       TEXT,
+    metadata   TEXT,
+    created_at TIMESTAMP,
+    PRIMARY KEY ((task_id), result_id)
+);
+
+-- Distributed lock table for scheduled background tasks.
+-- Primary key must be 'name'.
+CREATE TABLE IF NOT EXISTS nvct_api.lock (
+    name      TEXT,
+    lockuntil TIMESTAMP,
+    lockedat  TIMESTAMP,
+    lockedby  TEXT,
+    PRIMARY KEY ((name))
+);

migrations/cassandra/tests/test-execute-sqls.sh

@@ -16,6 +16,27 @@ fail()
   status=1
 }
 
+schema_inventory=$(
+  # shellcheck disable=SC2016
+  sed -n 's/^| `\([^`]*\)`[[:space:]]*| \[[^]]*\].*$/\1/p' \
+    "${keyspaces}/README.md"
+)
+if [ -z "${schema_inventory}" ]; then
+  fail "schema inventory is empty or malformed"
+fi
+for keyspace_name in ${schema_inventory}; do
+  for migration_name in \
+    01_init_keyspace.up.sql \
+    02_init_roles.up.sql \
+    03_init_tables.up.sql
+  do
+    migration="${keyspaces}/${keyspace_name}/${migration_name}"
+    if [ ! -f "${migration}" ]; then
+      fail "schema inventory entry ${keyspace_name} is missing ${migration_name}"
+    fi
+  done
+done
+
 if grep -R -n -F 'envOrDefault "REPLICA_COUNT"' "${keyspaces}"; then
   fail "keyspace migrations contain templates unsupported by stock golang-migrate"
 fi