refactor: remove hydra in favor of restate

apps/engineering/app/components/mermaid.tsx

@@ -0,0 +1,32 @@
+"use client";
+
+import mermaid from "mermaid";
+import { useTheme } from "next-themes";
+import { useEffect, useRef } from "react";
+
+export function Mermaid({ chart }: { chart: string }) {
+  const ref = useRef<HTMLDivElement>(null);
+  const { resolvedTheme } = useTheme();
+
+  useEffect(() => {
+    if (!ref.current) {
+      return;
+    }
+    mermaid.initialize({
+      startOnLoad: false,
+      theme: resolvedTheme === "dark" ? "dark" : "default",
+    });
+
+    void mermaid.run({
+      nodes: [ref.current],
+    });
+  }, [resolvedTheme]);
+
+  return (
+    <div className="my-4">
+      <div ref={ref} className="mermaid">
+        {chart}
+      </div>
+    </div>
+  );
+}

apps/engineering/app/docs/[[...slug]]/page.tsx

@@ -1,3 +1,4 @@
+import { Mermaid } from "@/app/components/mermaid";
 import { source } from "@/app/source";
 import { Banner } from "fumadocs-ui/components/banner";
 import { Tab, Tabs } from "fumadocs-ui/components/tabs";
@@ -39,6 +40,7 @@ export default async function Page(props: {
             Tabs,
             Tab,
             Banner,
+            Mermaid,
           }}
         />
       </DocsBody>

apps/engineering/content/docs/architecture/meta.json

@@ -3,5 +3,5 @@
   "description": "How does Unkey work",
   "icon": "Pencil",
   "root": false,
-  "pages": ["index", "services"]
+  "pages": ["index", "services", "workflows"]
 }

apps/engineering/content/docs/architecture/workflows/creating-services.mdx

@@ -0,0 +1,176 @@
+---
+title: Creating Workflow Services
+description: Guide to adding new Restate workflow services
+---
+
+# Creating Workflow Services
+
+## When to Use Workflows
+
+Use Restate workflows for operations that:
+
+- ✅ Are long-running (seconds to hours)
+- ✅ Need guaranteed completion despite failures
+- ✅ Involve multiple external systems
+- ✅ Must not run concurrently (use Virtual Objects)
+
+Don't use workflows for:
+
+- ❌ Simple CRUD operations
+- ❌ Synchronous API calls
+- ❌ Operations that complete in milliseconds
+
+## Steps
+
+### 1. Define the Proto
+
+Create `go/proto/hydra/v1/yourservice.proto`:
+
+```protobuf
+syntax = "proto3";
+package hydra.v1;
+
+import "dev/restate/sdk/go.proto";
+
+option go_package = "github.com/unkeyed/unkey/go/gen/proto/hydra/v1;hydrav1";
+
+service YourService {
+  option (dev.restate.sdk.go.service_type) = VIRTUAL_OBJECT;
+  rpc YourOperation(YourRequest) returns (YourResponse) {}
+}
+
+message YourRequest {
+  string key_field = 1;  // Used as Virtual Object key
+}
+
+message YourResponse {}
+```
+
+**Key decisions:**
+
+- Service type: `VIRTUAL_OBJECT` for serialization, `SERVICE` otherwise
+- Key field: The field used for Virtual Object key (e.g., `user_id`, `project_id`)
+
+### 2. Generate Code
+
+```bash
+cd go
+make generate
+```
+
+### 3. Implement the Service
+
+Create `go/apps/ctrl/workflows/yourservice/`:
+
+**service.go:**
+
+```go
+package yourservice
+
+import (
+    hydrav1 "github.com/unkeyed/unkey/go/gen/proto/hydra/v1"
+    "github.com/unkeyed/unkey/go/pkg/db"
+    "github.com/unkeyed/unkey/go/pkg/otel/logging"
+)
+
+type Service struct {
+    hydrav1.UnimplementedYourServiceServer
+    db     db.Database
+    logger logging.Logger
+}
+
+func New(cfg Config) *Service {
+    return &Service{db: cfg.DB, logger: cfg.Logger}
+}
+```
+
+**your_operation_handler.go:**
+
+```go
+func (s *Service) YourOperation(
+    ctx restate.ObjectContext,
+    req *hydrav1.YourRequest,
+) (*hydrav1.YourResponse, error) {
+    // Step 1: Durable step example
+    data, err := restate.Run(ctx, func(stepCtx restate.RunContext) (db.YourData, error) {
+        return db.Query.FindYourData(stepCtx, s.db.RO(), req.KeyField)
+    }, restate.WithName("fetch data"))
+    if err != nil {
+        return nil, err
+    }
+
+    // Step 2: Another durable step
+    _, err = restate.Run(ctx, func(stepCtx restate.RunContext) (restate.Void, error) {
+        // Your logic here
+        return restate.Void{}, nil
+    }, restate.WithName("process data"))
+
+    return &hydrav1.YourResponse{}, nil
+}
+```
+
+### 4. Register the Service
+
+Update `go/apps/ctrl/run.go`:
+
+```go
+import (
+    "github.com/unkeyed/unkey/go/apps/ctrl/workflows/yourservice"
+)
+
+func Run(ctx context.Context, cfg Config) error {
+    // ... existing setup ...
+
+    restateSrv.Bind(hydrav1.NewYourServiceServer(yourservice.New(yourservice.Config{
+        DB:     database,
+        Logger: logger,
+    })))
+}
+```
+
+### 5. Call the Service
+
+These are ugly, they're working on generating proper clients from the proto definitions
+https://github.com/restatedev/sdk-go/issues/103
+
+**Blocking call:**
+
+```go
+response, err := restateingress.Object[*hydrav1.YourRequest, *hydrav1.YourResponse](
+    restateClient,
+    "hydra.v1.YourService",
+    keyValue,
+    "YourOperation",
+).Request(ctx, request)
+```
+
+**Fire-and-forget:**
+
+```go
+invocation := restateingress.WorkflowSend[*hydrav1.YourRequest](
+    restateClient,
+    "hydra.v1.YourService",
+    keyValue,
+    "YourOperation",
+).Send(ctx, request)
+```
+
+## Best Practices
+
+1. **Small Steps**: Break operations into focused, single-purpose durable steps
+2. **Named Steps**: Always use `restate.WithName("step name")` for observability
+3. **Terminal Errors**: Use `restate.TerminalError(err, statusCode)` for validation failures
+4. **Virtual Object Keys**: Choose keys that represent the resource being protected
+
+## Examples
+
+See existing implementations:
+
+- **DeploymentService**: `go/apps/ctrl/workflows/deploy/`
+- **RoutingService**: `go/apps/ctrl/workflows/routing/`
+- **CertificateService**: `go/apps/ctrl/workflows/certificate/`
+
+## References
+
+- [Restate Go SDK Docs](https://docs.restate.dev/develop/go/)
+- [Restate Overview](./index)

apps/engineering/content/docs/architecture/workflows/deployment-service.mdx

@@ -0,0 +1,93 @@
+---
+title: Deployment Service
+description: Durable deployment workflow orchestration
+---
+
+# Deployment Service
+
+The `DeploymentService` orchestrates the complete deployment lifecycle, from building containers to assigning domains.
+
+**Location:** `go/apps/ctrl/workflows/deploy/`
+**Proto:** `go/proto/hydra/v1/deployment.proto`
+**Key:** `project_id`
+
+## Operations
+
+### Deploy
+
+<Mermaid chart={`flowchart TD
+    Start([Deploy Request]) --> FetchMeta[Fetch Metadata]
+    FetchMeta --> StatusBuilding[Status: Building]
+    StatusBuilding --> CreateKrane[Create in Krane]
+    CreateKrane --> PollStatus{Poll Until Ready}
+    PollStatus --> UpsertVMs[Upsert VM Records]
+    UpsertVMs --> PollStatus
+    PollStatus --> ScrapeAPI[Scrape OpenAPI Spec]
+    ScrapeAPI --> BuildDomains[Build Domain List]
+    BuildDomains --> AssignDomains[Call RoutingService]
+    AssignDomains --> StatusReady[Status: Ready]
+    StatusReady --> UpdateLive[Update Live Deployment]
+    UpdateLive --> End([Complete])
+
+    style AssignDomains fill:#e1f5fe
+    style StatusReady fill:#c8e6c9
+`} />
+
+Creates a new deployment:
+1. Fetch deployment, workspace, project, environment metadata
+2. Create deployment in Krane
+3. Poll until instances are running
+4. Scrape OpenAPI spec
+5. Call RoutingService to assign domains atomically
+6. Update project's live deployment ID
+
+Implementation: `go/apps/ctrl/workflows/deploy/deploy_handler.go`
+
+### Rollback
+
+<Mermaid chart={`flowchart TD
+    Start([Rollback Request]) --> Validate[Validate Deployments]
+    Validate --> CheckVMs[Check Target VMs]
+    CheckVMs --> FindDomains[Find Sticky Domains]
+    FindDomains --> SwitchDomains[Call RoutingService]
+    SwitchDomains --> UpdateProject[Update Live Deployment]
+    UpdateProject --> End([Success])
+
+    style SwitchDomains fill:#e1f5fe
+    style UpdateProject fill:#c8e6c9
+`} />
+
+Rolls back to a previous deployment:
+1. Validate source/target deployments
+2. Find sticky domains (live + environment level)
+3. Call RoutingService to switch domains atomically
+4. Update project metadata
+
+Implementation: `go/apps/ctrl/workflows/deploy/rollback_handler.go`
+
+### Promote
+
+<Mermaid chart={`flowchart TD
+    Start([Promote Request]) --> Validate[Validate Deployment]
+    Validate --> FindDomains[Find All Domains]
+    FindDomains --> SwitchDomains[Call RoutingService]
+    SwitchDomains --> ClearFlag[Clear Rolled Back Flag]
+    ClearFlag --> End([Success])
+
+    style SwitchDomains fill:#e1f5fe
+`} />
+
+Promotes a deployment to live, removing rolled-back state:
+1. Validate deployment is ready
+2. Find all project domains
+3. Call RoutingService to reassign domains
+4. Clear rolled_back flag
+
+Implementation: `go/apps/ctrl/workflows/deploy/promote_handler.go`
+
+## Why RoutingService?
+
+All domain/gateway operations are delegated to `RoutingService` to:
+- Ensure atomic updates (gateway configs → domains)
+- Serialize domain operations per project
+- Provide rollback capabilities for failed routing changes