diff --git a/MIGRATION_V1_TO_V2.md b/MIGRATION_V1_TO_V2.md deleted file mode 100644 index 835bbf68b5a..00000000000 --- a/MIGRATION_V1_TO_V2.md +++ /dev/null @@ -1,405 +0,0 @@ -# Unkey API v1 to v2 Migration Guide - -This guide will help you migrate from Unkey's v1 API to the new v2 API. The v2 API introduces significant improvements in structure, consistency, and functionality. - -## Overview of Changes - -### Major Structural Changes -- **Consistent Response Format**: All v2 responses follow a standardized format with `meta` and `data` fields -- **POST-based Operations**: Most operations now use POST requests instead of GET for better security and flexibility -- **Enhanced Error Handling**: More detailed error responses with validation details -- **New Resource Management**: Added support for identities, permissions, roles, and ratelimit overrides - -## URL Structure Changes - -### v1 → v2 Path Mapping - -| v1 Path | v2 Path | Method Change | -|---------|---------|---------------| -| `GET /v1/keys.getKey` | `POST /v2/keys.getKey` | GET → POST | -| `POST /v1/keys.createKey` | `POST /v2/keys.createKey` | No change | -| `POST /v1/keys.deleteKey` | `POST /v2/keys.deleteKey` | No change | -| `POST /v1/keys.updateKey` | `POST /v2/keys.updateKey` | No change | -| `POST /v1/keys.verifyKey` | `POST /v2/keys.verifyKey` | No change | -| `POST /v1/keys.whoami` | `POST /v2/keys.whoami` | No change | -| `GET /v1/apis.getApi` | `POST /v2/apis.getApi` | GET → POST | -| `POST /v1/apis.createApi` | `POST /v2/apis.createApi` | No change | -| `GET /v1/apis.listKeys` | `POST /v2/apis.listKeys` | GET → POST | - -## Response Format Changes - -### v1 Response Format -```json -{ - "keyId": "key_123", - "valid": true, - "name": "Customer X" -} -``` - -### v2 Response Format -```json -{ - "meta": { - "requestId": "req_1234" - }, - "data": { - "keyId": "key_123", - "valid": true, - "name": "Customer X" - } -} -``` - -## Endpoint-Specific Migration - -### 1. Key Operations - -#### Get Key -**v1:** -```http -GET /v1/keys.getKey?keyId=key_123&decrypt=false -Authorization: Bearer -``` - -**v2:** -```http -POST /v2/keys.getKey -Authorization: Bearer -Content-Type: application/json - -{ - "keyId": "key_123", - "decrypt": false -} -``` - -#### Verify Key -**v1:** -```json -{ - "apiId": "api_123", - "key": "sk_123" -} -``` - -**v2:** -```json -{ - "apiId": "api_123", - "key": "sk_123", - "permissions": { - "type": "and", - "permissions": ["read", "write"] - }, - "ratelimits": [ - { - "name": "requests", - "cost": 1 - } - ] -} -``` - -### 2. API Operations - -#### Get API -**v1:** -```http -GET /v1/apis.getApi?apiId=api_123 -Authorization: Bearer -``` - -**v2:** -```http -POST /v2/apis.getApi -Authorization: Bearer -Content-Type: application/json - -{ - "apiId": "api_123" -} -``` - -#### List Keys -**v1:** -```http -GET /v1/apis.listKeys?apiId=api_123&limit=100 -Authorization: Bearer -``` - -**v2:** -```http -POST /v2/apis.listKeys -Authorization: Bearer -Content-Type: application/json - -{ - "apiId": "api_123", - "limit": 100, - "cursor": "optional_cursor_for_pagination" -} -``` - -### 3. New Features in v2 - -#### Permissions and Roles Management -v2 introduces comprehensive RBAC support: - -```json -POST /v2/keys.addPermissions -{ - "keyId": "key_123", - "permissions": [ - { - "slug": "read_users", - "create": true - }, - { - "id": "perm_456" - } - ] -} -``` - -#### Identity Management -New identity system for better user association: - -```json -POST /v2/identities.createIdentity -{ - "externalId": "user_123", - "meta": { - "name": "John Doe", - "email": "john@example.com" - }, - "ratelimits": [ - { - "name": "requests", - "limit": 100, - "duration": 60000 - } - ] -} -``` - -#### Ratelimit Overrides -Dynamic ratelimit management: - -```json -POST /v2/ratelimit.setOverride -{ - "namespaceName": "api_requests", - "identifier": "user_123", - "limit": 1000, - "duration": 3600000 -} -``` - -## Field Changes and Deprecations - -### Deprecated Fields -- `ownerId` → Use `externalId` instead -- v1 ratelimit structure → Use new v2 ratelimit array format - -### New Required Fields -- All responses now include `meta.requestId` for debugging -- Enhanced validation with detailed error responses - -### Credits System Enhancement -**v1:** -```json -{ - "remaining": 100, - "refill": { - "interval": "monthly", - "amount": 100 - } -} -``` - -**v2:** -```json -{ - "remaining": 100, - "refill": { - "interval": "monthly", - "amount": 100, - "refillDay": 1, - "lastRefillAt": 1640995200000 - } -} -``` - -## Error Handling Changes - -### v1 Error Format -```json -{ - "error": { - "code": "BAD_REQUEST", - "message": "Invalid request", - "requestId": "req_123" - } -} -``` - -### v2 Error Format -```json -{ - "meta": { - "requestId": "req_123" - }, - "error": { - "title": "Bad Request", - "detail": "Validation failed", - "status": 400, - "type": "https://unkey.dev/docs/api-reference/errors/400", - "errors": [ - { - "location": "body.keyId", - "message": "Required field missing", - "fix": "Provide a valid keyId" - } - ] - } -} -``` - -## Migration Steps - -### 1. Update Base URLs -- No change needed - same base URL `https://api.unkey.dev` - -### 2. Update Request Methods -- Change GET requests to POST for: `getKey`, `getApi`, `listKeys` -- Move query parameters to request body - -### 3. Update Response Handling -```javascript -// v1 response handling -const response = await fetch('/v1/keys.getKey?keyId=key_123'); -const keyData = await response.json(); -console.log(keyData.name); // Direct access - -// v2 response handling -const response = await fetch('/v2/keys.getKey', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ keyId: 'key_123' }) -}); -const result = await response.json(); -console.log(result.data.name); // Access via data field -console.log(result.meta.requestId); // Access request ID for debugging -``` - -### 4. Update Error Handling -```javascript -// v1 error handling -if (!response.ok) { - const error = await response.json(); - console.error(error.error.message); -} - -// v2 error handling -if (!response.ok) { - const error = await response.json(); - console.error(error.error.title); - if (error.error.errors) { - error.error.errors.forEach(err => { - console.error(`${err.location}: ${err.message}`); - }); - } -} -``` - -### 5. Leverage New Features -- Implement RBAC using the new permissions and roles system -- Use identities for better user management -- Set up ratelimit overrides for dynamic rate limiting - -## Migration Checklist - -- [ ] Update request methods from GET to POST where applicable -- [ ] Move query parameters to request body -- [ ] Update response handling to access `data` field -- [ ] Update error handling for new error format -- [ ] Test all existing functionality with v2 endpoints -- [ ] Implement new v2 features (RBAC, identities, etc.) -- [ ] Update client libraries/SDKs -- [ ] Monitor `meta.requestId` for debugging - -## New v2 Endpoints - -### Identity Management -- `POST /v2/identities.createIdentity` -- `POST /v2/identities.getIdentity` -- `POST /v2/identities.listIdentities` -- `POST /v2/identities.updateIdentity` -- `POST /v2/identities.deleteIdentity` - -### Permissions Management -- `POST /v2/permissions.createPermission` -- `POST /v2/permissions.getPermission` -- `POST /v2/permissions.listPermissions` -- `POST /v2/permissions.deletePermission` - -### Roles Management -- `POST /v2/permissions.createRole` -- `POST /v2/permissions.getRole` -- `POST /v2/permissions.listRoles` -- `POST /v2/permissions.deleteRole` - -### Key Permissions & Roles -- `POST /v2/keys.addPermissions` -- `POST /v2/keys.removePermissions` -- `POST /v2/keys.setPermissions` -- `POST /v2/keys.addRoles` -- `POST /v2/keys.removeRoles` -- `POST /v2/keys.setRoles` - -### Ratelimit Management -- `POST /v2/ratelimit.setOverride` -- `POST /v2/ratelimit.getOverride` -- `POST /v2/ratelimit.listOverrides` -- `POST /v2/ratelimit.deleteOverride` - -## Backward Compatibility - -- v1 endpoints remain available during the transition period -- No breaking changes to existing v1 functionality -- Gradual migration is supported - -## Benefits of v2 - -1. **Consistent Structure**: All responses follow the same format -2. **Better Security**: POST requests for sensitive operations -3. **Enhanced RBAC**: Built-in permissions and roles system -4. **Identity Management**: Better user association and tracking -5. **Dynamic Ratelimiting**: Override limits on-the-fly -6. **Improved Debugging**: Request IDs in all responses -7. **Better Validation**: Detailed error messages with fix suggestions -8. **Enhanced Credits System**: More detailed refill information - -## Timeline and Support - -### Migration Timeline -- **Phase 1**: v2 API available alongside v1 -- **Phase 2**: Deprecation warnings for v1 endpoints -- **Phase 3**: v1 endpoints sunset (timeline TBD) - -### Getting Help -- Email: [support@unkey.dev](mailto:support@unkey.dev) -- Documentation: [unkey.dev/docs](https://unkey.dev/docs) -- Discord: [unkey.dev/discord](https://unkey.dev/discord) - -## Next Steps - -1. Review your current v1 API usage -2. Test v2 endpoints in your development environment -3. Update your client code to handle the new response format -4. Migrate critical endpoints first -5. Implement new v2 features like RBAC and identities -6. Complete migration and deprecate v1 usage - -For additional support during migration, contact [support@unkey.dev](mailto:support@unkey.dev) with your specific use cases and requirements. \ No newline at end of file diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml index 0841e81e88b..482f8f9b3df 100644 --- a/deployment/docker-compose.yaml +++ b/deployment/docker-compose.yaml @@ -32,6 +32,7 @@ services: - mysql ports: - 3900:3900 + apiv2_lb: container_name: apiv2_lb image: nginx:latest diff --git a/go/.golangci.yaml b/go/.golangci.yaml index f33ee620ee3..cd4c9320ecb 100644 --- a/go/.golangci.yaml +++ b/go/.golangci.yaml @@ -68,6 +68,8 @@ linters: - ^github.com/ClickHouse/clickhouse-go/v2.*$ - ^github.com/aws/aws-sdk-go-v2/.*$ - ^github.com/unkeyed/unkey/go/apps/api/openapi.Meta$ + - ^gorm.io/gorm.Config$ + - ^gorm.io/gorm.*$ funlen: lines: 100 statements: 50 diff --git a/go/GO_DOCUMENTATION_GUIDELINES.md b/go/GO_DOCUMENTATION_GUIDELINES.md new file mode 100644 index 00000000000..8f7960936a8 --- /dev/null +++ b/go/GO_DOCUMENTATION_GUIDELINES.md @@ -0,0 +1,472 @@ +# Go Documentation Guidelines + +This document outlines the documentation standards for our Go codebase. + +## Core Principles + +1. **Everything public MUST be documented** - No exceptions +2. **Internal code should explain "why", not "how"** - Focus on reasoning and trade-offs +3. **Be comprehensive and verbose** - Prefer thorough explanations over terse summaries +4. **Add substantial value** - Documentation should teach, not just restate the obvious +5. **Follow Go conventions** - Start with the item name, use present tense + +## Documentation Philosophy + +**Serve both beginners and experts.** Documentation should provide clear, accessible entry points for newcomers AND comprehensive details for experts who need to understand the full picture, including architectural decisions and edge cases. + +**Clarity is better than terse.** We prefer comprehensive documentation that fully explains what the code does in detail, why it exists and its role in the system, how it relates to other components, what callers need to know about behavior and performance characteristics, when to use it versus alternatives, and what can go wrong and why. + +**Every piece of documentation should add substantial value.** If the documentation doesn't teach something beyond what's obvious from the function signature, it needs to be expanded. + +**Prioritize practical examples over theory.** Every non-trivial function should include working code examples that developers can copy and adapt. Examples should demonstrate real usage patterns, not artificial toy cases. + +**Make functionality discoverable.** Use extensive cross-references to help developers find related functions and understand how pieces fit together. If a function works with or is an alternative to another function, mention that explicitly. + +**Write in full sentences, not bullet points.** Code documentation should read like well-written prose that flows naturally. Avoid bullet points for general explanations, behavior descriptions, or conceptual information. Only use bullet points when they genuinely improve readability for specific lists such as error codes, configuration options, or step-by-step procedures. Most documentation should be written as coherent paragraphs that explain concepts thoroughly. + +## Package-Level Documentation + +**Every package MUST have a dedicated `doc.go` file** containing only package documentation and the package declaration. Do not put package documentation in random `.go` files above the package declaration. + +The `doc.go` file should explain what the package does, why it exists, how it fits into the larger system, key concepts and terminology, basic usage examples, and cross-references to key types and functions. + +### Format - doc.go File + +Create a `doc.go` file in each package with this structure: + +```go +// Package ratelimit implements distributed rate limiting with lease-based coordination. +// +// The package uses a two-phase commit protocol to ensure consistency across +// multiple nodes in a cluster. Rate limits are enforced through sliding time +// windows with configurable burst allowances. +// +// This implementation was chosen over simpler approaches because we need +// strong consistency guarantees for billing and security use cases. +// +// # Key Types +// +// The main entry point is [RateLimiter], which provides the [RateLimiter.Allow] +// method for checking rate limits. Configuration is handled through [Config]. +// +// # Usage +// +// Basic rate limiting: +// +// cfg := ratelimit.Config{Window: time.Minute, Limit: 100} +// limiter := ratelimit.New(cfg) +// allowed, err := limiter.Allow(ctx, "user:123", 1) +// if err != nil { +// // Handle system error +// } +// if !allowed { +// // Rate limited - reject request +// } +// +// For advanced configuration and cluster setup, see the examples in the +// /examples directory. +// +// # Error Handling +// +// The package distinguishes between rate limiting (expected behavior) and +// system errors (unexpected failures). See [ErrRateLimited] and [ErrClusterUnavailable] +// for the main error types. +package ratelimit +``` + +### Requirements for doc.go Files +- **File name**: Must be exactly `doc.go` in the package root +- **Content**: Only package documentation and package declaration - no other code +- **Format**: Start with "Package [name] [verb]..." +- **Structure**: Use `#` headers to organize sections (Key Types, Usage, Error Handling, etc.) +- **Purpose**: Explain what the package does and why it exists +- **Architecture**: Include reasoning for non-trivial design decisions +- **Examples**: Provide complete, runnable code examples +- **Cross-references**: Use `[TypeName]` and `[FunctionName]` format extensively +- **Related packages**: Reference external dependencies and related internal packages + +## Function and Method Documentation + +Every exported function and method must be documented. Focus on: +- What it does +- What parameters mean (especially if not obvious) +- What it returns +- Important behavior or side effects +- When it might fail + +### Public Functions - Comprehensive Documentation + +Every public function must be thoroughly documented. Here's what comprehensive documentation looks like: + +```go +// Allow determines whether the specified identifier can perform the requested number +// of operations within the configured rate limit window. +// +// This method implements distributed rate limiting with strong consistency guarantees +// across all nodes in the cluster. It uses a lease-based algorithm to coordinate +// between nodes and ensure accurate rate limiting even under high concurrency. +// +// Parameters: +// - identifier: A unique string identifying the entity being rate limited. This is +// typically a user ID, API key, IP address, or other business identifier. The +// identifier is used as the key for rate limit bucketing and should be stable +// across requests from the same entity. +// - cost: The number of operations being requested. For most use cases this is 1, +// but can be higher for batch operations or when implementing weighted rate limiting. +// Must be positive; zero or negative values will return an error. +// +// Behavior: +// - Checks the current rate limit status for the identifier +// - Coordinates with other cluster nodes if necessary to maintain consistency +// - Updates the rate limit counters atomically if the request is allowed +// - Implements fair queuing to prevent starvation under high load +// +// Performance: +// - Typical latency: <1ms for cached identifiers, <10ms for distributed coordination +// - Scales linearly with cluster size up to ~100 nodes +// - Memory usage: ~100 bytes per active identifier +// +// Returns: +// - (true, nil): Request is allowed and counters have been updated +// - (false, nil): Request is rate limited, no error condition +// - (false, error): System error occurred, decision may be unreliable +// +// Error conditions (be specific about when each occurs): +// - ErrInvalidCost: cost <= 0 or cost > MaxCost +// - ErrClusterUnavailable: <50% of cluster nodes reachable +// - context.DeadlineExceeded: operation timeout (default 5s) +// - Network errors: underlying storage failures, retries exhausted +// +// Concurrency: +// This method is safe for concurrent use from multiple goroutines. Internal +// coordination ensures that concurrent requests for the same identifier are +// handled correctly without race conditions. +// +// Context handling: +// The context is used for request timeout and cancellation. If the context +// is cancelled before the rate limit check completes, the method returns +// the context error and no rate limit counters are modified. +// +// Context Guidelines: +// - Always document timeout behavior and defaults +// - Explain what happens on cancellation +// - Mention if context values are used +func (r *RateLimiter) Allow(ctx context.Context, identifier string, cost int) (bool, error) +``` + +Compare this to insufficient documentation: +```go +// Allow checks if a request is allowed. +// Returns true if allowed, false if rate limited. +func (r *RateLimiter) Allow(ctx context.Context, identifier string, cost int) (bool, error) +``` + +The second example adds almost no value beyond the function signature and would be rejected. + +### Function Documentation Approach + +Write function documentation as natural, flowing prose that explains what actually matters for each specific function. Start with what the function does, then include whatever information is genuinely relevant and useful for callers. Some functions might need detailed parameter explanations, others might need performance notes, and simple functions might just need a clear explanation of their purpose. Don't force every function into the same template - let the function's complexity and use case guide what information to include. + +### Internal Functions (Focus on "Why") +```go +// retryWithBackoff handles retries for failed lease acquisitions. +// +// We use exponential backoff with jitter instead of linear backoff because +// under high load, linear backoff causes thundering herd problems when many +// clients retry simultaneously. The exponential approach with randomization +// spreads out retry attempts and reduces system load. +// +// Max retry count is limited to prevent infinite loops during system outages. +func (r *RateLimiter) retryWithBackoff(ctx context.Context, fn func() error) error +``` + +## Type Documentation + +Document all exported types, focusing on: +- What the type represents +- Its role in the system +- Important invariants or constraints +- Lifecycle considerations + +### Structs +```go +// Config holds the configuration for a rate limiter instance. +// +// Window and Limit work together to define the rate limiting behavior. +// For example, Window=1m and Limit=100 means "100 operations per minute". +// +// ClusterNodes is required for distributed operation. For single-node +// deployments, use a slice with only the local node. +type Config struct { + // Window is the time period over which operations are counted + Window time.Duration + + // Limit is the maximum number of operations allowed within Window + Limit int64 + + // ClusterNodes lists all nodes participating in distributed rate limiting. + // Must include at least the local node. + ClusterNodes []string +} +``` + +### Interfaces +```go +// Cache provides a generic caching interface with support for distributed invalidation. +// +// Implementations must be safe for concurrent use. The cache may return stale data +// during network partitions to maintain availability, but will eventually converge +// when connectivity is restored. +// +// We chose this interface design over more specific cache types because our +// use cases vary widely (small config objects vs large binary data), and +// the generic approach allows for better testing and modularity. +type Cache[T any] interface { + // Get retrieves a value by key. Returns the value and whether it was found. + // A cache miss (found=false) is not an error. + Get(ctx context.Context, key string) (value T, found bool, err error) + + // Set stores a value. The value will be replicated to other cache nodes + // asynchronously. Use SetSync if you need immediate consistency. + Set(ctx context.Context, key string, value T) error +} +``` + +## Error Documentation + +Document error conditions and types: + +```go +var ( + // ErrRateLimited is returned when an operation exceeds the configured rate limit. + // This is expected behavior, not a system error. + ErrRateLimited = errors.New("rate limit exceeded") + + // ErrClusterUnavailable indicates that the required number of cluster nodes + // are not reachable. Operations may still succeed if configured to fail-open. + ErrClusterUnavailable = errors.New("insufficient cluster nodes available") +) + +// ProcessRequest handles incoming rate limit requests. +// +// Returns ErrRateLimited if the request exceeds the configured limits. +// Returns ErrClusterUnavailable if distributed consensus cannot be achieved. +// Other errors indicate system problems (network, storage, etc.). +func ProcessRequest(ctx context.Context, req *Request) (*Response, error) +``` + +## Constants and Variables + +Document the purpose and valid values: + +```go +const ( + // DefaultWindow is the standard rate limiting window for new limiters. + // Chosen as a balance between memory usage and granularity for most use cases. + DefaultWindow = time.Minute + + // MaxBurstRatio determines how much bursting is allowed above the base rate. + // Set to 1.5 based on analysis of traffic patterns in production. + MaxBurstRatio = 1.5 +) + +var ( + // GlobalRegistry tracks all active rate limiters for monitoring and cleanup. + // We use a global registry instead of dependency injection here because + // rate limiters need to be accessible from signal handlers for graceful shutdown. + GlobalRegistry = &Registry{limiters: make(map[string]*RateLimiter)} +) +``` + +## Complex Algorithm Documentation + +For complex internal logic, explain the approach and reasoning: + +```go +// distributeTokens implements the token bucket algorithm with cluster coordination. +// +// We chose token bucket over sliding window because: +// 1. Better burst handling for API use cases +// 2. Simpler mathematics for distributed scenarios +// 3. More predictable memory usage +// +// The algorithm works in two phases: +// 1. Local calculation of available tokens +// 2. Cluster consensus on token allocation +// +// Phase 2 is optimized away when the local node has sufficient tokens, +// reducing latency for the common case. +func (r *RateLimiter) distributeTokens(ctx context.Context, required int64) (granted int64, err error) { + // Local fast path - no cluster coordination needed + if r.localTokens.Load() >= required { + // ... implementation + } + + // Cluster coordination required + // We use Raft consensus here instead of eventual consistency because + // rate limiting must be strictly enforced for security and billing + // ... implementation +} +``` + +## Examples and Usage + +Include examples for non-trivial usage: + +```go +// Example_basicUsage demonstrates typical rate limiter setup and usage. +func Example_basicUsage() { + cfg := Config{ + Window: time.Minute, + Limit: 1000, + ClusterNodes: []string{"localhost:8080"}, + } + + limiter, err := New(cfg) + if err != nil { + log.Fatal(err) + } + defer limiter.Close() + + // Check if user can make 5 API calls + allowed, err := limiter.Allow(context.Background(), "user:alice", 5) + if err != nil { + log.Printf("System error: %v", err) + return + } + + if !allowed { + log.Println("Rate limit exceeded") + return + } + + log.Println("Request allowed") + // Output: Request allowed +} +``` + +## Testing Documentation + +Document test helpers and complex test scenarios: + +```go +// newTestLimiter creates a rate limiter configured for testing. +// +// Uses in-memory storage and shorter time windows to speed up tests. +// Not suitable for production use due to lack of persistence. +func newTestLimiter(t *testing.T, limit int64) *RateLimiter { + // ... implementation +} + +// TestConcurrentAccess verifies that the rate limiter maintains accuracy +// under high concurrency. +// +// This test is critical because our production workload often has hundreds +// of goroutines hitting the same rate limiter simultaneously. +func TestConcurrentAccess(t *testing.T) { + // ... implementation +} +``` + +## Consistency and Style + +**Terminology must be consistent** across the entire codebase: +- Use the same terms for the same concepts (e.g., always "identifier", never mix with "key" or "ID") +- Define domain-specific terms in package documentation +- Create a glossary for complex domains + +**Parameter naming should be predictable:** +- `ctx context.Context` (always first parameter) +- `id string` or `identifier string` for rate limit keys +- `cost int` or `count int` for operation quantities + +## Go Documentation Conventions + +Follow these formatting and style conventions: + +1. **Active voice and clear, concise explanations** +2. Use `//` for function comments, `/* */` only for package-level overviews +3. **Present tense** ("Returns..." not "Will return...") +4. **Omit redundant phrases** like "This function..." - go straight to the verb +5. **Document parameters by name** without quotes +6. **Start sentences with capital letters and end with periods** +7. **Reference RFC or standards** when implementing them +8. **Document side effects** or mutating behavior +9. **Self-contained documentation** - provide all necessary information +10. **Cross-references** using Go's `[Reference]` format: + - `[OtherFunc]` for functions + - `[TypeName]` for structs/interfaces + - `[ConstantName]` for constants + +## Best Practices and Anti-Patterns + +**Highlight non-obvious behaviors and edge cases** by documenting nil input handling, concurrency hazards, silent failures, performance bottlenecks, scalability concerns, and conditions where functions behave unexpectedly. + +**Document what NOT to do** with specific examples: + +```go +// Allow checks rate limits for the given identifier. +// +// IMPORTANT: Do not call Allow() in a loop without backoff - this can +// overwhelm the system. Instead use: +// +// // Bad: +// for !limiter.Allow(ctx, id, 1) { /* busy wait */ } +// +// // Good: +// if allowed, err := limiter.Allow(ctx, id, 1); !allowed { +// return ErrRateLimited +// } +``` + +**Examples should be high-quality and idiomatic.** Follow Go best practices including proper `defer` usage, preferring slices over arrays, using realistic data and real-world scenarios, showing both correct usage and common pitfalls, following Go naming conventions such as `err` for errors and `ctx` for contexts, and formatting using Go's `ExampleFunc` style for `godoc`. + +## Documentation Checklist + +Before submitting code, verify: + +- [ ] **Every package has a dedicated `doc.go` file** with comprehensive package documentation +- [ ] Every exported function, method, type, constant, and variable is documented +- [ ] Package documentation in `doc.go` explains purpose, key concepts, and includes examples +- [ ] Internal code explains "why" decisions were made, not just "what" it does +- [ ] Error conditions and return values are clearly explained +- [ ] Complex algorithms include reasoning for the chosen approach +- [ ] Examples are provided for non-trivial usage patterns +- [ ] Documentation starts with the item name and uses present tense +- [ ] All documentation follows Go formatting conventions (proper line breaks, etc.) +- [ ] Cross-references use proper `[Reference]` format +- [ ] Edge cases and non-obvious behaviors are documented +- [ ] Anti-patterns are prevented with clear guidance + +## Deprecation and Breaking Changes + +When deprecating APIs, provide clear migration paths: + +```go +// Deprecated: Use NewRateLimiterV2 instead. This function will be removed in v2.0. +// +// Migration example: +// // Old: +// limiter := NewRateLimiter(100, time.Minute) +// +// // New: +// limiter := NewRateLimiterV2(Config{Limit: 100, Window: time.Minute}) +func NewRateLimiter(limit int, window time.Duration) *RateLimiter +``` + +## Tools and Validation + +Use these tools to validate documentation: + +```bash +# Check for missing documentation +go vet ./... + +# Generate and review documentation +godoc -http=:6060 + +# Check documentation formatting +go fmt ./... +``` + +Remember: Good documentation is an investment in your future self and your teammates. Take the time to write it well. diff --git a/go/go.mod b/go/go.mod index dff36292373..5f3a4fcb2bc 100644 --- a/go/go.mod +++ b/go/go.mod @@ -4,93 +4,100 @@ go 1.24.0 require ( connectrpc.com/connect v1.18.1 - github.com/ClickHouse/clickhouse-go/v2 v2.34.0 - github.com/aws/aws-sdk-go-v2 v1.36.3 - github.com/aws/aws-sdk-go-v2/config v1.29.6 - github.com/aws/aws-sdk-go-v2/credentials v1.17.59 - github.com/aws/aws-sdk-go-v2/service/s3 v1.76.1 + github.com/ClickHouse/clickhouse-go/v2 v2.35.0 + github.com/aws/aws-sdk-go-v2 v1.36.5 + github.com/aws/aws-sdk-go-v2/config v1.29.17 + github.com/aws/aws-sdk-go-v2/credentials v1.17.70 + github.com/aws/aws-sdk-go-v2/service/s3 v1.82.0 github.com/btcsuite/btcutil v1.0.2 github.com/go-redis/redis/v8 v8.11.5 github.com/go-sql-driver/mysql v1.9.2 - github.com/lmittmann/tint v1.0.7 + github.com/lmittmann/tint v1.1.1 github.com/maypok86/otter v1.2.4 github.com/oapi-codegen/nullable v1.1.0 github.com/oapi-codegen/oapi-codegen/v2 v2.4.1 github.com/oapi-codegen/runtime v1.1.1 - github.com/ory/dockertest/v3 v3.11.0 - github.com/pb33f/libopenapi v0.21.8 - github.com/pb33f/libopenapi-validator v0.4.0 + github.com/ory/dockertest/v3 v3.12.0 + github.com/pb33f/libopenapi v0.22.2 + github.com/pb33f/libopenapi-validator v0.4.6 github.com/prometheus/client_golang v1.22.0 - github.com/redis/go-redis/v9 v9.7.3 - github.com/shirou/gopsutil/v4 v4.25.3 + github.com/redis/go-redis/v9 v9.9.0 + github.com/shirou/gopsutil/v4 v4.25.5 github.com/sqlc-dev/sqlc v1.28.0 github.com/stretchr/testify v1.10.0 - github.com/urfave/cli/v3 v3.2.0 - go.opentelemetry.io/contrib/bridges/otelslog v0.10.0 - go.opentelemetry.io/contrib/bridges/prometheus v0.60.0 - go.opentelemetry.io/contrib/processors/minsev v0.8.0 - go.opentelemetry.io/otel v1.35.0 - go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.11.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 - go.opentelemetry.io/otel/metric v1.35.0 - go.opentelemetry.io/otel/sdk v1.35.0 - go.opentelemetry.io/otel/sdk/log v0.11.0 - go.opentelemetry.io/otel/sdk/metric v1.35.0 - go.opentelemetry.io/otel/trace v1.35.0 - golang.org/x/text v0.24.0 + github.com/unkeyed/unkey/apps/agent v0.0.0-20250630183506-b8772db8eb6f + github.com/urfave/cli/v3 v3.3.3 + go.opentelemetry.io/contrib/bridges/otelslog v0.11.0 + go.opentelemetry.io/contrib/bridges/prometheus v0.61.0 + go.opentelemetry.io/contrib/processors/minsev v0.9.0 + go.opentelemetry.io/otel v1.36.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.12.2 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0 + go.opentelemetry.io/otel/metric v1.36.0 + go.opentelemetry.io/otel/sdk v1.36.0 + go.opentelemetry.io/otel/sdk/log v0.12.2 + go.opentelemetry.io/otel/sdk/metric v1.36.0 + go.opentelemetry.io/otel/trace v1.36.0 + golang.org/x/text v0.25.0 google.golang.org/protobuf v1.36.6 + gorm.io/driver/mysql v1.6.0 + gorm.io/driver/sqlite v1.6.0 + gorm.io/gorm v1.30.0 ) require ( cel.dev/expr v0.20.0 // indirect - dario.cat/mergo v1.0.1 // indirect + dario.cat/mergo v1.0.2 // indirect filippo.io/edwards25519 v1.1.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect - github.com/ClickHouse/ch-go v0.65.1 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/ClickHouse/ch-go v0.66.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect + github.com/Southclaws/fault v0.8.1 // indirect github.com/andybalholm/brotli v1.1.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect - github.com/aws/smithy-go v1.22.2 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect + github.com/aws/smithy-go v1.22.4 // indirect + github.com/axiomhq/axiom-go v0.22.0 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/containerd/continuity v0.4.3 // indirect + github.com/containerd/continuity v0.4.5 // indirect github.com/cubicdaiya/gonp v1.0.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/docker/cli v27.2.0+incompatible // indirect - github.com/docker/docker v28.0.4+incompatible // indirect + github.com/docker/cli v28.2.2+incompatible // indirect + github.com/docker/docker v28.2.2+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dolthub/maphash v0.1.0 // indirect - github.com/dprotaso/go-yit v0.0.0-20240618133044-5a0af90af097 // indirect + github.com/dprotaso/go-yit v0.0.0-20250513224043-18a80f8f6df4 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ebitengine/purego v0.8.2 // indirect + github.com/ebitengine/purego v0.8.4 // indirect github.com/fatih/structtag v1.2.0 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gammazero/deque v1.0.0 // indirect - github.com/getkin/kin-openapi v0.127.0 // indirect + github.com/getkin/kin-openapi v0.131.0 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -98,29 +105,35 @@ require ( github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/cel-go v0.22.1 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/invopop/yaml v0.3.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgx/v5 v5.7.4 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect github.com/mailru/easyjson v0.9.0 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/term v0.5.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/term v0.5.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect + github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0 // indirect - github.com/opencontainers/runc v1.1.13 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/opencontainers/runc v1.3.0 // indirect github.com/paulmach/orb v0.11.1 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pganalyze/pg_query_go/v5 v5.1.0 // indirect @@ -133,15 +146,16 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.63.0 // indirect + github.com/prometheus/common v0.64.0 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/riza-io/grpc-go v0.2.0 // indirect - github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect + github.com/rs/zerolog v1.33.0 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/speakeasy-api/jsonpath v0.6.1 // indirect + github.com/speakeasy-api/jsonpath v0.6.2 // indirect github.com/speakeasy-api/openapi-overlay v0.9.0 // indirect github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.6 // indirect @@ -158,22 +172,23 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect - go.opentelemetry.io/otel/log v0.11.0 // indirect - go.opentelemetry.io/proto/otlp v1.5.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect + go.opentelemetry.io/otel/log v0.12.2 // indirect + go.opentelemetry.io/proto/otlp v1.7.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.37.0 // indirect + golang.org/x/crypto v0.38.0 // indirect golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect golang.org/x/mod v0.24.0 // indirect - golang.org/x/net v0.39.0 // indirect - golang.org/x/sync v0.13.0 // indirect - golang.org/x/sys v0.32.0 // indirect + golang.org/x/net v0.40.0 // indirect + golang.org/x/sync v0.14.0 // indirect + golang.org/x/sys v0.33.0 // indirect golang.org/x/tools v0.31.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250422160041-2d3770c4ea7f // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250422160041-2d3770c4ea7f // indirect - google.golang.org/grpc v1.72.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect + google.golang.org/grpc v1.72.2 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go/go.sum b/go/go.sum index eddfd1bff90..a24d0b0b664 100644 --- a/go/go.sum +++ b/go/go.sum @@ -2,22 +2,24 @@ cel.dev/expr v0.20.0 h1:OunBvVCfvpWlt4dN7zg3FM6TDkzOePe1+foGJ9AXeeI= cel.dev/expr v0.20.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= connectrpc.com/connect v1.18.1 h1:PAg7CjSAGvscaf6YZKUefjoih5Z/qYkyaTrBW8xvYPw= connectrpc.com/connect v1.18.1/go.mod h1:0292hj1rnx8oFrStN7cB4jjVBeqs+Yx5yDIC2prWDO8= -dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= -dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/ClickHouse/ch-go v0.65.1 h1:SLuxmLl5Mjj44/XbINsK2HFvzqup0s6rwKLFH347ZhU= -github.com/ClickHouse/ch-go v0.65.1/go.mod h1:bsodgURwmrkvkBe5jw1qnGDgyITsYErfONKAHn05nv4= -github.com/ClickHouse/clickhouse-go/v2 v2.34.0 h1:Y4rqkdrRHgExvC4o/NTbLdY5LFQ3LHS77/RNFxFX3Co= -github.com/ClickHouse/clickhouse-go/v2 v2.34.0/go.mod h1:yioSINoRLVZkLyDzdMXPLRIqhDvel8iLBlwh6Iefso8= +github.com/ClickHouse/ch-go v0.66.0 h1:hLslxxAVb2PHpbHr4n0d6aP8CEIpUYGMVT1Yj/Q5Img= +github.com/ClickHouse/ch-go v0.66.0/go.mod h1:noiHWyLMJAZ5wYuq3R/K0TcRhrNA8h7o1AqHX0klEhM= +github.com/ClickHouse/clickhouse-go/v2 v2.35.0 h1:ZMLZqxu+NiW55f4JS32kzyEbMb7CthGn3ziCcULOvSE= +github.com/ClickHouse/clickhouse-go/v2 v2.35.0/go.mod h1:O2FFT/rugdpGEW2VKyEGyMUWyQU0ahmenY9/emxLPxs= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/Southclaws/fault v0.8.1 h1:mgqqdC6kUBQ6ExMALZ0nNaDfNJD5h2+wq3se5mAyX+8= +github.com/Southclaws/fault v0.8.1/go.mod h1:VUVkAWutC59SL16s6FTqf3I6I2z77RmnaW5XRz4bLOE= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= @@ -25,42 +27,44 @@ github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYW github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= -github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg= -github.com/aws/aws-sdk-go-v2/config v1.29.6 h1:fqgqEKK5HaZVWLQoLiC9Q+xDlSp+1LYidp6ybGE2OGg= -github.com/aws/aws-sdk-go-v2/config v1.29.6/go.mod h1:Ft+WLODzDQmCTHDvqAH1JfC2xxbZ0MxpZAcJqmE1LTQ= -github.com/aws/aws-sdk-go-v2/credentials v1.17.59 h1:9btwmrt//Q6JcSdgJOLI98sdr5p7tssS9yAsGe8aKP4= -github.com/aws/aws-sdk-go-v2/credentials v1.17.59/go.mod h1:NM8fM6ovI3zak23UISdWidyZuI1ghNe2xjzUZAyT+08= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 h1:KwsodFKVQTlI5EyhRSugALzsV6mG/SGrdjlMXSZSdso= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28/go.mod h1:EY3APf9MzygVhKuPXAc5H+MkGb8k/DOSQjWS0LgkKqI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 h1:BjUcr3X3K0wZPGFg2bxOWW3VPN8rkE3/61zhP+IHviA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32/go.mod h1:80+OGC/bgzzFFTUmcuwD0lb4YutwQeKLFpmt6hoWapU= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 h1:m1GeXHVMJsRsUAqG6HjZWx9dj7F5TR+cF1bjyfYyBd4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32/go.mod h1:IitoQxGfaKdVLNg0hD8/DXmAqNy0H4K2H2Sf91ti8sI= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 h1:Pg9URiobXy85kgFev3og2CuOZ8JZUBENF+dcgWBaYNk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 h1:OIHj/nAhVzIXGzbAE+4XmZ8FPvro3THr6NlqErJc3wY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32/go.mod h1:LiBEsDo34OJXqdDlRGsilhlIiXR7DL+6Cx2f4p1EgzI= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.0 h1:kT2WeWcFySdYpPgyqJMSUE7781Qucjtn6wBvrgm9P+M= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.0/go.mod h1:WYH1ABybY7JK9TITPnk6ZlP7gQB8psI4c9qDmMsnLSA= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 h1:SYVGSFQHlchIcy6e7x12bsrxClCXSP5et8cqVhL8cuw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13/go.mod h1:kizuDaLX37bG5WZaoxGPQR/LNFXpxp0vsUnqfkWXfNE= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 h1:OBsrtam3rk8NfBEq7OLOMm5HtQ9Yyw32X4UQMya/wjw= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13/go.mod h1:3U4gFA5pmoCOja7aq4nSaIAGbaOHv2Yl2ug018cmC+Q= -github.com/aws/aws-sdk-go-v2/service/s3 v1.76.1 h1:d4ZG8mELlLeUWFBMCqPtRfEP3J6aQgg/KTC9jLSlkMs= -github.com/aws/aws-sdk-go-v2/service/s3 v1.76.1/go.mod h1:uZoEIR6PzGOZEjgAZE4hfYfsqK2zOHhq68JLKEvvXj4= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 h1:/eE3DogBjYlvlbhd2ssWyeuovWunHLxfgw3s/OJa4GQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.15/go.mod h1:2PCJYpi7EKeA5SkStAmZlF6fi0uUABuhtF8ILHjGc3Y= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 h1:M/zwXiL2iXUrHputuXgmO94TVNmcenPHxgLXLutodKE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14/go.mod h1:RVwIw3y/IqxC2YEXSIkAzRDdEU1iRabDPaYjpGCbCGQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 h1:TzeR06UCMUq+KA3bDkujxK1GVGy+G8qQN/QVYzGLkQE= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.14/go.mod h1:dspXf/oYWGWo6DEvj98wpaTeqt5+DMidZD0A9BYTizc= -github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= -github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/aws/aws-sdk-go-v2 v1.36.5 h1:0OF9RiEMEdDdZEMqF9MRjevyxAQcf6gY+E7vwBILFj0= +github.com/aws/aws-sdk-go-v2 v1.36.5/go.mod h1:EYrzvCCN9CMUTa5+6lf6MM4tq3Zjp8UhSGR/cBsjai0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 h1:12SpdwU8Djs+YGklkinSSlcrPyj3H4VifVsKf78KbwA= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11/go.mod h1:dd+Lkp6YmMryke+qxW/VnKyhMBDTYP41Q2Bb+6gNZgY= +github.com/aws/aws-sdk-go-v2/config v1.29.17 h1:jSuiQ5jEe4SAMH6lLRMY9OVC+TqJLP5655pBGjmnjr0= +github.com/aws/aws-sdk-go-v2/config v1.29.17/go.mod h1:9P4wwACpbeXs9Pm9w1QTh6BwWwJjwYvJ1iCt5QbCXh8= +github.com/aws/aws-sdk-go-v2/credentials v1.17.70 h1:ONnH5CM16RTXRkS8Z1qg7/s2eDOhHhaXVd72mmyv4/0= +github.com/aws/aws-sdk-go-v2/credentials v1.17.70/go.mod h1:M+lWhhmomVGgtuPOhO85u4pEa3SmssPTdcYpP/5J/xc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 h1:KAXP9JSHO1vKGCr5f4O6WmlVKLFFXgWYAGoJosorxzU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32/go.mod h1:h4Sg6FQdexC1yYG9RDnOvLbW1a/P986++/Y/a+GyEM8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 h1:SsytQyTMHMDPspp+spo7XwXTP44aJZZAC7fBV2C5+5s= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36/go.mod h1:Q1lnJArKRXkenyog6+Y+zr7WDpk4e6XlR6gs20bbeNo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 h1:i2vNHQiXUvKhs3quBR6aqlgJaiaexz/aNvdCktW/kAM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36/go.mod h1:UdyGa7Q91id/sdyHPwth+043HhmP6yP9MBHgbZM0xo8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 h1:GMYy2EOWfzdP3wfVAGXBNKY5vK4K8vMET4sYOYltmqs= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36/go.mod h1:gDhdAV6wL3PmPqBhiPbnlS447GoWs8HTTOYef9/9Inw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 h1:CXV68E2dNqhuynZJPB80bhPQwAKqBWVer887figW6Jc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4/go.mod h1:/xFi9KtvBXP97ppCz1TAEvU1Uf66qvid89rbem3wCzQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 h1:nAP2GYbfh8dd2zGZqFRSMlq+/F6cMPBUuCsGAMkN074= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4/go.mod h1:LT10DsiGjLWh4GbjInf9LQejkYEhBgBCjLG5+lvk4EE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 h1:t0E6FzREdtCsiLIoLCWsYliNsRBgyGD/MCK571qk4MI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17/go.mod h1:ygpklyoaypuyDvOM5ujWGrYWpAK3h7ugnmKCU/76Ys4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 h1:qcLWgdhq45sDM9na4cvXax9dyLitn8EYBRl8Ak4XtG4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17/go.mod h1:M+jkjBFZ2J6DJrjMv2+vkBbuht6kxJYtJiwoVgX4p4U= +github.com/aws/aws-sdk-go-v2/service/s3 v1.82.0 h1:JubM8CGDDFaAOmBrd8CRYNr49ZNgEAiLwGwgNMdS0nw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.82.0/go.mod h1:kUklwasNoCn5YpyAqC/97r6dzTA1SRKJfKq16SXeoDU= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 h1:AIRJ3lfb2w/1/8wOOSqYb9fUKGwQbtysJ2H1MofRUPg= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.5/go.mod h1:b7SiVprpU+iGazDUqvRSLf5XmCdn+JtT1on7uNL6Ipc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 h1:BpOxT3yhLwSJ77qIY3DoHAQjZsc4HEGfMCE4NGy3uFg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3/go.mod h1:vq/GQR1gOFLquZMSrxUK/cpvKCNVYibNyJ1m7JrU88E= +github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 h1:NFOJ/NXEGV4Rq//71Hs1jC/NvPs1ezajK+yQmkwnPV0= +github.com/aws/aws-sdk-go-v2/service/sts v1.34.0/go.mod h1:7ph2tGpfQvwzgistp2+zga9f+bCjlQJPkPUmMgDSD7w= +github.com/aws/smithy-go v1.22.4 h1:uqXzVZNuNexwc/xrh6Tb56u89WDlJY6HS+KC0S4QSjw= +github.com/aws/smithy-go v1.22.4/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/axiomhq/axiom-go v0.22.0 h1:QFC09ugLrwc8DNaq8QgF4Q2R/B2V5xYTjZzvUfe72s8= +github.com/axiomhq/axiom-go v0.22.0/go.mod h1:ybDThTO73XgRNQjTRxXqUiZh3QM7Wf5/exaFbp9VgLY= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -85,13 +89,13 @@ github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMU github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= +github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= -github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= +github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -106,10 +110,10 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/docker/cli v27.2.0+incompatible h1:yHD1QEB1/0vr5eBNpu8tncu8gWxg8EydFPOSKHzXSMM= -github.com/docker/cli v27.2.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker v28.0.4+incompatible h1:JNNkBctYKurkw6FrHfKqY0nKIDf5nrbxjVBtS+cdcok= -github.com/docker/docker v28.0.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/cli v28.2.2+incompatible h1:qzx5BNUDFqlvyq4AHzdNB7gSyVTmU4cgsyN9SdInc1A= +github.com/docker/cli v28.2.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= +github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -117,29 +121,30 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ= github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4= github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960/go.mod h1:9HQzr9D/0PGwMEbC3d5AB7oi67+h4TsQqItC1GVYG58= -github.com/dprotaso/go-yit v0.0.0-20240618133044-5a0af90af097 h1:f5nA5Ys8RXqFXtKc0XofVRiuwNTuJzPIwTmbjLz9vj8= -github.com/dprotaso/go-yit v0.0.0-20240618133044-5a0af90af097/go.mod h1:FTAVyH6t+SlS97rv6EXRVuBDLkQqcIe/xQw9f4IFUI4= +github.com/dprotaso/go-yit v0.0.0-20250513224043-18a80f8f6df4 h1:JzpdVajvTuXQXL10D0vId1ZcW9alSJ3H0CnZczzz4ec= +github.com/dprotaso/go-yit v0.0.0-20250513224043-18a80f8f6df4/go.mod h1:lHwJo6jMevQL9tNpW6vLyhkK13bYHBcoh9tUakMhbnE= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I= -github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= +github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34= github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo= -github.com/getkin/kin-openapi v0.127.0 h1:Mghqi3Dhryf3F8vR370nN67pAERW+3a95vomb3MAREY= -github.com/getkin/kin-openapi v0.127.0/go.mod h1:OZrfXzUfGrNbsKj+xmFBx6E5c6yH3At/tAKSc2UszXM= +github.com/getkin/kin-openapi v0.131.0 h1:NO2UeHnFKRYhZ8wg6Nyh5Cq7dHk4suQQr72a4pMrDxE= +github.com/getkin/kin-openapi v0.131.0/go.mod h1:3OlG51PCYNsPByuiMB0t4fjnNlIDnaEDsjiKUV8nL58= github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= @@ -153,35 +158,26 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-sql-driver/mysql v1.9.2 h1:4cNKDYQ1I84SXslGddlsrMhc8k4LeDVj6Ad6WRjiHuU= github.com/go-sql-driver/mysql v1.9.2/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/cel-go v0.22.1 h1:AfVXx3chM2qwoSbM7Da8g8hX8OVSkBFwX+rz2+PcK40= github.com/google/cel-go v0.22.1/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= @@ -191,11 +187,8 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso= -github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -207,6 +200,8 @@ github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFr github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= @@ -228,20 +223,29 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lmittmann/tint v1.0.7 h1:D/0OqWZ0YOGZ6AyC+5Y2kD8PBEzBk6rFHVSfOqCkF9Y= -github.com/lmittmann/tint v1.0.7/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE= +github.com/lmittmann/tint v1.1.1 h1:xmmGuinUsCSxWdwH1OqMUQ4tzQsq3BdjJLAAmVKJ9Dw= +github.com/lmittmann/tint v1.1.1/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE= github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/maypok86/otter v1.2.4 h1:HhW1Pq6VdJkmWwcZZq19BlEQkHtI8xgsQzBVXJU0nfc= github.com/maypok86/otter v1.2.4/go.mod h1:mKLfoI7v1HOmQMwFgX4QkRk23mX6ge3RDvjdHOWG4R4= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= @@ -249,46 +253,42 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= +github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/oapi-codegen/nullable v1.1.0 h1:eAh8JVc5430VtYVnq00Hrbpag9PFRGWLjxR1/3KntMs= github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= github.com/oapi-codegen/oapi-codegen/v2 v2.4.1 h1:ykgG34472DWey7TSjd8vIfNykXgjOgYJZoQbKfEeY/Q= github.com/oapi-codegen/oapi-codegen/v2 v2.4.1/go.mod h1:N5+lY1tiTDV3V1BeHtOxeWXHoPVeApvsvjJqegfoaz8= github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= +github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//JalHPu/3yz+De2J+4aLtSRlHiY= +github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037/go.mod h1:2bpvgLBZEtENV5scfDFEtB/5+1M4hkQhDQrccEJ/qGw= +github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 h1:bQx3WeLcUWy+RletIKwUIt4x3t8n2SxavmoclizMb8c= +github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= -github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= +github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/opencontainers/runc v1.1.13 h1:98S2srgG9vw0zWcDpFMn5TRrh8kLxa/5OFUstuUhmRs= -github.com/opencontainers/runc v1.1.13/go.mod h1:R016aXacfp/gwQBYw2FDGa9m+n6atbLWrYY8hNMT/sA= -github.com/ory/dockertest/v3 v3.11.0 h1:OiHcxKAvSDUwsEVh2BjxQQc/5EHz9n0va9awCtNGuyA= -github.com/ory/dockertest/v3 v3.11.0/go.mod h1:VIPxS1gwT9NpPOrfD3rACs8Y9Z7yhzO4SB194iUDnUI= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opencontainers/runc v1.3.0 h1:cvP7xbEvD0QQAs0nZKLzkVog2OPZhI/V2w3WmTmUSXI= +github.com/opencontainers/runc v1.3.0/go.mod h1:9wbWt42gV+KRxKRVVugNP6D5+PQciRbenB4fLVsqGPs= +github.com/ory/dockertest/v3 v3.12.0 h1:3oV9d0sDzlSQfHtIaB5k6ghUCVMVLpAY8hwrqoCyRCw= +github.com/ory/dockertest/v3 v3.12.0/go.mod h1:aKNDTva3cp8dwOWwb9cWuX84aH5akkxXRvO7KCwWVjE= github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= -github.com/pb33f/libopenapi v0.21.8 h1:Fi2dAogMwC6av/5n3YIo7aMOGBZH/fBMO4OnzFB3dQA= -github.com/pb33f/libopenapi v0.21.8/go.mod h1:Gc8oQkjr2InxwumK0zOBtKN9gIlv9L2VmSVIUk2YxcU= -github.com/pb33f/libopenapi-validator v0.4.0 h1:3ZdmyyP1oztytrJTPU3BTYGxUgzsTTNBA2uQNgmjzqk= -github.com/pb33f/libopenapi-validator v0.4.0/go.mod h1:W+odPcfKledbm+G+Ic1YAPz+WoPHKqpHzQ9UoJMnjB0= +github.com/pb33f/libopenapi v0.22.2 h1:ChXG911vrr24KE7wzIib3eL8Td73ANFCNSpWf1C9hy4= +github.com/pb33f/libopenapi v0.22.2/go.mod h1:utT5sD2/mnN7YK68FfZT5yEPbI1wwRBpSS4Hi0oOrBU= +github.com/pb33f/libopenapi-validator v0.4.6 h1:ESkSxqFnb3LwLyDShOYe0PlGEM+pXXMI0271+Ib/pFE= +github.com/pb33f/libopenapi-validator v0.4.6/go.mod h1:NJaqqPxX2SX6kn+YTu+i588es/qIjP0vfGwK2NWg2Pw= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pganalyze/pg_query_go/v5 v5.1.0 h1:MlxQqHZnvA3cbRQYyIrjxEjzo560P6MyTgtlaf3pmXg= @@ -316,33 +316,36 @@ github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/ github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= -github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= +github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4= +github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= -github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= -github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= +github.com/redis/go-redis/v9 v9.9.0 h1:URbPQ4xVQSQhZ27WMQVmZSo3uT3pL+4IdHVcYq2nVfM= +github.com/redis/go-redis/v9 v9.9.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/riza-io/grpc-go v0.2.0 h1:2HxQKFVE7VuYstcJ8zqpN84VnAoJ4dCL6YFhJewNcHQ= github.com/riza-io/grpc-go v0.2.0/go.mod h1:2bDvR9KkKC3KhtlSHfR3dAXjUMT86kg4UfWFyVGWqi8= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= +github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 h1:PKK9DyHxif4LZo+uQSgXNqs0jj5+xZwwfKHgph2lxBw= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.1/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v4 v4.25.3 h1:SeA68lsu8gLggyMbmCn8cmp97V1TI9ld9sVzAUcKcKE= -github.com/shirou/gopsutil/v4 v4.25.3/go.mod h1:xbuxyoZj+UsgnZrENu3lQivsngRR5BdjbJwf2fv4szA= +github.com/shirou/gopsutil/v4 v4.25.5 h1:rtd9piuSMGeU8g1RMXjZs9y9luK5BwtnG7dZaQUJAsc= +github.com/shirou/gopsutil/v4 v4.25.5/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/speakeasy-api/jsonpath v0.6.1 h1:FWbuCEPGaJTVB60NZg2orcYHGZlelbNJAcIk/JGnZvo= -github.com/speakeasy-api/jsonpath v0.6.1/go.mod h1:ymb2iSkyOycmzKwbEAYPJV/yi2rSmvBCLZJcyD+VVWw= +github.com/speakeasy-api/jsonpath v0.6.2 h1:Mys71yd6u8kuowNCR0gCVPlVAHCmKtoGXYoAtcEbqXQ= +github.com/speakeasy-api/jsonpath v0.6.2/go.mod h1:ymb2iSkyOycmzKwbEAYPJV/yi2rSmvBCLZJcyD+VVWw= github.com/speakeasy-api/openapi-overlay v0.9.0 h1:Wrz6NO02cNlLzx1fB093lBlYxSI54VRhy1aSutx0PQg= github.com/speakeasy-api/openapi-overlay v0.9.0/go.mod h1:f5FloQrHA7MsxYg9djzMD5h6dxrHjVVByWKh7an8TRc= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= @@ -365,15 +368,25 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4= github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/urfave/cli/v3 v3.2.0 h1:m8WIXY0U9LCuUl5r+0fqLWDhNYWt6qvlW+GcF4EoXf8= -github.com/urfave/cli/v3 v3.2.0/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZtpYpo= +github.com/unkeyed/unkey/apps/agent v0.0.0-20250630183506-b8772db8eb6f h1:NF9XU+xj2IFJWimZoFla1ezyAWLWiIwPkPOMmp7sWN8= +github.com/unkeyed/unkey/apps/agent v0.0.0-20250630183506-b8772db8eb6f/go.mod h1:HSiiGRMpVNu1MPwm6MB0ajziWqPcAHZZap48/38b1kY= +github.com/urfave/cli/v3 v3.3.3 h1:byCBaVdIXuLPIDm5CYZRVG6NvT7tv1ECqdU4YzlEa3I= +github.com/urfave/cli/v3 v3.3.3/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZtpYpo= github.com/vmware-labs/yaml-jsonpath v0.3.2 h1:/5QKeCBGdsInyDCyVNLbXyilb61MXGi9NP674f9Hobk= github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN1BWz5V+51Ewf8k+rQ= github.com/wasilibs/go-pgquery v0.0.0-20240606042535-c0843d6592cc h1:Hgim1Xgk1+viV7p0aZh9OOrMRfG+E4mGA+JsI2uB0+k= @@ -402,36 +415,40 @@ github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQ go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/bridges/otelslog v0.10.0 h1:lRKWBp9nWoBe1HKXzc3ovkro7YZSb72X2+3zYNxfXiU= -go.opentelemetry.io/contrib/bridges/otelslog v0.10.0/go.mod h1:D+iyUv/Wxbw5LUDO5oh7x744ypftIryiWjoj42I6EKs= -go.opentelemetry.io/contrib/bridges/prometheus v0.60.0 h1:x7sPooQCwSg27SjtQee8GyIIRTQcF4s7eSkac6F2+VA= -go.opentelemetry.io/contrib/bridges/prometheus v0.60.0/go.mod h1:4K5UXgiHxV484efGs42ejD7E2J/sIlepYgdGoPXe7hE= -go.opentelemetry.io/contrib/processors/minsev v0.8.0 h1:/i0gaV0Z174Twy1/NfgQoE+oQvFVbQItNl8UMwe62Jc= -go.opentelemetry.io/contrib/processors/minsev v0.8.0/go.mod h1:5siKBWhXmdM2gNh8KHZ4b97bdS4MYhqPJEEu6JtHciw= -go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= -go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.11.0 h1:C/Wi2F8wEmbxJ9Kuzw/nhP+Z9XaHYMkyDmXy6yR2cjw= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.11.0/go.mod h1:0Lr9vmGKzadCTgsiBydxr6GEZ8SsZ7Ks53LzjWG5Ar4= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0 h1:0NIXxOCFx+SKbhCVxwl3ETG8ClLPAa0KuKV6p3yhxP8= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0/go.mod h1:ChZSJbbfbl/DcRZNc9Gqh6DYGlfjw4PvO1pEOZH1ZsE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 h1:1fTNlAIJZGWLP5FVu0fikVry1IsiUnXjf7QFvoNN3Xw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0/go.mod h1:zjPK58DtkqQFn+YUMbx0M2XV3QgKU0gS9LeGohREyK4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 h1:xJ2qHD0C1BeYVTLLR9sX12+Qb95kfeD/byKj6Ky1pXg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0/go.mod h1:u5BF1xyjstDowA1R5QAO9JHzqK+ublenEW/dyqTjBVk= -go.opentelemetry.io/otel/log v0.11.0 h1:c24Hrlk5WJ8JWcwbQxdBqxZdOK7PcP/LFtOtwpDTe3Y= -go.opentelemetry.io/otel/log v0.11.0/go.mod h1:U/sxQ83FPmT29trrifhQg+Zj2lo1/IPN1PF6RTFqdwc= -go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= -go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= -go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= -go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= -go.opentelemetry.io/otel/sdk/log v0.11.0 h1:7bAOpjpGglWhdEzP8z0VXc4jObOiDEwr3IYbhBnjk2c= -go.opentelemetry.io/otel/sdk/log v0.11.0/go.mod h1:dndLTxZbwBstZoqsJB3kGsRPkpAgaJrWfQg3lhlHFFY= -go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= -go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= -go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= -go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= -go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= -go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= +go.opentelemetry.io/contrib/bridges/otelslog v0.11.0 h1:EMIiYTms4Z4m3bBuKp1VmMNRLZcl6j4YbvOPL1IhlWo= +go.opentelemetry.io/contrib/bridges/otelslog v0.11.0/go.mod h1:DIEZmUR7tzuOOVUTDKvkGWtYWSHFV18Qg8+GMb8wPJw= +go.opentelemetry.io/contrib/bridges/prometheus v0.61.0 h1:RyrtJzu5MAmIcbRrwg75b+w3RlZCP0vJByDVzcpAe3M= +go.opentelemetry.io/contrib/bridges/prometheus v0.61.0/go.mod h1:tirr4p9NXbzjlbruiRGp53IzlYrDk5CO2fdHj0sSSaY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= +go.opentelemetry.io/contrib/processors/minsev v0.9.0 h1:eKlDcNp+GSygGk6PMJJyEdej+E1HteUy+KsY2YzaLbM= +go.opentelemetry.io/contrib/processors/minsev v0.9.0/go.mod h1:p8UCIy0r8hjrVD1Hb/4IUDSIpiZmlJl5DhCZOYgMWc4= +go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= +go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.12.2 h1:tPLwQlXbJ8NSOfZc4OkgU5h2A38M4c9kfHSVc4PFQGs= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.12.2/go.mod h1:QTnxBwT/1rBIgAG1goq6xMydfYOBKU6KTiYF4fp5zL8= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0 h1:gAU726w9J8fwr4qRDqu1GYMNNs4gXrU+Pv20/N1UpB4= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0/go.mod h1:RboSDkp7N292rgu+T0MgVt2qgFGu6qa1RpZDOtpL76w= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0 h1:nRVXXvf78e00EwY6Wp0YII8ww2JVWshZ20HfTlE11AM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0/go.mod h1:r49hO7CgrxY9Voaj3Xe8pANWtr0Oq916d0XAmOoCZAQ= +go.opentelemetry.io/otel/log v0.12.2 h1:yob9JVHn2ZY24byZeaXpTVoPS6l+UrrxmxmPKohXTwc= +go.opentelemetry.io/otel/log v0.12.2/go.mod h1:ShIItIxSYxufUMt+1H5a2wbckGli3/iCfuEbVZi/98E= +go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= +go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= +go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= +go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= +go.opentelemetry.io/otel/sdk/log v0.12.2 h1:yNoETvTByVKi7wHvYS6HMcZrN5hFLD7I++1xIZ/k6W0= +go.opentelemetry.io/otel/sdk/log v0.12.2/go.mod h1:DcpdmUXHJgSqN/dh+XMWa7Vf89u9ap0/AAk/XGLnEzY= +go.opentelemetry.io/otel/sdk/log/logtest v0.0.0-20250521073539-a85ae98dcedc h1:uqxdywfHqqCl6LmZzI3pUnXT1RGFYyUgxj0AkWPFxi0= +go.opentelemetry.io/otel/sdk/log/logtest v0.0.0-20250521073539-a85ae98dcedc/go.mod h1:TY/N/FT7dmFrP/r5ym3g0yysP1DefqGpAZr4f82P0dE= +go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= +go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= +go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= +go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= +go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -453,8 +470,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -467,57 +484,47 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= @@ -525,20 +532,13 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20250422160041-2d3770c4ea7f h1:tjZsroqekhC63+WMqzmWyW5Twj/ZfR5HAlpd5YQ1Vs0= -google.golang.org/genproto/googleapis/api v0.0.0-20250422160041-2d3770c4ea7f/go.mod h1:Cd8IzgPo5Akum2c9R6FsXNaZbH3Jpa2gpHlW89FqlyQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250422160041-2d3770c4ea7f h1:N/PrbTw4kdkqNRzVfWPrBekzLuarFREcbFOiOLkXon4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250422160041-2d3770c4ea7f/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= -google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a h1:SGktgSolFCo75dnHJF2yMvnns6jCmHFJ0vE4Vn2JKvQ= +google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a/go.mod h1:a77HrdMjoeKbnd2jmgcWdaS++ZLZAEq3orIOAEIKiVw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= +google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= @@ -558,15 +558,19 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg= +gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo= +gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= +gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= +gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs= +gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= modernc.org/cc/v4 v4.24.4 h1:TFkx1s6dCkQpd6dKurBNmpo+G8Zl4Sq/ztJ+2+DEsh0= diff --git a/go/pkg/circuitbreaker/interface.go b/go/pkg/circuitbreaker/interface.go index 784f04ad2a6..bc1332ae596 100644 --- a/go/pkg/circuitbreaker/interface.go +++ b/go/pkg/circuitbreaker/interface.go @@ -5,51 +5,25 @@ import ( "errors" ) -// State represents the current operational state of a circuit breaker. type State string var ( // Open state means the circuit breaker is open and requests are not allowed - // to pass through. This state is activated when the failure threshold is - // exceeded, preventing further requests to a failing service to allow it - // time to recover. + // to pass through Open State = "open" - // HalfOpen state means the circuit breaker is in a state of testing the - // upstream service to see if it has recovered. In this state, a limited - // number of requests are allowed through to test the service. If these - // succeed, the circuit will close; if they fail, it will reopen. + // upstream service to see if it has recovered HalfOpen State = "halfopen" - // Closed state means the circuit breaker is allowing requests to pass - // through to the upstream service. This is the normal operational state - // when the service is functioning correctly. + // through to the upstream service Closed State = "closed" ) var ( - // ErrTripped is returned when the circuit breaker is open and requests - // are being blocked to protect the downstream service. - ErrTripped = errors.New("circuit breaker is open") - - // ErrTooManyRequests is returned when too many requests have been made - // during the half-open state, exceeding the configured test request limit. + ErrTripped = errors.New("circuit breaker is open") ErrTooManyRequests = errors.New("too many requests during half open state") ) -// CircuitBreaker provides a mechanism to detect failures and prevent -// cascading failures by blocking requests to failing dependencies when -// they exceed a threshold of failures. -// -// The generic parameter Res represents the successful response type from -// the protected operation. type CircuitBreaker[Res any] interface { - // Do executes the provided function with circuit breaker protection. - // If the circuit is open, it will return ErrTripped without calling the function. - // If the circuit is half-open and too many test requests are in flight, - // it will return ErrTooManyRequests. - // - // The function is executed with the provided context, and the circuit breaker - // tracks its success or failure to determine whether to trip or reset the circuit. Do(ctx context.Context, fn func(context.Context) (Res, error)) (Res, error) } diff --git a/go/pkg/circuitbreaker/lib.go b/go/pkg/circuitbreaker/lib.go index d3b3a3aec9a..f9ef28f80a0 100644 --- a/go/pkg/circuitbreaker/lib.go +++ b/go/pkg/circuitbreaker/lib.go @@ -3,18 +3,14 @@ package circuitbreaker import ( "context" "fmt" - "log/slog" "sync" "time" "github.com/unkeyed/unkey/go/pkg/clock" "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/otel/tracing" - "github.com/unkeyed/unkey/go/pkg/prometheus/metrics" ) -// CB implements the CircuitBreaker interface with configurable failure detection -// and recovery behavior. type CB[Res any] struct { sync.Mutex // This is a pointer to the configuration of the circuit breaker because we @@ -40,7 +36,6 @@ type CB[Res any] struct { consecutiveFailures int } -// config holds the configuration parameters for a circuit breaker instance. type config struct { name string // Max requests that may pass through the circuit breaker in its half-open state @@ -69,70 +64,41 @@ type config struct { logger logging.Logger } -// WithMaxRequests configures the maximum number of requests allowed during -// the half-open state. Once this threshold is reached, the circuit will either -// close (if all requests succeeded) or remain open (if any request failed). -// -// Default: 10 func WithMaxRequests(maxRequests int) applyConfig { return func(c *config) { c.maxRequests = maxRequests } } -// WithCyclicPeriod sets the interval at which request counters are reset -// while the circuit is closed. This determines how frequently the circuit -// "forgets" about past failures. -// -// Default: 5 seconds func WithCyclicPeriod(cyclicPeriod time.Duration) applyConfig { return func(c *config) { c.cyclicPeriod = cyclicPeriod } } - -// WithIsDownstreamError provides a function to determine if an error should -// be counted towards the failure threshold. This allows the circuit breaker -// to ignore certain types of errors (e.g., validation errors) that don't -// indicate a problem with the downstream service. -// -// Default: func(err error) bool { return err != nil } func WithIsDownstreamError(isDownstreamError func(error) bool) applyConfig { return func(c *config) { c.isDownstreamError = isDownstreamError } } - -// WithTripThreshold sets how many failures must occur within a cyclic period -// before the circuit breaker trips and enters the open state. -// -// Default: 5 func WithTripThreshold(tripThreshold int) applyConfig { return func(c *config) { c.tripThreshold = tripThreshold } } -// WithTimeout sets how long the circuit breaker stays in the open state -// before transitioning to half-open to test if the service has recovered. -// -// Default: 1 minute func WithTimeout(timeout time.Duration) applyConfig { return func(c *config) { c.timeout = timeout } } -// WithClock provides a custom clock implementation for testing time-based behavior. -// This should only be used in test code. +// for testing func WithClock(clock clock.Clock) applyConfig { return func(c *config) { c.clock = clock } } -// WithLogger provides a custom logger for the circuit breaker to use. -// If not provided, a no-op logger will be used. func WithLogger(logger logging.Logger) applyConfig { return func(c *config) { c.logger = logger @@ -142,24 +108,6 @@ func WithLogger(logger logging.Logger) applyConfig { // applyConfig applies a config setting to the circuit breaker type applyConfig func(*config) -// New creates a new circuit breaker with configurable behavior. -// The name parameter identifies this circuit breaker for logging and metrics. -// The generic parameter Res specifies the response type from the protected operation. -// -// Configuration is provided via functional options: -// -// cb := New[*http.Response]("api_service", -// WithTripThreshold(10), -// WithTimeout(30 * time.Second), -// WithIsDownstreamError(func(err error) bool { -// // Only count 5xx errors as downstream failures -// var httpErr *HttpError -// if errors.As(err, &httpErr) { -// return httpErr.StatusCode >= 500 -// } -// return err != nil -// }), -// ) func New[Res any](name string, applyConfigs ...applyConfig) *CB[Res] { cfg := &config{ @@ -196,15 +144,20 @@ func New[Res any](name string, applyConfigs ...applyConfig) *CB[Res] { return cb } -var _ CircuitBreaker[any] = (*CB[any])(nil) +var _ CircuitBreaker[any] = &CB[any]{ + Mutex: sync.Mutex{}, + config: nil, + logger: nil, + state: Closed, + resetCountersAt: time.Time{}, + resetStateAt: time.Time{}, + requests: 0, + successes: 0, + failures: 0, + consecutiveSuccesses: 0, + consecutiveFailures: 0, +} -// Do executes a function with circuit breaker protection. -// If the circuit is open, it returns ErrTripped without executing the function. -// If the circuit is half-open and the maximum test requests are already in flight, -// it returns ErrTooManyRequests. -// -// The function is wrapped with appropriate tracing to track circuit breaker -// operations in observability systems. func (cb *CB[Res]) Do(ctx context.Context, fn func(context.Context) (Res, error)) (res Res, err error) { ctx, span := tracing.Start(ctx, fmt.Sprintf("circuitbreaker.%s.Do", cb.config.name)) defer span.End() @@ -224,10 +177,9 @@ func (cb *CB[Res]) Do(ctx context.Context, fn func(context.Context) (Res, error) } -// preflight checks if the circuit is ready to accept a request. -// It updates internal counters and state based on configured intervals. +// preflight checks if the circuit is ready to accept a request func (cb *CB[Res]) preflight(ctx context.Context) error { - _, span := tracing.Start(ctx, fmt.Sprintf("circuitbreaker.%s.preflight", cb.config.name)) + _, span := tracing.Start(ctx, fmt.Sprintf("circuitbreaker.%s.preflight", cb.config.name)) // nolint:ineffassign // Context is used by tracing defer span.End() cb.Lock() defer cb.Unlock() @@ -247,25 +199,22 @@ func (cb *CB[Res]) preflight(ctx context.Context) error { cb.resetStateAt = now.Add(cb.config.timeout) } + requests.WithLabelValues(cb.config.name, string(cb.state)).Inc() + if cb.state == Open { return ErrTripped } - cb.logger.Debug("circuit breaker state", - slog.String("state", string(cb.state)), - slog.Int("requests", cb.requests), - slog.Int("maxRequests", cb.config.maxRequests), - ) + cb.logger.Debug("circuit breaker state", "state", string(cb.state), "requests", cb.requests, "maxRequests", cb.config.maxRequests) if cb.state == HalfOpen && cb.requests >= cb.config.maxRequests { return ErrTooManyRequests } return nil } -// postflight updates the circuit breaker state based on the result of the request. -// It tracks successes and failures to determine whether to open or close the circuit. +// postflight updates the circuit breaker state based on the result of the request func (cb *CB[Res]) postflight(ctx context.Context, err error) { - _, span := tracing.Start(ctx, fmt.Sprintf("circuitbreaker.%s.postflight", cb.config.name)) + _, span := tracing.Start(ctx, fmt.Sprintf("circuitbreaker.%s.postflight", cb.config.name)) // nolint:ineffassign // Context is used by tracing defer span.End() cb.Lock() defer cb.Unlock() @@ -280,9 +229,8 @@ func (cb *CB[Res]) postflight(ctx context.Context, err error) { cb.consecutiveFailures = 0 } - metrics.CircuitBreakerRequests.WithLabelValues(cb.config.name, string(cb.state)).Inc() - switch cb.state { + case Closed: if cb.failures >= cb.config.tripThreshold { cb.state = Open diff --git a/go/pkg/circuitbreaker/lib_test.go b/go/pkg/circuitbreaker/lib_test.go index 557764010ad..f632c0cfeaa 100644 --- a/go/pkg/circuitbreaker/lib_test.go +++ b/go/pkg/circuitbreaker/lib_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/unkeyed/unkey/go/pkg/clock" + "github.com/unkeyed/unkey/go/pkg/otel/logging" ) var errTestDownstream = errors.New("downstream test error") @@ -15,7 +16,7 @@ var errTestDownstream = errors.New("downstream test error") func TestCircuitBreakerStates(t *testing.T) { c := clock.NewTestClock() - cb := New[int]("test", WithCyclicPeriod(5*time.Second), WithClock(c), WithTripThreshold(3)) + cb := New[int]("test", WithCyclicPeriod(5*time.Second), WithClock(c), WithTripThreshold(3), WithLogger(logging.NewNoop())) // Test Closed State for i := 0; i < 3; i++ { diff --git a/go/pkg/circuitbreaker/metrics.go b/go/pkg/circuitbreaker/metrics.go new file mode 100644 index 00000000000..490cb0332f6 --- /dev/null +++ b/go/pkg/circuitbreaker/metrics.go @@ -0,0 +1,15 @@ +package circuitbreaker + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + requests = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "agent", + Subsystem: "circuitbreaker", + Name: "requests_total", + Help: "Total number of requests processed by circuit breaker", + }, []string{"name", "state"}) +) diff --git a/go/pkg/clock/interface.go b/go/pkg/clock/interface.go index 97e12866fe4..94f8b590571 100644 --- a/go/pkg/clock/interface.go +++ b/go/pkg/clock/interface.go @@ -2,7 +2,7 @@ package clock import "time" -// Clock is an interface for getting the current time. +// Clock is an interface for getting the current time and creating tickers. // By abstracting time operations behind this interface, code can be written // that works with both the system clock in production and controlled time // in tests, enabling deterministic testing of time-dependent logic. @@ -15,4 +15,21 @@ type Clock interface { // In test implementations, this returns a controlled time that // can be manipulated for testing purposes. Now() time.Time + + // NewTicker returns a new Ticker containing a channel that will send + // the current time on the channel after each tick. + // In production implementations, this creates a real ticker. + // In test implementations, this creates a controllable ticker. + NewTicker(d time.Duration) Ticker +} + +// Ticker represents a ticker that sends the current time at regular intervals. +// This interface abstracts the standard library's time.Ticker to enable +// deterministic testing of ticker-based code. +type Ticker interface { + // C returns the channel on which the ticks are delivered. + C() <-chan time.Time + + // Stop turns off a ticker. After Stop, no more ticks will be sent. + Stop() } diff --git a/go/pkg/clock/real_clock.go b/go/pkg/clock/real_clock.go index ca7aa8fad74..cf9c4ab7d77 100644 --- a/go/pkg/clock/real_clock.go +++ b/go/pkg/clock/real_clock.go @@ -27,3 +27,25 @@ var _ Clock = &RealClock{} func (c *RealClock) Now() time.Time { return time.Now() } + +// NewTicker returns a new real ticker that sends the current time +// on the channel after each tick. This implementation delegates +// to time.NewTicker(). +func (c *RealClock) NewTicker(d time.Duration) Ticker { + return &realTicker{ticker: time.NewTicker(d)} +} + +// realTicker wraps time.Ticker to implement the Ticker interface +type realTicker struct { + ticker *time.Ticker +} + +// C returns the channel on which the ticks are delivered. +func (t *realTicker) C() <-chan time.Time { + return t.ticker.C +} + +// Stop turns off the ticker. +func (t *realTicker) Stop() { + t.ticker.Stop() +} diff --git a/go/pkg/clock/test_clock.go b/go/pkg/clock/test_clock.go index 6d701d0edbf..dcff8f9d0b6 100644 --- a/go/pkg/clock/test_clock.go +++ b/go/pkg/clock/test_clock.go @@ -9,8 +9,9 @@ import ( // It allows tests to manually set and advance time to create deterministic // test scenarios for time-dependent code. type TestClock struct { - mu sync.RWMutex - now time.Time + mu sync.RWMutex + now time.Time + tickers []*testTicker } // NewTestClock creates a new TestClock instance. @@ -29,7 +30,7 @@ func NewTestClock(now ...time.Time) *TestClock { if len(now) == 0 { now = append(now, time.Now()) } - return &TestClock{mu: sync.RWMutex{}, now: now[0]} + return &TestClock{mu: sync.RWMutex{}, now: now[0], tickers: []*testTicker{}} } // Ensure TestClock implements the Clock interface @@ -46,7 +47,8 @@ func (c *TestClock) Now() time.Time { // Tick advances the clock by the given duration and returns the new time. // This method is particularly useful for testing time-dependent behavior -// without waiting for real time to pass. +// without waiting for real time to pass. It also triggers any tickers +// that should fire during this time advancement. // // Example: // @@ -61,12 +63,19 @@ func (c *TestClock) Tick(d time.Duration) time.Time { c.mu.Lock() defer c.mu.Unlock() c.now = c.now.Add(d) + + // Notify all tickers about the time advancement + for _, ticker := range c.tickers { + ticker.checkForTick(c.now) + } + return c.now } // Set changes the clock to the given time and returns the new time. // This allows tests to jump to specific points in time for testing -// time-dependent behavior. +// time-dependent behavior. It also triggers any tickers that should +// fire during this time change. // // Example: // @@ -79,5 +88,98 @@ func (c *TestClock) Set(t time.Time) time.Time { c.mu.Lock() defer c.mu.Unlock() c.now = t + + // Notify all tickers about the time change + for _, ticker := range c.tickers { + ticker.checkForTick(c.now) + } + return c.now } + +// NewTicker returns a new test ticker that can be manually controlled. +// The ticker will only send ticks when the clock is advanced using Tick() or Set(). +// This enables deterministic testing of ticker-based code. +func (c *TestClock) NewTicker(d time.Duration) Ticker { + c.mu.Lock() + defer c.mu.Unlock() + + ch := make(chan time.Time, 1) // Buffered to prevent blocking + ticker := &testTicker{ + mu: sync.Mutex{}, + clock: c, + interval: d, + lastTick: c.now, + ch: ch, + stopped: false, + } + + // Register this ticker with the clock + c.tickers = append(c.tickers, ticker) + + return ticker +} + +// testTicker implements a controllable ticker for testing +type testTicker struct { + clock *TestClock + interval time.Duration + lastTick time.Time + ch chan time.Time + stopped bool + mu sync.Mutex +} + +// C returns the channel on which the ticks are delivered. +func (t *testTicker) C() <-chan time.Time { + return t.ch +} + +// Stop turns off the ticker. +func (t *testTicker) Stop() { + t.mu.Lock() + defer t.mu.Unlock() + t.stopped = true + close(t.ch) + + // Remove this ticker from the clock's list + t.clock.removeTicker(t) +} + +// removeTicker removes a ticker from the clock's list (internal method) +func (c *TestClock) removeTicker(tickerToRemove *testTicker) { + c.mu.Lock() + defer c.mu.Unlock() + + // Find and remove the ticker + for i, ticker := range c.tickers { + if ticker == tickerToRemove { + // Remove by swapping with last element and truncating + c.tickers[i] = c.tickers[len(c.tickers)-1] + c.tickers = c.tickers[:len(c.tickers)-1] + break + } + } +} + +// checkForTick checks if enough time has passed to send a tick. +// This is called internally when the clock advances. +func (t *testTicker) checkForTick(currentTime time.Time) { + t.mu.Lock() + defer t.mu.Unlock() + + if t.stopped { + return + } + + // Check if enough time has passed since last tick + if currentTime.Sub(t.lastTick) >= t.interval { + // Send tick if channel has space (non-blocking) + select { + case t.ch <- currentTime: + t.lastTick = currentTime + default: + // Channel is full, skip this tick (mimics real ticker behavior) + } + } +} diff --git a/go/pkg/hydra/README.md b/go/pkg/hydra/README.md new file mode 100644 index 00000000000..5c38b8f594c --- /dev/null +++ b/go/pkg/hydra/README.md @@ -0,0 +1,519 @@ +# Hydra 🌊 + +> **Distributed workflow orchestration engine for Go** + +Hydra is a robust, scalable workflow orchestration engine designed for reliable execution of multi-step business processes. Built with exactly-once execution guarantees, automatic retries, and comprehensive observability. + +## Features + +🚀 **Exactly-Once Execution** - Workflows and steps execute exactly once, even with failures +⚡ **Durable State** - All state persisted to database, survives crashes and restarts +🔄 **Automatic Retries** - Configurable retry policies with exponential backoff +📊 **Rich Observability** - Built-in Prometheus metrics and structured logging +⏰ **Flexible Scheduling** - Immediate execution, cron schedules, and sleep states +🏗️ **Distributed Coordination** - Multiple workers with lease-based coordination +🎯 **Type Safety** - Strongly-typed workflows with compile-time guarantees +🔧 **Checkpointing** - Automatic step result caching for fault tolerance + +## Quick Start + +### Installation + +```bash +go get github.com/unkeyed/unkey/go/pkg/hydra +``` + +### Basic Example + +```go +package main + +import ( + "context" + "fmt" + "time" + + "github.com/unkeyed/unkey/go/pkg/clock" + "github.com/unkeyed/unkey/go/pkg/hydra" + "github.com/unkeyed/unkey/go/pkg/hydra/store/gorm" + "gorm.io/driver/mysql" + gormDriver "gorm.io/gorm" +) + +// Define your workflow +type OrderWorkflow struct{} + +func (w *OrderWorkflow) Name() string { + return "order-processing" +} + +func (w *OrderWorkflow) Run(ctx hydra.WorkflowContext, req *OrderRequest) error { + // Step 1: Validate payment + payment, err := hydra.Step(ctx, "validate-payment", func(stepCtx context.Context) (*Payment, error) { + return validatePayment(stepCtx, req.PaymentID) + }) + if err != nil { + return err + } + + // Step 2: Reserve inventory + _, err = hydra.Step(ctx, "reserve-inventory", func(stepCtx context.Context) (*Reservation, error) { + return reserveInventory(stepCtx, req.Items) + }) + if err != nil { + return err + } + + // Step 3: Process order + _, err = hydra.Step(ctx, "process-order", func(stepCtx context.Context) (*Order, error) { + return processOrder(stepCtx, payment, req.Items) + }) + + return err +} + +func main() { + // Set up database + db, err := gormDriver.Open(mysql.Open("dsn"), &gormDriver.Config{}) + if err != nil { + panic(err) + } + + // Create store + store := hydra.NewGORMStore(db, clock.New()) + + // Create engine + engine := hydra.New(hydra.Config{ + Store: store, + Namespace: "production", + }) + + // Create worker + worker, err := hydra.NewWorker(engine, hydra.WorkerConfig{ + WorkerID: "worker-1", + Concurrency: 10, + }) + if err != nil { + panic(err) + } + + // Register workflow + err = hydra.RegisterWorkflow(worker, &OrderWorkflow{}) + if err != nil { + panic(err) + } + + // Start worker + ctx := context.Background() + err = worker.Start(ctx) + if err != nil { + panic(err) + } + defer worker.Shutdown(ctx) + + // Submit workflow + executionID, err := engine.StartWorkflow(ctx, "order-processing", &OrderRequest{ + CustomerID: "cust_123", + Items: []Item{{SKU: "item_456", Quantity: 2}}, + PaymentID: "pay_789", + }) + if err != nil { + panic(err) + } + + fmt.Printf("Started workflow: %s\n", executionID) +} +``` + +## Core Concepts + +### Engine +The central orchestration component that manages workflow lifecycle and coordinates execution across workers. + +```go +engine := hydra.New(hydra.Config{ + Store: store, + Namespace: "production", + Logger: logger, +}) +``` + +### Workers +Distributed processing units that poll for workflows, acquire leases, and execute workflow logic. + +```go +worker, err := hydra.NewWorker(engine, hydra.WorkerConfig{ + WorkerID: "worker-1", + Concurrency: 20, + PollInterval: 100 * time.Millisecond, + HeartbeatInterval: 30 * time.Second, + ClaimTimeout: 5 * time.Minute, +}) +``` + +### Workflows +Business logic containers that define a series of steps with exactly-once execution guarantees. + +```go +type MyWorkflow struct{} + +func (w *MyWorkflow) Name() string { return "my-workflow" } + +func (w *MyWorkflow) Run(ctx hydra.WorkflowContext, req *MyRequest) error { + // Implement your business logic using hydra.Step() + return nil +} +``` + +### Steps +Individual units of work with automatic checkpointing and retry logic. + +```go +result, err := hydra.Step(ctx, "api-call", func(stepCtx context.Context) (*APIResponse, error) { + return apiClient.Call(stepCtx, request) +}) +``` + +## Advanced Features + +### Sleep States +Suspend workflows for time-based coordination: + +```go +// Sleep for 24 hours for manual approval +err = hydra.Sleep(ctx, 24*time.Hour) +if err != nil { + return err +} + +// Continue after sleep +result, err := hydra.Step(ctx, "post-approval", func(stepCtx context.Context) (string, error) { + return processApprovedRequest(stepCtx) +}) +``` + +### Cron Scheduling +Schedule workflows to run automatically: + +```go +err = engine.RegisterCron("0 0 * * *", "daily-report", func(ctx context.Context) error { + // Generate daily report + return generateDailyReport(ctx) +}) +``` + +### Error Handling & Retries +Configure retry behavior per workflow: + +```go +executionID, err := engine.StartWorkflow(ctx, "order-processing", request, + hydra.WithMaxAttempts(5), + hydra.WithRetryBackoff(2*time.Second), + hydra.WithTimeout(10*time.Minute), +) +``` + +### Custom Marshallers +Use custom serialization formats: + +```go +type ProtobufMarshaller struct{} + +func (p *ProtobufMarshaller) Marshal(v any) ([]byte, error) { + // Implement protobuf marshalling +} + +func (p *ProtobufMarshaller) Unmarshal(data []byte, v any) error { + // Implement protobuf unmarshalling +} + +engine := hydra.New(hydra.Config{ + Store: store, + Marshaller: &ProtobufMarshaller{}, +}) +``` + +## Observability + +### Prometheus Metrics + +Hydra provides comprehensive metrics out of the box: + +**Workflow Metrics:** +- `hydra_workflows_started_total` - Total workflows started +- `hydra_workflows_completed_total` - Total workflows completed/failed +- `hydra_workflow_duration_seconds` - Workflow execution time +- `hydra_workflow_queue_time_seconds` - Time spent waiting for execution +- `hydra_workflows_active` - Currently running workflows per worker + +**Step Metrics:** +- `hydra_steps_executed_total` - Total steps executed with status +- `hydra_step_duration_seconds` - Individual step execution time +- `hydra_steps_cached_total` - Steps served from checkpoint cache +- `hydra_steps_retried_total` - Step retry attempts + +**Worker Metrics:** +- `hydra_worker_polls_total` - Worker polling operations +- `hydra_worker_heartbeats_total` - Worker heartbeat operations +- `hydra_lease_acquisitions_total` - Workflow lease acquisitions +- `hydra_worker_concurrency_current` - Current workflow concurrency per worker + +### Example Grafana Queries + +```promql +# Workflow throughput +rate(hydra_workflows_completed_total[5m]) + +# Average workflow duration +rate(hydra_workflow_duration_seconds_sum[5m]) / rate(hydra_workflow_duration_seconds_count[5m]) + +# Step cache hit rate +rate(hydra_steps_cached_total[5m]) / rate(hydra_steps_executed_total[5m]) + +# Worker utilization +hydra_workflows_active / hydra_worker_concurrency_current +``` + +## Architecture + +Hydra uses a lease-based coordination model for distributed execution: + +``` +┌─────────────┐ ┌─────────────┐ ┌─────────────┐ +│ Worker 1 │ │ Worker 2 │ │ Worker N │ +│ │ │ │ │ │ +│ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ +│ │ Poll │ │ │ │ Poll │ │ │ │ Poll │ │ +│ │ Execute │ │ │ │ Execute │ │ │ │ Execute │ │ +│ │ Heartbeat│ │ │ │ Heartbeat│ │ │ │ Heartbeat│ │ +│ └─────────┘ │ │ └─────────┘ │ │ └─────────┘ │ +└─────────────┘ └─────────────┘ └─────────────┘ + │ │ │ + └───────────────────┼───────────────────┘ + │ + ┌─────────────────┐ + │ Database │ + │ │ + │ • Workflows │ + │ • Steps │ + │ • Leases │ + │ • Cron Jobs │ + └─────────────────┘ +``` + +1. **Workers poll** the database for pending workflows +2. **Workers acquire leases** on available workflows for exclusive execution +3. **Workers execute** workflow logic with step-by-step checkpointing +4. **Workers send heartbeats** to maintain lease ownership +5. **Completed workflows** update status and release leases + +## Database Schema + +Hydra requires the following tables (auto-migrated with GORM): + +```sql +-- Workflow executions +CREATE TABLE workflow_executions ( + id VARCHAR(255) PRIMARY KEY, + workflow_name VARCHAR(255) NOT NULL, + status VARCHAR(50) NOT NULL, + namespace VARCHAR(255) NOT NULL, + input_data LONGBLOB, + output_data LONGBLOB, + error_message TEXT, + max_attempts INT NOT NULL, + remaining_attempts INT NOT NULL, + created_at BIGINT NOT NULL, + started_at BIGINT, + completed_at BIGINT, + trigger_type VARCHAR(50), + trigger_source VARCHAR(255), + INDEX idx_workflow_executions_status_namespace (status, namespace), + INDEX idx_workflow_executions_workflow_name (workflow_name) +); + +-- Workflow steps +CREATE TABLE workflow_steps ( + id VARCHAR(255) PRIMARY KEY, + execution_id VARCHAR(255) NOT NULL, + step_name VARCHAR(255) NOT NULL, + step_order INT NOT NULL, + status VARCHAR(50) NOT NULL, + namespace VARCHAR(255) NOT NULL, + input_data LONGBLOB, + output_data LONGBLOB, + error_message TEXT, + max_attempts INT NOT NULL, + remaining_attempts INT NOT NULL, + started_at BIGINT, + completed_at BIGINT, + UNIQUE KEY unique_execution_step (execution_id, step_name), + INDEX idx_workflow_steps_execution_id (execution_id) +); + +-- Leases for coordination +CREATE TABLE leases ( + resource_id VARCHAR(255) PRIMARY KEY, + kind VARCHAR(50) NOT NULL, + namespace VARCHAR(255) NOT NULL, + worker_id VARCHAR(255) NOT NULL, + acquired_at BIGINT NOT NULL, + expires_at BIGINT NOT NULL, + heartbeat_at BIGINT NOT NULL, + INDEX idx_leases_expires_at (expires_at), + INDEX idx_leases_worker_id (worker_id) +); + +-- Cron jobs +CREATE TABLE cron_jobs ( + id VARCHAR(255) PRIMARY KEY, + name VARCHAR(255) NOT NULL, + cron_spec VARCHAR(255) NOT NULL, + namespace VARCHAR(255) NOT NULL, + workflow_name VARCHAR(255), + enabled BOOLEAN NOT NULL DEFAULT TRUE, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + next_run_at BIGINT NOT NULL, + UNIQUE KEY unique_namespace_name (namespace, name), + INDEX idx_cron_jobs_next_run_at (next_run_at, enabled) +); +``` + +## Performance Considerations + +### Scaling Workers +- **Horizontal scaling**: Add more worker instances +- **Vertical scaling**: Increase concurrency per worker +- **Database optimization**: Ensure proper indexing and connection pooling + +### Optimizing Workflows +- **Idempotent steps**: Ensure steps can be safely retried +- **Minimize step payload size**: Reduce serialization overhead +- **Batch operations**: Combine multiple operations in single steps +- **Use appropriate timeouts**: Balance responsiveness vs. reliability + +### Database Tuning +```sql +-- Recommended indexes for performance +CREATE INDEX idx_workflow_executions_polling +ON workflow_executions (status, namespace, created_at); + +CREATE INDEX idx_leases_cleanup +ON leases (expires_at); + +CREATE INDEX idx_workflow_steps_execution_order +ON workflow_steps (execution_id, step_order); +``` + +## Best Practices + +### Workflow Design +- ✅ **Keep workflows stateless** - Store state in steps, not workflow instances +- ✅ **Make steps idempotent** - Steps should be safe to retry +- ✅ **Use descriptive step names** - Names should be stable across deployments +- ✅ **Handle errors gracefully** - Distinguish between retryable and permanent errors +- ✅ **Minimize external dependencies** - Use timeouts and circuit breakers + +### Production Deployment +- ✅ **Monitor metrics** - Set up alerts for error rates and latency +- ✅ **Configure retries** - Set appropriate retry policies for your use case +- ✅ **Database backup** - Ensure workflow state is backed up +- ✅ **Graceful shutdown** - Handle SIGTERM to finish active workflows +- ✅ **Resource limits** - Set memory and CPU limits for workers + +## Examples + +### Order Processing Workflow +```go +type OrderWorkflow struct { + paymentService PaymentService + inventoryService InventoryService + shippingService ShippingService +} + +func (w *OrderWorkflow) Run(ctx hydra.WorkflowContext, req *OrderRequest) error { + // Validate and charge payment + payment, err := hydra.Step(ctx, "process-payment", func(stepCtx context.Context) (*Payment, error) { + return w.paymentService.ProcessPayment(stepCtx, &PaymentRequest{ + Amount: req.TotalAmount, + Method: req.PaymentMethod, + Customer: req.CustomerID, + }) + }) + if err != nil { + return err + } + + // Reserve inventory + reservation, err := hydra.Step(ctx, "reserve-inventory", func(stepCtx context.Context) (*Reservation, error) { + return w.inventoryService.ReserveItems(stepCtx, req.Items) + }) + if err != nil { + // Refund payment on inventory failure + hydra.Step(ctx, "refund-payment", func(stepCtx context.Context) (any, error) { + return nil, w.paymentService.RefundPayment(stepCtx, payment.ID) + }) + return err + } + + // Create shipping label + _, err = hydra.Step(ctx, "create-shipping", func(stepCtx context.Context) (*ShippingLabel, error) { + return w.shippingService.CreateLabel(stepCtx, &ShippingRequest{ + Address: req.ShippingAddress, + Items: req.Items, + Reservation: reservation.ID, + }) + }) + + return err +} +``` + +### Approval Workflow with Sleep +```go +func (w *ApprovalWorkflow) Run(ctx hydra.WorkflowContext, req *ApprovalRequest) error { + // Submit for review + _, err := hydra.Step(ctx, "submit-review", func(stepCtx context.Context) (*Review, error) { + return w.reviewService.SubmitForReview(stepCtx, req) + }) + if err != nil { + return err + } + + // Sleep for 48 hours to allow manual review + err = hydra.Sleep(ctx, 48*time.Hour) + if err != nil { + return err + } + + // Check approval status + approval, err := hydra.Step(ctx, "check-approval", func(stepCtx context.Context) (*Approval, error) { + return w.reviewService.GetApprovalStatus(stepCtx, req.ID) + }) + if err != nil { + return err + } + + if approval.Status == "approved" { + // Process approved request + _, err = hydra.Step(ctx, "process-approved", func(stepCtx context.Context) (any, error) { + return nil, w.processApprovedRequest(stepCtx, req) + }) + } + + return err +} +``` + +## Contributing + +We welcome contributions! Please see our [Contributing Guide](../../CONTRIBUTING.md) for details. + +## License + +This project is licensed under the MIT License - see the [LICENSE](../../LICENSE) file for details. + +--- + +**Need help?** Check out our [documentation](https://docs.unkey.com) or join our [Discord community](https://discord.gg/unkey). \ No newline at end of file diff --git a/go/pkg/hydra/chaos_simulation_test.go b/go/pkg/hydra/chaos_simulation_test.go new file mode 100644 index 00000000000..584b602e495 --- /dev/null +++ b/go/pkg/hydra/chaos_simulation_test.go @@ -0,0 +1,474 @@ +package hydra + +import ( + "context" + "fmt" + "math/rand" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/unkeyed/unkey/go/pkg/clock" + "github.com/unkeyed/unkey/go/pkg/hydra/store" +) + +// ChaosSimulator orchestrates chaos testing scenarios +type ChaosSimulator struct { + engine *Engine + workers []Worker + workflowMetrics map[string]*WorkflowMetrics + mu sync.RWMutex + + // Chaos controls + databaseFailureRate atomic.Value // float64 + workerCrashRate atomic.Value // float64 + networkPartitionRate atomic.Value // float64 + slowQueryRate atomic.Value // float64 + + // Tracking + workerCrashes atomic.Int64 + databaseFailures atomic.Int64 + workflowsStarted atomic.Int64 +} + +func NewChaosSimulator(engine *Engine) *ChaosSimulator { + cs := &ChaosSimulator{ + engine: engine, + workers: make([]Worker, 0), + workflowMetrics: make(map[string]*WorkflowMetrics), + } + + // Initialize rates + cs.databaseFailureRate.Store(0.0) + cs.workerCrashRate.Store(0.0) + cs.networkPartitionRate.Store(0.0) + cs.slowQueryRate.Store(0.0) + + return cs +} + +// TestChaosSimulation runs a comprehensive chaos simulation with complex workflows +func TestChaosSimulation(t *testing.T) { + realClock := clock.New() + engine := newTestEngineWithClock(t, realClock) + + simulator := NewChaosSimulator(engine) + + // Create metrics for each workflow type + billingMetrics := NewWorkflowMetrics() + pipelineMetrics := NewWorkflowMetrics() + stateMachineMetrics := NewWorkflowMetrics() + + // Register complex workflows + billingWorkflow := &ComplexBillingWorkflow{ + engine: engine, + name: "complex-billing-workflow", + failureRate: 0.1, // 10% step failure rate + chaosEnabled: true, + metrics: billingMetrics, + } + + pipelineWorkflow := &ComplexDataPipelineWorkflow{ + engine: engine, + name: "complex-pipeline-workflow", + chaosEnabled: true, + metrics: pipelineMetrics, + } + + stateMachineWorkflow := &ComplexStateMachineWorkflow{ + engine: engine, + name: "complex-state-machine-workflow", + chaosEnabled: true, + metrics: stateMachineMetrics, + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + // Phase 1: Start with stable conditions + numWorkers := 5 + for i := 0; i < numWorkers; i++ { + worker, err := simulator.CreateWorker(fmt.Sprintf("chaos-worker-%d", i)) + require.NoError(t, err) + + // Register all workflows with each worker + err = RegisterWorkflow(worker, billingWorkflow) + require.NoError(t, err) + err = RegisterWorkflow(worker, pipelineWorkflow) + require.NoError(t, err) + err = RegisterWorkflow(worker, stateMachineWorkflow) + require.NoError(t, err) + + err = worker.Start(ctx) + require.NoError(t, err) + simulator.workers = append(simulator.workers, worker) + } + + // Submit initial workflows + for i := 0; i < 20; i++ { + // Mix of different workflow types + var err error + + switch i % 3 { + case 0: + _, err = billingWorkflow.Start(ctx, fmt.Sprintf("billing-%d", i)) + case 1: + _, err = pipelineWorkflow.Start(ctx, fmt.Sprintf("pipeline-%d", i)) + case 2: + _, err = stateMachineWorkflow.Start(ctx, fmt.Sprintf("state-%d", i)) + } + + require.NoError(t, err) + simulator.workflowsStarted.Add(1) + } + + // Let workflows run for a bit + time.Sleep(5 * time.Second) + + // Phase 2: Introduce moderate chaos + simulator.databaseFailureRate.Store(0.05) // 5% database failures + simulator.workerCrashRate.Store(0.02) // 2% worker crash rate + simulator.slowQueryRate.Store(0.1) // 10% slow queries + + // Submit more workflows under chaos + for i := 0; i < 30; i++ { + var err error + + switch rand.Intn(3) { + case 0: + _, err = billingWorkflow.Start(ctx, fmt.Sprintf("chaos-billing-%d", i)) + case 1: + _, err = pipelineWorkflow.Start(ctx, fmt.Sprintf("chaos-pipeline-%d", i)) + case 2: + _, err = stateMachineWorkflow.Start(ctx, fmt.Sprintf("chaos-state-%d", i)) + } + + if err != nil { + // Database might be failing + simulator.databaseFailures.Add(1) + continue + } + simulator.workflowsStarted.Add(1) + + // Randomly crash a worker + if rand.Float64() < simulator.workerCrashRate.Load().(float64) { + simulator.CrashRandomWorker(t) + } + + time.Sleep(200 * time.Millisecond) + } + + // Phase 3: Extreme chaos + simulator.databaseFailureRate.Store(0.2) // 20% database failures + simulator.workerCrashRate.Store(0.1) // 10% worker crash rate + simulator.networkPartitionRate.Store(0.15) // 15% network partitions + simulator.slowQueryRate.Store(0.3) // 30% slow queries + + // Crash multiple workers + for i := 0; i < 2; i++ { + simulator.CrashRandomWorker(t) + time.Sleep(500 * time.Millisecond) + } + + // Submit burst of workflows + var burstWG sync.WaitGroup + for i := 0; i < 50; i++ { + burstWG.Add(1) + go func(idx int) { + defer burstWG.Done() + + workflowType := rand.Intn(3) + var err error + + switch workflowType { + case 0: + _, err = billingWorkflow.Start(ctx, fmt.Sprintf("burst-billing-%d", idx)) + case 1: + _, err = pipelineWorkflow.Start(ctx, fmt.Sprintf("burst-pipeline-%d", idx)) + case 2: + _, err = stateMachineWorkflow.Start(ctx, fmt.Sprintf("burst-state-%d", idx)) + } + + if err == nil { + simulator.workflowsStarted.Add(1) + } else { + simulator.databaseFailures.Add(1) + } + }(i) + + if i%10 == 0 { + time.Sleep(100 * time.Millisecond) + } + } + burstWG.Wait() + + // Phase 4: Recovery + simulator.databaseFailureRate.Store(0.0) + simulator.workerCrashRate.Store(0.0) + simulator.networkPartitionRate.Store(0.0) + simulator.slowQueryRate.Store(0.0) + + // Restart crashed workers + numActiveWorkers := len(simulator.workers) // Simplified for now + + // Add replacement workers + for i := numActiveWorkers; i < numWorkers; i++ { + worker, err := simulator.CreateWorker(fmt.Sprintf("recovery-worker-%d", i)) + require.NoError(t, err) + + err = RegisterWorkflow(worker, billingWorkflow) + require.NoError(t, err) + err = RegisterWorkflow(worker, pipelineWorkflow) + require.NoError(t, err) + err = RegisterWorkflow(worker, stateMachineWorkflow) + require.NoError(t, err) + + err = worker.Start(ctx) + require.NoError(t, err) + } + + // Wait for system to stabilize + time.Sleep(10 * time.Second) + + // Analyze results + + // Check workflow completion rates + pendingWorkflows := 0 + completedWorkflows := 0 + failedWorkflows := 0 + + workflows, err := engine.store.GetAllWorkflows(ctx, engine.GetNamespace()) + require.NoError(t, err) + + for _, wf := range workflows { + switch wf.Status { + case store.WorkflowStatusPending, store.WorkflowStatusRunning: + pendingWorkflows++ + case store.WorkflowStatusCompleted: + completedWorkflows++ + case store.WorkflowStatusFailed: + failedWorkflows++ + case store.WorkflowStatusSleeping: + // Sleeping workflows count as pending for our metrics + } + } + + // Workflow metrics + + // Chaos metrics + + // Assertions + require.Greater(t, completedWorkflows, 0, "At least some workflows should complete") + require.Less(t, float64(failedWorkflows)/float64(len(workflows)), 0.5, + "Less than 50% of workflows should fail even under extreme chaos") + + // Check for data consistency + stepExecutions := make(map[string]int) + steps, err := engine.store.GetAllSteps(ctx, engine.GetNamespace()) + require.NoError(t, err) + + for _, step := range steps { + key := fmt.Sprintf("%s-%s", step.ExecutionID, step.StepName) + stepExecutions[key]++ + } + + // Verify no duplicate step executions + duplicateSteps := 0 + for stepKey, count := range stepExecutions { + if count > 1 { + duplicateSteps++ + t.Errorf("Duplicate step execution detected: %s (count: %d)", stepKey, count) + } + } + + require.Equal(t, 0, duplicateSteps, "No steps should be executed more than once") + +} + +func (cs *ChaosSimulator) CreateWorker(workerID string) (Worker, error) { + return NewWorker(cs.engine, WorkerConfig{ + WorkerID: workerID, + Concurrency: 3, + PollInterval: 100 * time.Millisecond, + HeartbeatInterval: 2 * time.Second, + ClaimTimeout: 30 * time.Second, + }) +} + +func (cs *ChaosSimulator) CrashRandomWorker(t *testing.T) { + cs.mu.Lock() + defer cs.mu.Unlock() + + if len(cs.workers) == 0 { + return + } + + // For now, just log that we would crash a worker + // Actual worker crashing would require access to internal fields + cs.workerCrashes.Add(1) +} + +// TestDatabaseFailureScenarios specifically tests database failure handling +func TestDatabaseFailureScenarios(t *testing.T) { + realClock := clock.New() + engine := newTestEngineWithClock(t, realClock) + + // Wrap the store to inject failures + failureStore := &FailureInjectingStore{ + Store: engine.store, + failureRate: atomic.Value{}, + } + failureStore.failureRate.Store(0.0) + engine.store = failureStore + + metrics := NewWorkflowMetrics() + workflow := &ComplexBillingWorkflow{ + engine: engine, + name: "db-failure-test-workflow", + failureRate: 0.0, // No workflow failures, only DB failures + chaosEnabled: false, + metrics: metrics, + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Start worker + worker, err := NewWorker(engine, WorkerConfig{ + WorkerID: "db-failure-worker", + Concurrency: 2, + PollInterval: 100 * time.Millisecond, + HeartbeatInterval: 2 * time.Second, + ClaimTimeout: 10 * time.Second, + }) + require.NoError(t, err) + + err = RegisterWorkflow(worker, workflow) + require.NoError(t, err) + + err = worker.Start(ctx) + require.NoError(t, err) + defer worker.Shutdown(ctx) + + // Test 1: Database failures during workflow submission + failureStore.failureRate.Store(0.5) // 50% failure rate + + submissionFailures := 0 + submissionSuccesses := 0 + + for i := 0; i < 20; i++ { + _, err := workflow.Start(ctx, fmt.Sprintf("submission-test-%d", i)) + if err != nil { + submissionFailures++ + } else { + submissionSuccesses++ + } + } + + require.Greater(t, submissionFailures, 0, "Should have some submission failures") + require.Greater(t, submissionSuccesses, 0, "Should have some submission successes") + + // Test 2: Database recovery + failureStore.failureRate.Store(0.0) // Restore database + + // Submit workflows that should succeed + var recoveryWorkflows []string + for i := 0; i < 10; i++ { + workflowID, err := workflow.Start(ctx, fmt.Sprintf("recovery-test-%d", i)) + require.NoError(t, err) + recoveryWorkflows = append(recoveryWorkflows, workflowID) + } + + // Wait for processing + time.Sleep(5 * time.Second) + + // Verify workflows completed + completedCount := 0 + for _, workflowID := range recoveryWorkflows { + wf, err := engine.store.GetWorkflow(ctx, engine.GetNamespace(), workflowID) + require.NoError(t, err) + if wf.Status == store.WorkflowStatusCompleted { + completedCount++ + } + } + + require.GreaterOrEqual(t, completedCount, len(recoveryWorkflows)/2, "At least half of workflows should complete after recovery") + + // Test 3: Database failures during processing + + // Submit workflows + var processingWorkflows []string + for i := 0; i < 5; i++ { + workflowID, err := workflow.Start(ctx, fmt.Sprintf("processing-test-%d", i)) + require.NoError(t, err) + processingWorkflows = append(processingWorkflows, workflowID) + } + + // Introduce failures during processing + time.Sleep(500 * time.Millisecond) + failureStore.failureRate.Store(0.3) // 30% failure rate + + // Wait for processing attempts + time.Sleep(5 * time.Second) + + // Restore database + failureStore.failureRate.Store(0.0) + + // Wait for recovery + time.Sleep(5 * time.Second) + + // Check final state + for _, workflowID := range processingWorkflows { + _, err := engine.store.GetWorkflow(ctx, engine.GetNamespace(), workflowID) + require.NoError(t, err) + } + +} + +// FailureInjectingStore wraps a store and randomly injects failures +type FailureInjectingStore struct { + store.Store + failureRate atomic.Value // float64 +} + +func (f *FailureInjectingStore) shouldFail() bool { + rate := f.failureRate.Load().(float64) + return rand.Float64() < rate +} + +func (f *FailureInjectingStore) CreateWorkflow(ctx context.Context, workflow *store.WorkflowExecution) error { + if f.shouldFail() { + return fmt.Errorf("simulated database failure") + } + return f.Store.CreateWorkflow(ctx, workflow) +} + +func (f *FailureInjectingStore) GetPendingWorkflows(ctx context.Context, namespace string, limit int, workflowNames []string) ([]store.WorkflowExecution, error) { + if f.shouldFail() { + return nil, fmt.Errorf("simulated database failure") + } + return f.Store.GetPendingWorkflows(ctx, namespace, limit, workflowNames) +} + +func (f *FailureInjectingStore) UpdateWorkflowStatus(ctx context.Context, namespace, workflowID string, status store.WorkflowStatus, errorMsg string) error { + if f.shouldFail() { + return fmt.Errorf("simulated database failure") + } + return f.Store.UpdateWorkflowStatus(ctx, namespace, workflowID, status, errorMsg) +} + +func (f *FailureInjectingStore) CreateStep(ctx context.Context, step *store.WorkflowStep) error { + if f.shouldFail() { + return fmt.Errorf("simulated database failure") + } + return f.Store.CreateStep(ctx, step) +} + +func (f *FailureInjectingStore) UpdateStepStatus(ctx context.Context, namespace, workflowID, stepName string, status store.StepStatus, output []byte, errorMsg string) error { + if f.shouldFail() { + return fmt.Errorf("simulated database failure") + } + return f.Store.UpdateStepStatus(ctx, namespace, workflowID, stepName, status, output, errorMsg) +} diff --git a/go/pkg/hydra/circuit_breaker_test.go b/go/pkg/hydra/circuit_breaker_test.go new file mode 100644 index 00000000000..aa575ab22f8 --- /dev/null +++ b/go/pkg/hydra/circuit_breaker_test.go @@ -0,0 +1,96 @@ +package hydra + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/unkeyed/unkey/go/pkg/clock" +) + +// TestCircuitBreakerIntegration verifies that circuit breakers are properly +// integrated into the worker and protect database operations +func TestCircuitBreakerIntegration(t *testing.T) { + testClock := clock.NewTestClock() + engine := newTestEngineWithClock(t, testClock) + + // Create worker with circuit breaker protection + worker, err := NewWorker(engine, WorkerConfig{ + WorkerID: "circuit-breaker-test-worker", + Concurrency: 1, + PollInterval: 100 * time.Millisecond, + HeartbeatInterval: 5 * time.Second, + ClaimTimeout: 30 * time.Second, + }) + require.NoError(t, err) + + // Register a simple workflow + circuitTestWorkflow := &circuitBreakerTestWorkflow{ + engine: engine, + name: "circuit-breaker-workflow", + } + + err = RegisterWorkflow(worker, circuitTestWorkflow) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Start worker + err = worker.Start(ctx) + require.NoError(t, err) + defer worker.Shutdown(ctx) + + // Create a workflow to test circuit breaker protection + executionID, err := circuitTestWorkflow.Start(ctx, struct{}{}) + require.NoError(t, err) + require.NotEmpty(t, executionID) + + // Advance time to trigger worker polling + for i := 0; i < 5; i++ { + testClock.Tick(200 * time.Millisecond) + time.Sleep(10 * time.Millisecond) + } + + // Verify workflow was processed (circuit breaker didn't block) + finalWorkflow := waitForWorkflowCompletion(t, engine, executionID, 3*time.Second) + require.NotNil(t, finalWorkflow) + +} + +// TestCircuitBreakerCompilation ensures the circuit breaker types compile correctly +func TestCircuitBreakerCompilation(t *testing.T) { + testClock := clock.NewTestClock() + engine := newTestEngineWithClock(t, testClock) + + // This test primarily ensures compilation works + worker, err := NewWorker(engine, WorkerConfig{ + WorkerID: "compilation-test-worker", + Concurrency: 1, + }) + require.NoError(t, err) + require.NotNil(t, worker) + +} + +// circuitBreakerTestWorkflow is a minimal workflow for testing circuit breaker integration +type circuitBreakerTestWorkflow struct { + engine *Engine + name string +} + +func (w *circuitBreakerTestWorkflow) Name() string { + return w.name +} + +func (w *circuitBreakerTestWorkflow) Run(ctx WorkflowContext, req any) error { + _, err := Step(ctx, "circuit-breaker-step", func(context.Context) (string, error) { + return "protected", nil + }) + return err +} + +func (w *circuitBreakerTestWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} diff --git a/go/pkg/hydra/complex_workflows_test.go b/go/pkg/hydra/complex_workflows_test.go new file mode 100644 index 00000000000..c391d49cc6d --- /dev/null +++ b/go/pkg/hydra/complex_workflows_test.go @@ -0,0 +1,535 @@ +package hydra + +import ( + "context" + "fmt" + "math/rand" + "sync" + "sync/atomic" + "time" +) + +// ComplexBillingWorkflow simulates a realistic billing workflow with multiple steps, +// error handling, retries, and conditional logic +type ComplexBillingWorkflow struct { + engine *Engine + name string + failureRate float64 // Probability of step failure (0.0-1.0) + chaosEnabled bool + metrics *WorkflowMetrics +} + +// WorkflowMetrics tracks detailed execution metrics +type WorkflowMetrics struct { + StepsExecuted atomic.Int64 + StepsRetried atomic.Int64 + StepsFailed atomic.Int64 + WorkflowsCompleted atomic.Int64 + WorkflowsFailed atomic.Int64 + TotalDuration atomic.Int64 // in milliseconds + mu sync.RWMutex + StepDurations map[string][]time.Duration +} + +func NewWorkflowMetrics() *WorkflowMetrics { + return &WorkflowMetrics{ + StepDurations: make(map[string][]time.Duration), + } +} + +func (m *WorkflowMetrics) RecordStepDuration(stepName string, duration time.Duration) { + m.mu.Lock() + defer m.mu.Unlock() + m.StepDurations[stepName] = append(m.StepDurations[stepName], duration) +} + +func (w *ComplexBillingWorkflow) Name() string { + return w.name +} + +func (w *ComplexBillingWorkflow) Run(ctx WorkflowContext, req any) error { + startTime := time.Now() + defer func() { + w.metrics.TotalDuration.Add(time.Since(startTime).Milliseconds()) + }() + + // Step 1: Validate customer data + customerID, err := Step(ctx, "validate-customer", func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + + if w.shouldFail("validate-customer") { + w.metrics.StepsFailed.Add(1) + return "", fmt.Errorf("customer validation failed") + } + + // Simulate API call + time.Sleep(time.Duration(rand.Intn(50)+10) * time.Millisecond) + return "customer-123", nil + }) + + if err != nil { + // Retry with exponential backoff + w.metrics.StepsRetried.Add(1) + time.Sleep(100 * time.Millisecond) + + customerID, err = Step(ctx, "validate-customer-retry", func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + time.Sleep(time.Duration(rand.Intn(30)+20) * time.Millisecond) + return "customer-123", nil + }) + + if err != nil { + w.metrics.WorkflowsFailed.Add(1) + return fmt.Errorf("customer validation failed after retry: %w", err) + } + } + + // Step 2: Calculate invoice amount (parallel with usage fetch) + var invoiceAmount float64 + + // Use goroutines to simulate parallel step execution + var wg sync.WaitGroup + var calcErr, usageErr error + + wg.Add(2) + + // Calculate invoice in parallel + go func() { + defer wg.Done() + var amountStr string + amountStr, calcErr = Step(ctx, "calculate-invoice", func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + + if w.shouldFail("calculate-invoice") { + w.metrics.StepsFailed.Add(1) + return "", fmt.Errorf("invoice calculation error") + } + + // Simulate complex calculation + time.Sleep(time.Duration(rand.Intn(100)+50) * time.Millisecond) + amount := float64(rand.Intn(10000)+100) / 100.0 + return fmt.Sprintf("%.2f", amount), nil + }) + + if calcErr == nil { + fmt.Sscanf(amountStr, "%f", &invoiceAmount) + } + err = calcErr + }() + + // Fetch usage data in parallel + go func() { + defer wg.Done() + _, fetchErr := Step(ctx, "fetch-usage-data", func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + + if w.shouldFail("fetch-usage-data") { + w.metrics.StepsFailed.Add(1) + return "", fmt.Errorf("usage data fetch failed") + } + + // Simulate database query + time.Sleep(time.Duration(rand.Intn(80)+30) * time.Millisecond) + return fmt.Sprintf("usage-%d-units", rand.Intn(1000)), nil + }) + + usageErr = fetchErr + }() + + wg.Wait() + + if calcErr != nil || usageErr != nil { + w.metrics.WorkflowsFailed.Add(1) + return fmt.Errorf("parallel steps failed: calc=%v, usage=%v", calcErr, usageErr) + } + + // Step 3: Apply discounts (conditional) + if invoiceAmount > 100 { + discountedAmount, discountErr := Step(ctx, "apply-discounts", func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + + if w.shouldFail("apply-discounts") { + w.metrics.StepsFailed.Add(1) + return "", fmt.Errorf("discount calculation failed") + } + + // Simulate discount calculation + time.Sleep(time.Duration(rand.Intn(40)+10) * time.Millisecond) + discount := invoiceAmount * 0.1 + return fmt.Sprintf("%.2f", invoiceAmount-discount), nil + }) + + if discountErr == nil { + fmt.Sscanf(discountedAmount, "%f", &invoiceAmount) + } + } + + // Step 4: Generate PDF invoice + _, err = Step(ctx, "generate-pdf", func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + + if w.shouldFail("generate-pdf") { + w.metrics.StepsFailed.Add(1) + return "", fmt.Errorf("PDF generation failed") + } + + // Simulate PDF generation (slow operation) + time.Sleep(time.Duration(rand.Intn(200)+100) * time.Millisecond) + return fmt.Sprintf("https://invoices.example.com/%s.pdf", customerID), nil + }) + + if err != nil { + // Non-critical failure, continue + // PDF generation is optional + _ = err // Intentionally ignored + } + + // Step 5: Send invoice email + _, err = Step(ctx, "send-email", func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + + if w.shouldFail("send-email") { + w.metrics.StepsFailed.Add(1) + return "", fmt.Errorf("email sending failed") + } + + // Simulate email API call + time.Sleep(time.Duration(rand.Intn(60)+20) * time.Millisecond) + return fmt.Sprintf("email-sent-to-%s", customerID), nil + }) + + if err != nil { + // Retry email sending + w.metrics.StepsRetried.Add(1) + _, retryErr := Step(ctx, "send-email-retry", func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + time.Sleep(time.Duration(rand.Intn(40)+20) * time.Millisecond) + return "email-sent-on-retry", nil + }) + + if retryErr != nil { + w.metrics.WorkflowsFailed.Add(1) + return fmt.Errorf("email sending failed after retry: %w", retryErr) + } + } + + // Step 6: Update billing status + _, err = Step(ctx, "update-billing-status", func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + + if w.shouldFail("update-billing-status") { + w.metrics.StepsFailed.Add(1) + return "", fmt.Errorf("status update failed") + } + + // Simulate database update + time.Sleep(time.Duration(rand.Intn(30)+10) * time.Millisecond) + return "status-updated", nil + }) + + if err != nil { + w.metrics.WorkflowsFailed.Add(1) + return fmt.Errorf("billing status update failed: %w", err) + } + + w.metrics.WorkflowsCompleted.Add(1) + return nil +} + +func (w *ComplexBillingWorkflow) shouldFail(stepName string) bool { + if !w.chaosEnabled { + return false + } + + // Introduce targeted chaos for specific steps + failureRates := map[string]float64{ + "validate-customer": w.failureRate * 0.5, // Less likely to fail + "calculate-invoice": w.failureRate, + "fetch-usage-data": w.failureRate * 1.2, // More likely to fail + "generate-pdf": w.failureRate * 2.0, // Much more likely to fail + "send-email": w.failureRate * 1.5, + "update-billing-status": w.failureRate * 0.8, + } + + rate, ok := failureRates[stepName] + if !ok { + rate = w.failureRate + } + + return rand.Float64() < rate +} + +func (w *ComplexBillingWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} + +// ComplexDataPipelineWorkflow simulates a data processing pipeline with +// conditional branching, loops, and complex error handling +type ComplexDataPipelineWorkflow struct { + engine *Engine + name string + chaosEnabled bool + metrics *WorkflowMetrics +} + +func (w *ComplexDataPipelineWorkflow) Name() string { + return w.name +} + +func (w *ComplexDataPipelineWorkflow) Run(ctx WorkflowContext, req any) error { + // Step 1: Fetch data sources + sources, err := Step(ctx, "fetch-data-sources", func(stepCtx context.Context) ([]string, error) { + w.metrics.StepsExecuted.Add(1) + + // Simulate fetching multiple data sources + time.Sleep(time.Duration(rand.Intn(50)+20) * time.Millisecond) + + numSources := rand.Intn(5) + 3 + sources := make([]string, numSources) + for i := 0; i < numSources; i++ { + sources[i] = fmt.Sprintf("source-%d", i) + } + return sources, nil + }) + + if err != nil { + w.metrics.WorkflowsFailed.Add(1) + return fmt.Errorf("failed to fetch data sources: %w", err) + } + + // Step 2: Process each source (loop with error handling) + var processedCount int + for i, source := range sources { + stepName := fmt.Sprintf("process-source-%d", i) + + _, stepErr := Step(ctx, stepName, func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + + // Simulate processing with variable duration + processingTime := time.Duration(rand.Intn(100)+50) * time.Millisecond + time.Sleep(processingTime) + + // Random failures + if w.chaosEnabled && rand.Float64() < 0.2 { + w.metrics.StepsFailed.Add(1) + return "", fmt.Errorf("processing failed for %s", source) + } + + return fmt.Sprintf("processed-%s", source), nil + }) + + if stepErr != nil { + // Continue processing other sources + continue + } + processedCount++ + } + + // Step 3: Validate processing results + if processedCount < len(sources)/2 { + // Too many failures, trigger cleanup + _, cleanupErr := Step(ctx, "cleanup-failed-processing", func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + time.Sleep(50 * time.Millisecond) + return "cleanup-complete", nil + }) + + if cleanupErr != nil { + w.metrics.WorkflowsFailed.Add(1) + return fmt.Errorf("cleanup failed: %w", cleanupErr) + } + + w.metrics.WorkflowsFailed.Add(1) + return fmt.Errorf("too many source processing failures: %d/%d", processedCount, len(sources)) + } + + // Step 4: Aggregate results + _, err = Step(ctx, "aggregate-results", func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + + // Simulate complex aggregation + time.Sleep(time.Duration(rand.Intn(150)+100) * time.Millisecond) + + if w.chaosEnabled && rand.Float64() < 0.1 { + w.metrics.StepsFailed.Add(1) + return "", fmt.Errorf("aggregation failed") + } + + return fmt.Sprintf("aggregated-%d-results", processedCount), nil + }) + + if err != nil { + w.metrics.WorkflowsFailed.Add(1) + return fmt.Errorf("result aggregation failed: %w", err) + } + + // Step 5: Publish results (with circuit breaker pattern) + var publishAttempts int + for publishAttempts < 3 { + _, err = Step(ctx, fmt.Sprintf("publish-attempt-%d", publishAttempts), func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + publishAttempts++ + + // Simulate flaky external service + if w.chaosEnabled && rand.Float64() < 0.4 { + w.metrics.StepsFailed.Add(1) + return "", fmt.Errorf("publish service unavailable") + } + + time.Sleep(time.Duration(rand.Intn(80)+40) * time.Millisecond) + return "published-successfully", nil + }) + + if err == nil { + break + } + + // Exponential backoff + w.metrics.StepsRetried.Add(1) + time.Sleep(time.Duration(publishAttempts*100) * time.Millisecond) + } + + if err != nil { + w.metrics.WorkflowsFailed.Add(1) + return fmt.Errorf("failed to publish after %d attempts: %w", publishAttempts, err) + } + + w.metrics.WorkflowsCompleted.Add(1) + return nil +} + +func (w *ComplexDataPipelineWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} + +// ComplexStateMachineWorkflow tests complex state transitions and decision points +type ComplexStateMachineWorkflow struct { + engine *Engine + name string + chaosEnabled bool + metrics *WorkflowMetrics +} + +func (w *ComplexStateMachineWorkflow) Name() string { + return w.name +} + +func (w *ComplexStateMachineWorkflow) Run(ctx WorkflowContext, req any) error { + // Initialize with random state + initialState := rand.Intn(3) + + // Step 1: Determine initial action based on state + action, err := Step(ctx, "determine-initial-action", func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + + actions := []string{"process", "review", "escalate"} + return actions[initialState], nil + }) + + if err != nil { + w.metrics.WorkflowsFailed.Add(1) + return err + } + + // Step 2: Execute state machine transitions + currentState := action + transitions := 0 + maxTransitions := 10 + + for transitions < maxTransitions { + nextState, transitionErr := Step(ctx, fmt.Sprintf("transition-%d-from-%s", transitions, currentState), + func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + + // Simulate state transition logic + time.Sleep(time.Duration(rand.Intn(50)+20) * time.Millisecond) + + // Random transition failures + if w.chaosEnabled && rand.Float64() < 0.15 { + w.metrics.StepsFailed.Add(1) + return "", fmt.Errorf("transition failed from %s", currentState) + } + + // State transition rules + switch currentState { + case "process": + if rand.Float64() < 0.7 { + return "review", nil + } + return "escalate", nil + case "review": + if rand.Float64() < 0.5 { + return "approve", nil + } else if rand.Float64() < 0.8 { + return "reject", nil + } + return "process", nil + case "escalate": + if rand.Float64() < 0.6 { + return "review", nil + } + return "terminate", nil + case "approve", "reject", "terminate": + return currentState, nil // Terminal states + default: + return "error", nil + } + }) + + if transitionErr != nil { + // Handle transition failure + _, recoveryErr := Step(ctx, fmt.Sprintf("recover-transition-%d", transitions), + func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + w.metrics.StepsRetried.Add(1) + time.Sleep(30 * time.Millisecond) + return "review", nil // Safe state + }) + + if recoveryErr != nil { + w.metrics.WorkflowsFailed.Add(1) + return fmt.Errorf("state machine recovery failed: %w", recoveryErr) + } + nextState = "review" + } + + currentState = nextState + transitions++ + + // Check for terminal states + if currentState == "approve" || currentState == "reject" || currentState == "terminate" { + break + } + } + + // Step 3: Finalize based on terminal state + _, err = Step(ctx, fmt.Sprintf("finalize-%s", currentState), func(stepCtx context.Context) (string, error) { + w.metrics.StepsExecuted.Add(1) + + switch currentState { + case "approve": + time.Sleep(80 * time.Millisecond) + return "approved-and-processed", nil + case "reject": + time.Sleep(40 * time.Millisecond) + return "rejected-and-notified", nil + case "terminate": + time.Sleep(20 * time.Millisecond) + return "terminated-with-cleanup", nil + default: + return "", fmt.Errorf("invalid terminal state: %s", currentState) + } + }) + + if err != nil { + w.metrics.WorkflowsFailed.Add(1) + return fmt.Errorf("finalization failed: %w", err) + } + + w.metrics.WorkflowsCompleted.Add(1) + return nil +} + +func (w *ComplexStateMachineWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} diff --git a/go/pkg/hydra/cron.go b/go/pkg/hydra/cron.go new file mode 100644 index 00000000000..4477f554213 --- /dev/null +++ b/go/pkg/hydra/cron.go @@ -0,0 +1,229 @@ +package hydra + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" +) + +// CronHandler defines the function signature for cron job handlers +type CronHandler func(ctx context.Context, payload CronPayload) error + +type CronPayload struct { + CronJobID string `json:"cron_job_id"` + CronName string `json:"cron_name"` + ScheduledAt int64 `json:"scheduled_at"` // When this execution was scheduled + ActualRunAt int64 `json:"actual_run_at"` // When it actually ran + Namespace string `json:"namespace"` +} + +func (p CronPayload) Marshal() ([]byte, error) { + return json.Marshal(p) +} + +func (p *CronPayload) Unmarshal(data []byte) error { + return json.Unmarshal(data, p) +} + +func calculateNextRun(cronSpec string, from time.Time) int64 { + schedule, err := parseCronSpec(cronSpec) + if err != nil { + return from.Add(5 * time.Minute).UnixMilli() + } + + next := schedule.next(from) + return next.UnixMilli() +} + +type cronSchedule struct { + minute uint64 // bits 0-59 + hour uint64 // bits 0-23 + dom uint64 // bits 1-31, day of month + month uint64 // bits 1-12 + dow uint64 // bits 0-6, day of week (0=Sunday) +} + +func parseCronSpec(spec string) (*cronSchedule, error) { + fields := strings.Fields(spec) + if len(fields) != 5 { + return nil, errors.New("cron spec must have 5 fields") + } + + minute, err := parseField(fields[0], 0, 59) + if err != nil { + return nil, fmt.Errorf("invalid minute field: %w", err) + } + + hour, err := parseField(fields[1], 0, 23) + if err != nil { + return nil, fmt.Errorf("invalid hour field: %w", err) + } + + dom, err := parseField(fields[2], 1, 31) + if err != nil { + return nil, fmt.Errorf("invalid day of month field: %w", err) + } + + month, err := parseField(fields[3], 1, 12) + if err != nil { + return nil, fmt.Errorf("invalid month field: %w", err) + } + + dow, err := parseField(fields[4], 0, 6) + if err != nil { + return nil, fmt.Errorf("invalid day of week field: %w", err) + } + + return &cronSchedule{ + minute: minute, + hour: hour, + dom: dom, + month: month, + dow: dow, + }, nil +} + +func parseField(field string, minimum, maximum int) (uint64, error) { + if field == "*" { + var mask uint64 + for i := minimum; i <= maximum; i++ { + mask |= 1 << i + } + return mask, nil + } + + parts := strings.Split(field, ",") + var mask uint64 + + for _, part := range parts { + // nolint:nestif + if strings.Contains(part, "/") { + stepParts := strings.Split(part, "/") + if len(stepParts) != 2 { + return 0, errors.New("invalid step syntax") + } + + step, err := strconv.Atoi(stepParts[1]) + if err != nil || step <= 0 { + return 0, errors.New("invalid step value") + } + + rangeStart := minimum + rangeEnd := maximum + + if stepParts[0] != "*" { + if strings.Contains(stepParts[0], "-") { + rangeParts := strings.Split(stepParts[0], "-") + if len(rangeParts) != 2 { + return 0, errors.New("invalid range syntax") + } + rangeStart, err = strconv.Atoi(rangeParts[0]) + if err != nil || rangeStart < minimum || rangeStart > maximum { + return 0, errors.New("invalid range start") + } + rangeEnd, err = strconv.Atoi(rangeParts[1]) + if err != nil || rangeEnd < minimum || rangeEnd > maximum { + return 0, errors.New("invalid range end") + } + } else { + rangeStart, err = strconv.Atoi(stepParts[0]) + if err != nil || rangeStart < minimum || rangeStart > maximum { + return 0, errors.New("invalid step start value") + } + rangeEnd = rangeStart + } + } + + for i := rangeStart; i <= rangeEnd; i += step { + mask |= 1 << i + } + + } else if strings.Contains(part, "-") { + rangeParts := strings.Split(part, "-") + if len(rangeParts) != 2 { + return 0, errors.New("invalid range syntax") + } + + start, err := strconv.Atoi(rangeParts[0]) + if err != nil || start < minimum || start > maximum { + return 0, errors.New("invalid range start") + } + + end, err := strconv.Atoi(rangeParts[1]) + if err != nil || end < minimum || end > maximum { + return 0, errors.New("invalid range end") + } + + for i := start; i <= end; i++ { + mask |= 1 << i + } + + } else { + val, err := strconv.Atoi(part) + if err != nil || val < minimum || val > maximum { + return 0, errors.New("invalid single value") + } + mask |= 1 << val + } + } + + return mask, nil +} + +func (s *cronSchedule) next(t time.Time) time.Time { + next := t.Add(time.Minute).Truncate(time.Minute) + + end := t.Add(4 * 365 * 24 * time.Hour) // 4 years + + for next.Before(end) { + if s.matches(next) { + return next + } + + next = next.Add(time.Minute) + } + + return t.Add(365 * 24 * time.Hour) +} + +func (s *cronSchedule) matches(t time.Time) bool { + if s.minute&(1< 0 { + count++ + } + } + return count +} + +func (t *ConcurrentExecutionTracker) GetMissingWorkflows(allWorkflowIDs []string) []string { + t.mu.Lock() + defer t.mu.Unlock() + + var missing []string + for _, workflowID := range allWorkflowIDs { + if record, exists := t.executions[workflowID]; !exists || record.ExecutionCount == 0 { + missing = append(missing, workflowID) + } + } + return missing +} + +func (t *ConcurrentExecutionTracker) AnalyzeResults(testCtx *testing.T) ConsistencyResults { + t.mu.Lock() + defer t.mu.Unlock() + + results := ConsistencyResults{} + + for workflowID, record := range t.executions { + if record.ExecutionCount > 0 { + results.WorkflowsExecuted++ + } + + if record.ExecutionCount > 1 { + results.DuplicateExecutions++ + testCtx.Errorf("DUPLICATE EXECUTION: Workflow %s executed %d times by workers %v", + workflowID, record.ExecutionCount, record.WorkerIDs) + } + + if record.Failed { + results.FailedWorkflows++ + } + + // Detect race conditions (multiple workers starting execution within 100ms) + if len(record.Timestamps) > 1 { + for i := 1; i < len(record.Timestamps); i++ { + if record.Timestamps[i].Sub(record.Timestamps[i-1]) < 100*time.Millisecond { + results.RaceConditions++ + testCtx.Errorf("RACE CONDITION: Workflow %s had concurrent executions by %v", + workflowID, record.WorkerIDs) + break + } + } + } + } + + return results +} + +// consistencyTestWorkflow tracks executions to detect consistency violations +type consistencyTestWorkflow struct { + engine *Engine + name string + tracker *ConcurrentExecutionTracker +} + +func (w *consistencyTestWorkflow) Name() string { + return w.name +} + +func (w *consistencyTestWorkflow) Run(ctx WorkflowContext, req any) error { + workflowID := ctx.ExecutionID() + + // Record that this workflow started executing + w.tracker.RecordExecution(workflowID, "unknown-worker") // We could get worker ID from context + + // Simulate some work with a step + _, err := Step(ctx, "consistency-step", func(context.Context) (string, error) { + // Small delay to increase chance of race conditions + time.Sleep(10 * time.Millisecond) + return "consistent", nil + }) + + // Record completion + w.tracker.RecordCompletion(workflowID, err == nil) + + return err +} + +func (w *consistencyTestWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} + +// raceConditionTestWorkflow creates multiple steps to test for race conditions +type raceConditionTestWorkflow struct { + engine *Engine + name string +} + +func (w *raceConditionTestWorkflow) Name() string { + return w.name +} + +func (w *raceConditionTestWorkflow) Run(ctx WorkflowContext, req any) error { + // Create multiple steps that might race with each other + const numSteps = 20 + + // Use a WaitGroup to ensure all steps complete + var wg sync.WaitGroup + var stepErrors atomic.Int64 + + for i := 0; i < numSteps; i++ { + wg.Add(1) + go func(stepIndex int) { + defer wg.Done() + + stepName := fmt.Sprintf("race-step-%d", stepIndex) + _, err := Step(ctx, stepName, func(context.Context) (string, error) { + return fmt.Sprintf("result-%d", stepIndex), nil + }) + + if err != nil { + stepErrors.Add(1) + } + }(i) + } + + wg.Wait() + + if stepErrors.Load() > 0 { + return fmt.Errorf("race condition test failed: %d step errors", stepErrors.Load()) + } + + return nil +} + +func (w *raceConditionTestWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} + +// transactionTestWorkflow tests database transaction integrity +type transactionTestWorkflow struct { + engine *Engine + name string +} + +func (w *transactionTestWorkflow) Name() string { + return w.name +} + +func (w *transactionTestWorkflow) Run(ctx WorkflowContext, req any) error { + mode, ok := req.(string) + if !ok { + mode = "normal" + } + + // Create a step that tests transaction boundaries + _, err := Step(ctx, "transaction-step", func(stepCtx context.Context) (string, error) { + switch mode { + case "normal": + return "transaction-success", nil + case "error": + return "", fmt.Errorf("simulated step error") + default: + return "unknown-mode", nil + } + }) + + return err +} + +func (w *transactionTestWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} diff --git a/go/pkg/hydra/database_performance_test.go b/go/pkg/hydra/database_performance_test.go new file mode 100644 index 00000000000..1c01c4400e6 --- /dev/null +++ b/go/pkg/hydra/database_performance_test.go @@ -0,0 +1,311 @@ +package hydra + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/unkeyed/unkey/go/pkg/clock" + "github.com/unkeyed/unkey/go/pkg/hydra/store" +) + +// loadTestWorkflow is a minimal workflow for load testing +type loadTestWorkflow struct { + engine *Engine + name string +} + +func (w *loadTestWorkflow) Name() string { + return w.name +} + +func (w *loadTestWorkflow) Run(ctx WorkflowContext, req any) error { + // Minimal work to avoid affecting performance measurements + _, err := Step(ctx, "load-test-step", func(context.Context) (string, error) { + return "done", nil + }) + return err +} + +func (w *loadTestWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} + +// TestDatabaseQueryPerformanceUnderLoad tests how database queries perform +// when the system has thousands of pending workflows and multiple workers +// are polling concurrently. +func TestDatabaseQueryPerformanceUnderLoad(t *testing.T) { + // Arrange: Create engine with many pending workflows + testClock := clock.NewTestClock() + engine := newTestEngineWithClock(t, testClock) + + const ( + numPendingWorkflows = 1000 // Create backlog to stress query performance + numConcurrentWorkers = 10 // Multiple workers polling simultaneously + testDurationSec = 5 // Run load test for 5 seconds + ) + + // Create a large backlog of pending workflows to stress GetPendingWorkflows query + executionIDs := make([]string, numPendingWorkflows) + + loadTestWorkflow := &loadTestWorkflow{ + engine: engine, + name: "load-test-workflow", + } + + ctx := context.Background() + for i := 0; i < numPendingWorkflows; i++ { + executionID, err := loadTestWorkflow.Start(ctx, struct{}{}) + require.NoError(t, err) + executionIDs[i] = executionID + } + + // Verify workflows are pending + pendingCount, err := countPendingWorkflows(engine.store, ctx, engine.GetNamespace()) + require.NoError(t, err) + require.Equal(t, numPendingWorkflows, pendingCount, + "Should have created %d pending workflows", numPendingWorkflows) + + // Performance tracking + var totalQueries atomic.Int64 + var totalQueryTime atomic.Int64 // nanoseconds + var slowQueries atomic.Int64 // queries > 100ms + var errors atomic.Int64 + + // Act: Start multiple workers that will poll the database concurrently + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(testDurationSec)*time.Second) + defer cancel() + + var wg sync.WaitGroup + + // Start concurrent workers that stress GetPendingWorkflows query + for i := 0; i < numConcurrentWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + + for { + select { + case <-ctx.Done(): + return + default: + // Measure query performance + start := time.Now() + _, err := engine.store.GetPendingWorkflows(ctx, engine.GetNamespace(), 50, nil) + queryDuration := time.Since(start) + + totalQueries.Add(1) + totalQueryTime.Add(int64(queryDuration)) + + if err != nil { + errors.Add(1) + // Don't spam error logs during load test + if errors.Load() <= 10 { + t.Logf("Error #%d: %v", errors.Load(), err) + } + } + + // Track slow queries (potential performance issue) + if queryDuration > 100*time.Millisecond { + slowQueries.Add(1) + if slowQueries.Load() <= 10 { + t.Logf("Slow query #%d: %v", slowQueries.Load(), queryDuration) + } + } + + // Small pause to prevent overwhelming the database + time.Sleep(10 * time.Millisecond) + } + } + }() + } + + // Let the load test run + + wg.Wait() + + // Assert: Analyze performance results + finalQueries := totalQueries.Load() + finalQueryTime := totalQueryTime.Load() + finalSlowQueries := slowQueries.Load() + finalErrors := errors.Load() + + avgQueryTime := time.Duration(finalQueryTime / finalQueries) + queriesPerSecond := float64(finalQueries) / float64(testDurationSec) + slowQueryPercentage := float64(finalSlowQueries) / float64(finalQueries) * 100 + errorPercentage := float64(finalErrors) / float64(finalQueries) * 100 + + // Performance assertions + require.Less(t, avgQueryTime, 100*time.Millisecond, + "Average query time should be <100ms even with %d pending workflows", numPendingWorkflows) + + require.Less(t, slowQueryPercentage, 10.0, + "Less than 10%% of queries should be slow (>100ms), got %.1f%%", slowQueryPercentage) + + require.Less(t, errorPercentage, 5.0, + "Less than 5%% of queries should error, got %.1f%%", errorPercentage) + + require.Greater(t, queriesPerSecond, 50.0, + "Should sustain >50 queries/sec under load, got %.1f", queriesPerSecond) + +} + +// TestDatabaseIndexOptimization verifies that database queries use appropriate indexes +// and don't degrade significantly as the number of workflows grows +func TestDatabaseIndexOptimization(t *testing.T) { + testClock := clock.NewTestClock() + engine := newTestEngineWithClock(t, testClock) + ctx := context.Background() + + // Test query performance with different workflow counts + testCases := []struct { + workflowCount int + maxQueryTime time.Duration + }{ + {100, 10 * time.Millisecond}, + {1000, 50 * time.Millisecond}, + {5000, 100 * time.Millisecond}, + } + + for _, tc := range testCases { + t.Run(fmt.Sprintf("workflows_%d", tc.workflowCount), func(t *testing.T) { + // Clean slate for each test + engine = newTestEngineWithClock(t, testClock) + + // Create new workflow instance for this engine + testWorkflow := &loadTestWorkflow{ + engine: engine, + name: "index-test-workflow", + } + + // Create workflows + for i := 0; i < tc.workflowCount; i++ { + _, err := testWorkflow.Start(ctx, struct{}{}) + require.NoError(t, err) + } + + // Measure query performance + const numQueries = 10 + var totalTime time.Duration + + for i := 0; i < numQueries; i++ { + start := time.Now() + workflows, err := engine.store.GetPendingWorkflows(ctx, engine.GetNamespace(), 50, nil) + duration := time.Since(start) + + require.NoError(t, err) + require.NotEmpty(t, workflows, "Should find pending workflows") + + totalTime += duration + } + + avgTime := totalTime / numQueries + + require.Less(t, avgTime, tc.maxQueryTime, + "Query time should scale well with workflow count") + }) + } +} + +// TestConcurrentLeaseAcquisition tests how the database handles multiple workers +// trying to acquire leases on the same workflows simultaneously +func TestConcurrentLeaseAcquisition(t *testing.T) { + testClock := clock.NewTestClock() + engine := newTestEngineWithClock(t, testClock) + ctx := context.Background() + + const numWorkflows = 100 + const numWorkers = 20 + + // Create pending workflows + loadTestWorkflow := &loadTestWorkflow{ + engine: engine, + name: "lease-test-workflow", + } + + executionIDs := make([]string, numWorkflows) + for i := 0; i < numWorkflows; i++ { + executionID, err := loadTestWorkflow.Start(ctx, struct{}{}) + require.NoError(t, err) + executionIDs[i] = executionID + } + + // Track lease acquisition results + var successfulLeases atomic.Int64 + var failedLeases atomic.Int64 + var duplicateLeases atomic.Int64 + + leaseOwnership := make(map[string]string) // workflowID -> workerID + var mu sync.Mutex + + // Start workers trying to acquire leases concurrently + var wg sync.WaitGroup + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + + workerIDStr := fmt.Sprintf("worker-%d", workerID) + + for _, executionID := range executionIDs { + // Try to acquire lease + lease := &store.Lease{ + ResourceID: executionID, + Kind: "workflow", + Namespace: engine.GetNamespace(), + WorkerID: workerIDStr, + AcquiredAt: testClock.Now().UnixMilli(), + ExpiresAt: testClock.Now().Add(30 * time.Second).UnixMilli(), + HeartbeatAt: testClock.Now().UnixMilli(), + } + err := engine.store.AcquireLease(ctx, lease) + + if err != nil { + failedLeases.Add(1) + continue + } + + // Check for duplicate lease (data race/corruption) + mu.Lock() + if existingWorker, exists := leaseOwnership[executionID]; exists { + duplicateLeases.Add(1) + t.Errorf("DUPLICATE LEASE: Workflow %s acquired by both %s and %s", + executionID, existingWorker, workerIDStr) + } else { + leaseOwnership[executionID] = workerIDStr + successfulLeases.Add(1) + } + mu.Unlock() + } + }(i) + } + + wg.Wait() + + // Analyze results + successful := successfulLeases.Load() + _ = failedLeases.Load() + duplicates := duplicateLeases.Load() + + // Assertions for data integrity + require.Equal(t, int64(0), duplicates, + "No duplicate leases should occur - indicates race condition") + + require.Equal(t, int64(numWorkflows), successful, + "All workflows should be successfully leased exactly once") + + // Most failures are expected due to workers competing for same workflows +} + +// Helper function to count pending workflows +func countPendingWorkflows(s store.Store, ctx context.Context, namespace string) (int, error) { + workflows, err := s.GetPendingWorkflows(ctx, namespace, 10000, nil) // Large limit + if err != nil { + return 0, err + } + return len(workflows), nil +} diff --git a/go/pkg/hydra/debug_test.go b/go/pkg/hydra/debug_test.go new file mode 100644 index 00000000000..5d4f73df477 --- /dev/null +++ b/go/pkg/hydra/debug_test.go @@ -0,0 +1,74 @@ +package hydra + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/unkeyed/unkey/go/pkg/clock" +) + +// TestBasicWorkflowExecution tests the most basic workflow execution +func TestBasicWorkflowExecution(t *testing.T) { + realClock := clock.New() + engine := newTestEngineWithClock(t, realClock) + + // Create a very simple workflow + simpleWorkflow := &debugWorkflow{ + engine: engine, + name: "debug-workflow", + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Start worker + worker, err := NewWorker(engine, WorkerConfig{ + WorkerID: "debug-worker", + Concurrency: 1, + PollInterval: 100 * time.Millisecond, + HeartbeatInterval: 2 * time.Second, + ClaimTimeout: 10 * time.Second, + }) + require.NoError(t, err) + + err = RegisterWorkflow(worker, simpleWorkflow) + require.NoError(t, err) + + err = worker.Start(ctx) + require.NoError(t, err) + defer worker.Shutdown(ctx) + + // Submit a single workflow + workflowID, err := simpleWorkflow.Start(ctx, "test-payload") + require.NoError(t, err) + require.NotEmpty(t, workflowID) + + // Wait for completion + finalWorkflow := waitForWorkflowCompletion(t, engine, workflowID, 8*time.Second) + require.NotNil(t, finalWorkflow, "Workflow should complete") + +} + +type debugWorkflow struct { + engine *Engine + name string +} + +func (w *debugWorkflow) Name() string { + return w.name +} + +func (w *debugWorkflow) Run(ctx WorkflowContext, req any) error { + // Very simple step + _, err := Step(ctx, "debug-step", func(context.Context) (string, error) { + time.Sleep(50 * time.Millisecond) + return "success", nil + }) + return err +} + +func (w *debugWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} diff --git a/go/pkg/hydra/doc.go b/go/pkg/hydra/doc.go new file mode 100644 index 00000000000..dc8e85c155f --- /dev/null +++ b/go/pkg/hydra/doc.go @@ -0,0 +1,245 @@ +// Package hydra provides a distributed workflow orchestration engine designed +// for reliable execution of multi-step business processes at scale. +// +// Hydra implements the Temporal-style workflow pattern with durable execution, +// automatic retries, checkpointing, and distributed coordination. It supports +// both simple sequential workflows and complex long-running processes with +// sleep states, cron scheduling, and step-level fault tolerance. +// +// # Core Concepts +// +// Engine: The central orchestration component that manages workflow lifecycle, +// worker coordination, and persistence. Each engine instance operates within +// a specific namespace for tenant isolation. +// +// Workers: Distributed processing units that poll for pending workflows, +// acquire leases for exclusive execution, and run workflow logic. Workers +// support concurrent execution with configurable limits and automatic +// heartbeat management. +// +// Workflows: Business logic containers that define a series of steps to be +// executed. Workflows are stateless functions that can be suspended, resumed, +// and retried while maintaining exactly-once execution guarantees. +// +// Steps: Individual units of work within a workflow. Steps support automatic +// checkpointing, retry logic, and result caching to ensure idempotent execution +// even across worker failures or restarts. +// +// # Key Features +// +// Exactly-Once Execution: Workflows and steps execute exactly once, even in +// the presence of worker failures, network partitions, or duplicate deliveries. +// +// Durable State: All workflow state is persisted to a database, allowing +// workflows to survive process restarts and infrastructure failures. +// +// Distributed Coordination: Multiple workers can safely operate on the same +// workflow queue using lease-based coordination and circuit breaker protection. +// +// Comprehensive Observability: Built-in Prometheus metrics track workflow +// throughput, latency, error rates, and system health across all components. +// +// Flexible Scheduling: Support for immediate execution, cron-based scheduling, +// and workflow sleep states for time-based coordination. +// +// # Basic Usage +// +// Creating an engine and worker: +// +// // Set up the storage layer +// db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{}) +// if err != nil { +// return err +// } +// store := gorm.NewGORMStore(db, clock.New()) +// +// // Create the engine +// engine := hydra.New(hydra.Config{ +// Store: store, +// Namespace: "production", +// Logger: logger, +// }) +// +// // Create and configure a worker +// worker, err := hydra.NewWorker(engine, hydra.WorkerConfig{ +// WorkerID: "worker-1", +// Concurrency: 10, +// PollInterval: 100 * time.Millisecond, +// HeartbeatInterval: 30 * time.Second, +// ClaimTimeout: 5 * time.Minute, +// }) +// +// Defining a workflow: +// +// type OrderWorkflow struct { +// engine *hydra.Engine +// } +// +// func (w *OrderWorkflow) Name() string { +// return "order-processing" +// } +// +// func (w *OrderWorkflow) Run(ctx hydra.WorkflowContext, req *OrderRequest) error { +// // Step 1: Validate payment +// payment, err := hydra.Step(ctx, "validate-payment", func(stepCtx context.Context) (*Payment, error) { +// return validatePayment(stepCtx, req.PaymentID) +// }) +// if err != nil { +// return err +// } +// +// // Step 2: Reserve inventory +// reservation, err := hydra.Step(ctx, "reserve-inventory", func(stepCtx context.Context) (*Reservation, error) { +// return reserveInventory(stepCtx, req.Items) +// }) +// if err != nil { +// return err +// } +// +// // Step 3: Process order +// _, err = hydra.Step(ctx, "process-order", func(stepCtx context.Context) (*Order, error) { +// return processOrder(stepCtx, payment, reservation) +// }) +// +// return err +// } +// +// Starting workflows: +// +// // Register the workflow with the worker +// orderWorkflow := &OrderWorkflow{engine: engine} +// err = hydra.RegisterWorkflow(worker, orderWorkflow) +// if err != nil { +// return err +// } +// +// // Start the worker +// ctx := context.Background() +// err = worker.Start(ctx) +// if err != nil { +// return err +// } +// defer worker.Shutdown(ctx) +// +// // Submit a workflow for execution +// request := &OrderRequest{ +// CustomerID: "cust_123", +// Items: []Item{{SKU: "item_456", Quantity: 2}}, +// PaymentID: "pay_789", +// } +// +// executionID, err := engine.StartWorkflow(ctx, "order-processing", request) +// if err != nil { +// return err +// } +// +// fmt.Printf("Started workflow execution: %s\n", executionID) +// +// # Advanced Features +// +// Sleep States: Workflows can suspend execution and resume after a specified +// duration, allowing for time-based coordination and human approval processes: +// +// // Sleep for 24 hours for manual approval +// return hydra.Sleep(ctx, 24*time.Hour) +// +// Cron Scheduling: Register workflows to run on a schedule: +// +// err = engine.RegisterCron("0 0 * * *", "daily-report", func(ctx context.Context) error { +// // Generate daily report +// return generateDailyReport(ctx) +// }) +// +// Error Handling and Retries: Configure retry behavior at the workflow level: +// +// executionID, err := engine.StartWorkflow(ctx, "order-processing", request, +// hydra.WithMaxAttempts(5), +// hydra.WithRetryBackoff(2*time.Second), +// hydra.WithTimeout(10*time.Minute), +// ) +// +// # Observability +// +// Hydra provides comprehensive Prometheus metrics out of the box: +// +// Workflow Metrics: +// - hydra_workflows_started_total: Total workflows started +// - hydra_workflows_completed_total: Total workflows completed/failed +// - hydra_workflow_duration_seconds: Workflow execution time +// - hydra_workflow_queue_time_seconds: Time spent waiting for execution +// - hydra_workflows_active: Currently running workflows per worker +// +// Step Metrics: +// - hydra_steps_executed_total: Total steps executed with status +// - hydra_step_duration_seconds: Individual step execution time +// - hydra_steps_cached_total: Steps served from checkpoint cache +// - hydra_steps_retried_total: Step retry attempts +// +// Worker Metrics: +// - hydra_worker_polls_total: Worker polling operations +// - hydra_worker_heartbeats_total: Worker heartbeat operations +// - hydra_lease_acquisitions_total: Workflow lease acquisitions +// - hydra_worker_concurrency_current: Current workflow concurrency per worker +// +// Error and Performance Metrics: +// - hydra_errors_total: Categorized error counts +// - hydra_payload_size_bytes: Workflow and step payload sizes +// - hydra_db_operations_total: Database operation counts and latency +// +// All metrics include rich labels for namespace, workflow names, worker IDs, +// and status information, enabling detailed monitoring and alerting. +// +// # Architecture +// +// Hydra uses a lease-based coordination model to ensure exactly-once execution: +// +// 1. Workers poll the database for pending workflows in their namespace +// 2. Workers attempt to acquire exclusive leases on available workflows +// 3. Successful lease holders execute the workflow logic +// 4. Workers send periodic heartbeats to maintain lease ownership +// 5. Completed workflows update their status and release the lease +// 6. Failed workers automatically lose their leases after timeout +// +// This design provides fault tolerance without requiring complex consensus +// protocols or external coordination services. +// +// # Database Schema +// +// Hydra requires the following database tables: +// +// - workflow_executions: Stores workflow state, status, and metadata +// - workflow_steps: Tracks individual step execution and results +// - leases: Manages worker coordination and exclusive access +// - cron_jobs: Stores scheduled workflow definitions +// +// The schema automatically migrates when using the GORM store implementation. +// +// # Error Handling +// +// Hydra distinguishes between different types of errors: +// +// Transient Errors: Network timeouts, temporary database failures, etc. +// These trigger automatic retries based on the configured retry policy. +// +// Permanent Errors: Validation failures, business logic errors, etc. +// These immediately fail the workflow without retries. +// +// Workflow Suspension: Controlled suspension using Sleep() for time-based +// coordination or external event waiting. +// +// # Performance Considerations +// +// - Workers use circuit breakers to prevent cascading failures +// - Database queries are optimized with appropriate indexes +// - Lease timeouts prevent stuck workflows from blocking execution +// - Configurable concurrency limits prevent resource exhaustion +// - Built-in connection pooling and retry logic for database operations +// +// # Thread Safety +// +// All Hydra components are thread-safe and designed for concurrent access: +// - Multiple workers can safely operate on the same workflow queue +// - Step execution is atomic and isolated using database transactions +// - Workflow state updates use optimistic locking to prevent race conditions +// - Metrics collection is thread-safe and non-blocking +package hydra diff --git a/go/pkg/hydra/engine.go b/go/pkg/hydra/engine.go new file mode 100644 index 00000000000..0f759b9feda --- /dev/null +++ b/go/pkg/hydra/engine.go @@ -0,0 +1,279 @@ +package hydra + +import ( + "context" + "fmt" + "time" + + "github.com/unkeyed/unkey/go/pkg/clock" + "github.com/unkeyed/unkey/go/pkg/hydra/metrics" + "github.com/unkeyed/unkey/go/pkg/hydra/store" + "github.com/unkeyed/unkey/go/pkg/otel/logging" + "github.com/unkeyed/unkey/go/pkg/uid" +) + +// Config holds the configuration for creating a new Engine instance. +// +// All fields except Store are optional and will use sensible defaults +// if not provided. +type Config struct { + // Store is the persistence layer for workflow state and metadata. + // This field is required and cannot be nil. + Store store.Store + + // Namespace provides tenant isolation for workflows. All workflows + // created by this engine will be scoped to this namespace. + // Defaults to "default" if not specified. + Namespace string + + // Clock provides time-related operations for testing and scheduling. + // Defaults to a real clock implementation if not specified. + Clock clock.Clock + + // Logger handles structured logging for the engine operations. + // Defaults to a no-op logger if not specified. + Logger logging.Logger + + // Marshaller handles serialization of workflow payloads and step results. + // Defaults to JSON marshalling if not specified. + Marshaller Marshaller +} + +// NewConfig creates a default config with sensible defaults. +// +// The returned config uses: +// - "default" namespace +// - Real clock implementation +// - All other fields will be set to their defaults when passed to New() +func NewConfig() Config { + return Config{ + Store: nil, + Namespace: "default", + Clock: clock.New(), + Logger: nil, + Marshaller: nil, + } +} + +// Engine is the core workflow orchestration engine that manages workflow +// lifecycle, coordination, and execution. +// +// The engine is responsible for: +// - Starting new workflows and managing their state +// - Coordinating workflow execution across multiple workers +// - Handling cron-based scheduled workflows +// - Providing namespace isolation for multi-tenant deployments +// - Recording metrics and managing observability +// +// Engine instances are thread-safe and can be shared across multiple +// workers and goroutines. +type Engine struct { + store store.Store + namespace string + cronHandlers map[string]CronHandler + clock clock.Clock + logger logging.Logger + marshaller Marshaller +} + +// New creates a new Engine instance with the provided configuration. +// +// The engine will validate the configuration and apply defaults for +// any missing optional fields. The Store field is required and the +// function will panic if it is nil. +// +// Example: +// +// engine := hydra.New(hydra.Config{ +// Store: gormStore, +// Namespace: "production", +// Logger: logger, +// }) +func New(config Config) *Engine { + if config.Store == nil { + panic("hydra: config.Store cannot be nil") + } + + namespace := config.Namespace + if namespace == "" { + namespace = "default" + } + + clk := config.Clock + if clk == nil { + clk = clock.New() // Default to real clock + } + + logger := config.Logger + if logger == nil { + logger = logging.NewNoop() // Default logger + } + + marshaller := config.Marshaller + if marshaller == nil { + marshaller = NewJSONMarshaller() // Default to JSON marshaller + } + + return &Engine{ + store: config.Store, + namespace: namespace, + cronHandlers: make(map[string]CronHandler), + clock: clk, + logger: logger, + marshaller: marshaller, + } +} + +// NewWithStore creates a new Engine with the provided store and default config. +// +// This is a convenience function for creating an engine with minimal configuration. +// Other configuration options will use their default values. +// +// Deprecated: Use New(Config{...}) for more explicit configuration. +func NewWithStore(st store.Store, namespace string, clk clock.Clock) *Engine { + return New(Config{ + Store: st, + Namespace: namespace, + Clock: clk, + Logger: nil, + Marshaller: nil, + }) +} + +// GetNamespace returns the namespace for this engine instance. +// +// This method is primarily used by workers and internal components +// to scope database operations to the correct tenant namespace. +func (e *Engine) GetNamespace() string { + return e.namespace +} + +// RegisterCron registers a cron job with the given schedule and handler. +// +// The cronSpec follows standard cron syntax (e.g., "0 0 * * *" for daily at midnight). +// The name must be unique within this engine's namespace. The handler will be +// called according to the schedule. +// +// Example: +// +// err := engine.RegisterCron("0 */6 * * *", "cleanup-task", func(ctx context.Context) error { +// return performCleanup(ctx) +// }) +// +// Returns an error if a cron job with the same name is already registered. +func (e *Engine) RegisterCron(cronSpec, name string, handler CronHandler) error { + if _, exists := e.cronHandlers[name]; exists { + return fmt.Errorf("cron %q is already registered", name) + } + + e.cronHandlers[name] = handler + + cronJob := &store.CronJob{ + ID: uid.New(uid.CronJobPrefix), + Name: name, + CronSpec: cronSpec, + Namespace: e.namespace, + WorkflowName: "", // Empty since this uses a handler, not a workflow + Enabled: true, + CreatedAt: e.clock.Now().UnixMilli(), + UpdatedAt: e.clock.Now().UnixMilli(), + LastRunAt: nil, + NextRunAt: calculateNextRun(cronSpec, e.clock.Now()), + } + + return e.store.UpsertCronJob(context.Background(), cronJob) +} + +// StartWorkflow starts a new workflow execution with the given name and payload. +// +// This method creates a new workflow execution record in the database and makes +// it available for workers to pick up and execute. The workflow will be queued +// in a pending state until a worker acquires a lease and begins execution. +// +// Parameters: +// - ctx: Context for the operation, which may include cancellation and timeouts +// - workflowName: Must match the Name() method of a registered workflow type +// - payload: The input data for the workflow, which will be serialized and stored +// - opts: Optional configuration for retry behavior, timeouts, and trigger metadata +// +// Returns: +// - executionID: A unique identifier for this workflow execution +// - error: Any error that occurred during workflow creation +// +// The payload will be marshalled using the engine's configured marshaller (JSON by default) +// and must be serializable. The workflow will be executed with the configured retry +// policy and timeout settings. +// +// Example: +// +// executionID, err := engine.StartWorkflow(ctx, "order-processing", &OrderRequest{ +// CustomerID: "cust_123", +// Items: []Item{{SKU: "item_456", Quantity: 2}}, +// }, hydra.WithMaxAttempts(5), hydra.WithTimeout(30*time.Minute)) +// +// Metrics recorded: +// - hydra_workflows_started_total (counter) +// - hydra_workflows_queued (gauge) +// - hydra_payload_size_bytes (histogram) +func (e *Engine) StartWorkflow(ctx context.Context, workflowName string, payload any, opts ...WorkflowOption) (string, error) { + + executionID := uid.New("wf") + + config := &WorkflowConfig{ + MaxAttempts: 3, // Default to 3 attempts total (1 initial + 2 retries) + TimeoutDuration: 1 * time.Hour, + RetryBackoff: 1 * time.Second, + TriggerType: TriggerTypeAPI, // Default trigger type + TriggerSource: nil, + } + for _, opt := range opts { + opt(config) + } + + data, err := e.marshaller.Marshal(payload) + if err != nil { + metrics.SerializationErrorsTotal.WithLabelValues(e.namespace, workflowName, "input").Inc() + return "", fmt.Errorf("failed to marshal payload: %w", err) + } + + // Record payload size + metrics.RecordPayloadSize(e.namespace, workflowName, "input", len(data)) + + workflow := &store.WorkflowExecution{ + ID: executionID, + WorkflowName: workflowName, + Status: store.WorkflowStatusPending, + InputData: data, + OutputData: nil, + ErrorMessage: "", + Namespace: e.namespace, + MaxAttempts: config.MaxAttempts, + RemainingAttempts: config.MaxAttempts, // Start with full attempts available + CreatedAt: e.clock.Now().UnixMilli(), + StartedAt: nil, + CompletedAt: nil, + NextRetryAt: nil, + SleepUntil: nil, + TriggerType: config.TriggerType, + TriggerSource: config.TriggerSource, + TraceID: "", + } + + err = e.store.CreateWorkflow(ctx, workflow) + if err != nil { + metrics.RecordError(e.namespace, "engine", "workflow_creation_failed") + return "", fmt.Errorf("failed to create workflow: %w", err) + } + + // Record workflow started + triggerTypeStr := string(config.TriggerType) + metrics.WorkflowsStartedTotal.WithLabelValues(e.namespace, workflowName, triggerTypeStr).Inc() + metrics.WorkflowsQueued.WithLabelValues(e.namespace, "pending").Inc() + + return workflow.ID, nil +} + +// GetStore returns the underlying store (for testing purposes) +func (e *Engine) GetStore() store.Store { + return e.store +} diff --git a/go/pkg/hydra/engine_test.go b/go/pkg/hydra/engine_test.go new file mode 100644 index 00000000000..28b4bd01395 --- /dev/null +++ b/go/pkg/hydra/engine_test.go @@ -0,0 +1,160 @@ +package hydra + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/unkeyed/unkey/go/pkg/clock" + "github.com/unkeyed/unkey/go/pkg/hydra/testharness" +) + +// Test workflow that counts executions and emits events +type CountingWorkflow struct { + counter *int64 + events *testharness.EventCollector +} + +func (w *CountingWorkflow) Name() string { + return "counting-workflow" +} + +func (w *CountingWorkflow) Run(ctx WorkflowContext, req struct{}) error { + w.events.Emit(ctx, testharness.WorkflowStarted, "Starting counting workflow") + + _, err := Step(ctx, "increment", func(stepCtx context.Context) (string, error) { + w.events.Emit(ctx, testharness.StepExecuting, "Executing increment step", "step_name", "increment") + + // This should only execute exactly once + atomic.AddInt64(w.counter, 1) + + w.events.Emit(ctx, testharness.StepExecuted, "Completed increment step", "step_name", "increment", "result", "incremented") + + return "incremented", nil + }) + + if err != nil { + w.events.Emit(ctx, testharness.WorkflowFailed, "Workflow failed", "error", err.Error()) + } else { + w.events.Emit(ctx, testharness.WorkflowCompleted, "Workflow completed successfully") + } + + return err +} + +// CRITICAL CORRECTNESS TESTS + +func TestBasicWorkflowRegistration(t *testing.T) { + // Given: An engine instance and workflow + e := newTestEngine(t) + events := testharness.NewEventCollector() + workflow := &CountingWorkflow{ + counter: new(int64), + events: events, + } + + // When: Creating worker and registering workflow + worker, err := NewWorker(e, WorkerConfig{ + Concurrency: 1, + }) + require.NoError(t, err) + + err = RegisterWorkflow(worker, workflow) + require.NoError(t, err) + + err = worker.Start(context.Background()) + + // Then: Worker should start without error + require.NoError(t, err) + require.NotNil(t, worker) + defer worker.Shutdown(context.Background()) + + // And: We should be able to start a workflow + executionID, err := e.StartWorkflow(context.Background(), workflow.Name(), struct{}{}) + require.NoError(t, err) + require.NotEmpty(t, executionID) +} + +func TestStepExecutesExactlyOnce(t *testing.T) { + // Given: A workflow with a step that increments a counter and emits events + testClock := clock.NewTestClock() + e := newTestEngineWithClock(t, testClock) + events := testharness.NewEventCollector() + counter := int64(0) + workflow := &CountingWorkflow{ + counter: &counter, + events: events, + } + + // When: Creating worker, registering workflow, and starting + worker, err := NewWorker(e, WorkerConfig{ + Concurrency: 1, + PollInterval: 100 * time.Millisecond, // Fast polling for test + }) + require.NoError(t, err) + defer worker.Shutdown(context.Background()) + + err = RegisterWorkflow(worker, workflow) + require.NoError(t, err) + + err = worker.Start(context.Background()) + require.NoError(t, err) + + // Give worker time to start polling + time.Sleep(50 * time.Millisecond) + + // Start workflow execution + executionID, err := e.StartWorkflow(context.Background(), workflow.Name(), struct{}{}) + require.NoError(t, err) + require.NotEmpty(t, executionID) + + // Trigger worker polling with test clock + for i := 0; i < 10; i++ { + testClock.Tick(200 * time.Millisecond) + time.Sleep(10 * time.Millisecond) + + // Check if workflow has been picked up + currentStatus, err := e.store.GetWorkflow(context.Background(), e.GetNamespace(), executionID) + require.NoError(t, err) + if currentStatus.Status != WorkflowStatusPending { + break + } + } + + // Wait for completion + completedWorkflow := waitForWorkflowCompletion(t, e, executionID, 3*time.Second) + require.NotNil(t, completedWorkflow) + + // Then: Assert using both counter and events + finalCount := atomic.LoadInt64(&counter) + + // Check events for detailed analysis + stepExecutions := events.FilterWithData(testharness.StepExecuting, "step_name", "increment") + stepCompletions := events.FilterWithData(testharness.StepExecuted, "step_name", "increment") + workflowCompletions := events.Filter(testharness.WorkflowCompleted) + + // The critical assertion: step should execute exactly once + assert.Equal(t, int64(1), finalCount, "Counter should be incremented exactly once") + assert.Equal(t, 1, len(stepExecutions), "Step should be executed exactly once") + assert.Equal(t, 1, len(stepCompletions), "Step should complete exactly once") + assert.Equal(t, 1, len(workflowCompletions), "Workflow should complete exactly once") +} + +func TestStepCheckpointingPreventsReExecution(t *testing.T) { + t.Skip("TODO: Implement checkpointing test") +} + +func TestWorkflowTerminatesEventually(t *testing.T) { + t.Skip("TODO: Implement retry limit testing") +} + +func TestWorkerCrashRecovery(t *testing.T) { + t.Skip("TODO: Implement worker crash recovery testing") +} + +func TestNoDuplicateStepExecution(t *testing.T) { + t.Skip("TODO: Implement concurrency safety testing") +} diff --git a/go/pkg/hydra/event_driven_consistency_test.go b/go/pkg/hydra/event_driven_consistency_test.go new file mode 100644 index 00000000000..0bc323066fc --- /dev/null +++ b/go/pkg/hydra/event_driven_consistency_test.go @@ -0,0 +1,316 @@ +package hydra + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/unkeyed/unkey/go/pkg/clock" + "github.com/unkeyed/unkey/go/pkg/hydra/store" + "github.com/unkeyed/unkey/go/pkg/hydra/testharness" +) + +// TestEventDrivenConsistency uses the event collection system to deterministically +// verify exactly-once execution guarantees +func TestEventDrivenConsistency(t *testing.T) { + realClock := clock.New() + engine := newTestEngineWithClock(t, realClock) + + // Create event collector + eventCollector := testharness.NewEventCollector() + + // Create event-aware workflow + workflow := &eventAwareWorkflow{ + engine: engine, + name: "event-driven-test-workflow", + collector: eventCollector, + } + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + // Start worker + worker, err := NewWorker(engine, WorkerConfig{ + WorkerID: "event-test-worker", + Concurrency: 2, + PollInterval: 100 * time.Millisecond, + HeartbeatInterval: 2 * time.Second, + ClaimTimeout: 10 * time.Second, + }) + require.NoError(t, err) + + err = RegisterWorkflow(worker, workflow) + require.NoError(t, err) + + err = worker.Start(ctx) + require.NoError(t, err) + defer worker.Shutdown(ctx) + + // Submit workflows + const numWorkflows = 10 + workflowIDs := make([]string, numWorkflows) + + for i := 0; i < numWorkflows; i++ { + workflowID, startErr := workflow.Start(ctx, fmt.Sprintf("payload-%d", i)) + require.NoError(t, startErr) + workflowIDs[i] = workflowID + } + + // Wait for all workflows to be processed by checking events + // This is deterministic - we wait until we see completion events for all workflows + require.Eventually(t, func() bool { + completedEvents := eventCollector.Count(testharness.WorkflowCompleted) + failedEvents := eventCollector.Count(testharness.WorkflowFailed) + totalFinished := completedEvents + failedEvents + + return totalFinished == numWorkflows + }, 12*time.Second, 200*time.Millisecond, "All workflows should finish (complete or fail)") + + // Analyze events for consistency violations + + // Verify exactly-once execution per workflow + for _, workflowID := range workflowIDs { + // Each workflow should have exactly one started event + startedEvents := eventCollector.FilterWithData(testharness.WorkflowStarted, "execution_id", workflowID) + require.Len(t, startedEvents, 1, "Workflow %s should start exactly once", workflowID) + + // Each workflow should have exactly one completion event (completed OR failed) + completedEvents := eventCollector.FilterWithData(testharness.WorkflowCompleted, "execution_id", workflowID) + failedEvents := eventCollector.FilterWithData(testharness.WorkflowFailed, "execution_id", workflowID) + + totalCompletionEvents := len(completedEvents) + len(failedEvents) + require.Equal(t, 1, totalCompletionEvents, + "Workflow %s should have exactly one completion event, got %d completed + %d failed", + workflowID, len(completedEvents), len(failedEvents)) + + // Each step should execute exactly once + stepExecutingEvents := eventCollector.FilterWithData(testharness.StepExecuting, "execution_id", workflowID) + stepExecutedEvents := eventCollector.FilterWithData(testharness.StepExecuted, "execution_id", workflowID) + stepFailedEvents := eventCollector.FilterWithData(testharness.StepFailed, "execution_id", workflowID) + + // Should have exactly one step executing event + require.Len(t, stepExecutingEvents, 1, "Workflow %s should have exactly one step executing event", workflowID) + + // Should have exactly one step completion event (executed OR failed) + totalStepCompletions := len(stepExecutedEvents) + len(stepFailedEvents) + require.Equal(t, 1, totalStepCompletions, + "Workflow %s should have exactly one step completion event, got %d executed + %d failed", + workflowID, len(stepExecutedEvents), len(stepFailedEvents)) + } + + // Verify database consistency matches events + allWorkflows, err := engine.store.GetAllWorkflows(ctx, engine.GetNamespace()) + require.NoError(t, err) + + workflowStatusCounts := make(map[store.WorkflowStatus]int) + for _, wf := range allWorkflows { + workflowStatusCounts[wf.Status]++ + } + + // Database should match event counts + completedInDB := workflowStatusCounts[store.WorkflowStatusCompleted] + failedInDB := workflowStatusCounts[store.WorkflowStatusFailed] + + eventCompletedCount := eventCollector.Count(testharness.WorkflowCompleted) + eventFailedCount := eventCollector.Count(testharness.WorkflowFailed) + + require.Equal(t, eventCompletedCount, completedInDB, + "Completed workflows in DB should match completed events") + require.Equal(t, eventFailedCount, failedInDB, + "Failed workflows in DB should match failed events") + + // Verify step consistency + allSteps, err := engine.store.GetAllSteps(ctx, engine.GetNamespace()) + require.NoError(t, err) + + stepStatusCounts := make(map[store.StepStatus]int) + for _, step := range allSteps { + stepStatusCounts[step.Status]++ + } + + // Steps in database should match step events + completedStepsInDB := stepStatusCounts[store.StepStatusCompleted] + failedStepsInDB := stepStatusCounts[store.StepStatusFailed] + + eventStepExecutedCount := eventCollector.Count(testharness.StepExecuted) + eventStepFailedCount := eventCollector.Count(testharness.StepFailed) + + require.Equal(t, eventStepExecutedCount, completedStepsInDB, + "Completed steps in DB should match step executed events") + require.Equal(t, eventStepFailedCount, failedStepsInDB, + "Failed steps in DB should match step failed events") + +} + +// TestEventDrivenConcurrentConsistency tests consistency with multiple workers using events +func TestEventDrivenConcurrentConsistency(t *testing.T) { + realClock := clock.New() + engine := newTestEngineWithClock(t, realClock) + + // Create event collector + eventCollector := testharness.NewEventCollector() + + // Create event-aware workflow + workflow := &eventAwareWorkflow{ + engine: engine, + name: "concurrent-event-test-workflow", + collector: eventCollector, + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + // Start multiple workers + const numWorkers = 3 + workers := make([]Worker, numWorkers) + + for i := 0; i < numWorkers; i++ { + worker, err := NewWorker(engine, WorkerConfig{ + WorkerID: fmt.Sprintf("concurrent-event-worker-%d", i), + Concurrency: 2, + PollInterval: 50 * time.Millisecond, + HeartbeatInterval: 2 * time.Second, + ClaimTimeout: 10 * time.Second, + }) + require.NoError(t, err) + + err = RegisterWorkflow(worker, workflow) + require.NoError(t, err) + + err = worker.Start(ctx) + require.NoError(t, err) + defer worker.Shutdown(ctx) + + workers[i] = worker + } + + // Submit workflows + const numWorkflows = 15 + workflowIDs := make([]string, numWorkflows) + + for i := 0; i < numWorkflows; i++ { + workflowID, err := workflow.Start(ctx, fmt.Sprintf("concurrent-payload-%d", i)) + require.NoError(t, err) + workflowIDs[i] = workflowID + } + + // Wait for reasonable workflow processing (focus on consistency, not completion rate) + // Give workers time to process what they can + time.Sleep(8 * time.Second) + + // Check what actually happened + completedEvents := eventCollector.Count(testharness.WorkflowCompleted) + failedEvents := eventCollector.Count(testharness.WorkflowFailed) + totalFinished := completedEvents + failedEvents + + // Ensure we processed at least some workflows (not zero) + require.Greater(t, totalFinished, 0, "Should process at least some workflows") + + // Verify exactly-once execution for workflows that were processed + duplicateExecutions := 0 + duplicateCompletions := 0 + processedWorkflows := 0 + + for _, workflowID := range workflowIDs { + // Check if this workflow was processed at all + startedEvents := eventCollector.FilterWithData(testharness.WorkflowStarted, "execution_id", workflowID) + completedEvents := eventCollector.FilterWithData(testharness.WorkflowCompleted, "execution_id", workflowID) + failedEvents := eventCollector.FilterWithData(testharness.WorkflowFailed, "execution_id", workflowID) + + // Skip workflows that weren't processed + if len(startedEvents) == 0 { + continue + } + + processedWorkflows++ + + // Check for duplicate workflow executions + if len(startedEvents) > 1 { + duplicateExecutions++ + t.Errorf("DUPLICATE EXECUTION: Workflow %s started %d times", workflowID, len(startedEvents)) + } + + // Check for duplicate completions + totalCompletions := len(completedEvents) + len(failedEvents) + if totalCompletions > 1 { + duplicateCompletions++ + t.Errorf("DUPLICATE COMPLETION: Workflow %s completed %d times (%d completed + %d failed)", + workflowID, totalCompletions, len(completedEvents), len(failedEvents)) + } + } + + // Assert no duplicates + require.Equal(t, 0, duplicateExecutions, "Should have zero duplicate workflow executions") + require.Equal(t, 0, duplicateCompletions, "Should have zero duplicate workflow completions") + + // Verify step-level consistency for processed workflows + duplicateStepExecutions := 0 + for _, workflowID := range workflowIDs { + startedEvents := eventCollector.FilterWithData(testharness.WorkflowStarted, "execution_id", workflowID) + if len(startedEvents) == 0 { + continue // Skip unprocessed workflows + } + + stepExecutingEvents := eventCollector.FilterWithData(testharness.StepExecuting, "execution_id", workflowID) + if len(stepExecutingEvents) > 1 { + duplicateStepExecutions++ + t.Errorf("DUPLICATE STEP EXECUTION: Workflow %s had %d step executing events", + workflowID, len(stepExecutingEvents)) + } + } + + require.Equal(t, 0, duplicateStepExecutions, "Should have zero duplicate step executions") + +} + +// eventAwareWorkflow emits events during execution for testing +type eventAwareWorkflow struct { + engine *Engine + name string + collector *testharness.EventCollector +} + +func (w *eventAwareWorkflow) Name() string { + return w.name +} + +func (w *eventAwareWorkflow) Run(ctx WorkflowContext, req any) error { + // Emit workflow started event + w.collector.Emit(ctx, testharness.WorkflowStarted, "Workflow execution started") + + // Emit step executing event + w.collector.Emit(ctx, testharness.StepExecuting, "Step execution started", "step_name", "test-step") + + // Execute the step + result, err := Step(ctx, "test-step", func(stepCtx context.Context) (string, error) { + // Simulate some work + time.Sleep(20 * time.Millisecond) + return "step-completed", nil + }) + + if err != nil { + // Emit step failed event + w.collector.Emit(ctx, testharness.StepFailed, "Step execution failed", + "step_name", "test-step", "error", err.Error()) + + // Emit workflow failed event + w.collector.Emit(ctx, testharness.WorkflowFailed, "Workflow execution failed", "error", err.Error()) + + return err + } + + // Emit step executed event + w.collector.Emit(ctx, testharness.StepExecuted, "Step execution completed", + "step_name", "test-step", "result", result) + + // Emit workflow completed event + w.collector.Emit(ctx, testharness.WorkflowCompleted, "Workflow execution completed") + + return nil +} + +func (w *eventAwareWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} diff --git a/go/pkg/hydra/hydra.test b/go/pkg/hydra/hydra.test new file mode 100755 index 00000000000..2194af73f91 Binary files /dev/null and b/go/pkg/hydra/hydra.test differ diff --git a/go/pkg/hydra/marshaller.go b/go/pkg/hydra/marshaller.go new file mode 100644 index 00000000000..39f9f63d3f2 --- /dev/null +++ b/go/pkg/hydra/marshaller.go @@ -0,0 +1,56 @@ +package hydra + +import "encoding/json" + +// Marshaller defines the interface for serializing workflow payloads and step results. +// +// The marshaller is responsible for converting Go values to and from byte arrays +// for storage in the database. Custom marshallers can be implemented to support +// different serialization formats like Protocol Buffers, MessagePack, or custom +// binary formats. +// +// Implementations must ensure that: +// - Marshal and Unmarshal are inverse operations +// - The same input always produces the same output (deterministic) +// - All workflow payload types are supported +// - Error handling is consistent and informative +type Marshaller interface { + // Marshal converts a Go value to bytes for storage. + // The value may be any type used in workflow payloads or step results. + Marshal(v any) ([]byte, error) + + // Unmarshal converts stored bytes back to a Go value. + // The target value should be a pointer to the desired type. + Unmarshal(data []byte, v any) error +} + +// JSONMarshaller implements Marshaller using standard Go JSON encoding. +// +// This is the default marshaller used by Hydra engines. It provides +// good compatibility with most Go types and is human-readable for +// debugging purposes. +// +// Limitations: +// - Cannot handle circular references +// - Maps with non-string keys are not supported +// - Precision may be lost for large integers +// - Custom types need JSON tags for proper serialization +type JSONMarshaller struct{} + +// NewJSONMarshaller creates a new JSON-based marshaller. +// +// This is the default marshaller used when no custom marshaller +// is provided to the engine configuration. +func NewJSONMarshaller() Marshaller { + return &JSONMarshaller{} +} + +// Marshal implements Marshaller.Marshal using encoding/json. +func (j *JSONMarshaller) Marshal(v any) ([]byte, error) { + return json.Marshal(v) +} + +// Unmarshal implements Marshaller.Unmarshal using encoding/json. +func (j *JSONMarshaller) Unmarshal(data []byte, v any) error { + return json.Unmarshal(data, v) +} diff --git a/go/pkg/hydra/metrics/example_usage.go b/go/pkg/hydra/metrics/example_usage.go new file mode 100644 index 00000000000..775b7b65940 --- /dev/null +++ b/go/pkg/hydra/metrics/example_usage.go @@ -0,0 +1,89 @@ +package metrics + +import ( + "time" +) + +const exampleNamespace = "production" + +func ExampleWorkflowMetrics() { + namespace := exampleNamespace + workflowName := "user-onboarding" + + WorkflowsStartedTotal.WithLabelValues(namespace, workflowName, "manual").Inc() + + WorkflowsQueued.WithLabelValues(namespace, "pending").Set(42) + WorkflowsActive.WithLabelValues(namespace, "worker-1").Set(5) + + start := time.Now() + ObserveWorkflowDuration(namespace, workflowName, "completed", start) + WorkflowsCompletedTotal.WithLabelValues(namespace, workflowName, "completed").Inc() +} + +func ExampleStepMetrics() { + namespace := exampleNamespace + workflowName := "order-processing" + stepName := "charge-payment" + + start := time.Now() + ObserveStepDuration(namespace, workflowName, stepName, "completed", start) + StepsExecutedTotal.WithLabelValues(namespace, workflowName, stepName, "completed").Inc() + + StepsCachedTotal.WithLabelValues(namespace, workflowName, stepName).Inc() +} + +func ExampleDatabaseMetrics() { + start := time.Now() + ObserveDbOperation("select", "workflow_executions", "success", start) + + DbConnectionsActive.WithLabelValues("worker-1").Set(15) +} + +func ExampleSleepMetrics() { + namespace := exampleNamespace + workflowName := "user-onboarding" + + SleepsStartedTotal.WithLabelValues(namespace, workflowName).Inc() + SleepsResumedTotal.WithLabelValues(namespace, workflowName).Inc() + + actualSleepDuration := 25 * time.Minute // actual time slept + SleepDurationSeconds.WithLabelValues(namespace, workflowName).Observe(actualSleepDuration.Seconds()) + + CronTriggersTotal.WithLabelValues(namespace, "daily-report", "success").Inc() +} + +func ExampleErrorMetrics() { + namespace := exampleNamespace + + RecordError(namespace, "step", "timeout") + RecordError(namespace, "client", "serialization") + RecordError(namespace, "store", "connection") + + PanicsTotal.WithLabelValues("worker-1", "step_execution").Inc() + + TimeoutsTotal.WithLabelValues(namespace, "workflow_execution").Inc() +} + +func ExamplePayloadMetrics() { + namespace := exampleNamespace + workflowName := "image-processing" + + inputSize := 1024 * 50 // 50KB input + outputSize := 1024 * 5 // 5KB output + + RecordPayloadSize(namespace, workflowName, "input", inputSize) + RecordPayloadSize(namespace, workflowName, "output", outputSize) + + SerializationErrorsTotal.WithLabelValues(namespace, workflowName, "input").Inc() +} + +func ExampleWorkerMetrics() { + workerID := "worker-1" + namespace := exampleNamespace + + WorkerHeartbeatsTotal.WithLabelValues(workerID, namespace, "success").Inc() + WorkerPollsTotal.WithLabelValues(workerID, namespace, "found_work").Inc() + LeaseAcquisitionsTotal.WithLabelValues(workerID, "workflow", "success").Inc() + + WorkerConcurrencyCurrent.WithLabelValues(workerID, namespace).Set(8) +} diff --git a/go/pkg/hydra/metrics/metrics.go b/go/pkg/hydra/metrics/metrics.go new file mode 100644 index 00000000000..f90aabd6d01 --- /dev/null +++ b/go/pkg/hydra/metrics/metrics.go @@ -0,0 +1,351 @@ +package metrics + +import ( + "os" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/unkeyed/unkey/go/pkg/version" +) + +var constLabels = prometheus.Labels{ + "region": os.Getenv("UNKEY_REGION"), + "version": version.Version, +} + +var workflowLatencyBuckets = []float64{ + 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, + 10.0, 30.0, 60.0, 120.0, 300.0, 600.0, +} + +var stepLatencyBuckets = []float64{ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, + 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, +} + +var dbLatencyBuckets = []float64{ + 0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, + 0.2, 0.5, 1.0, 2.0, 5.0, +} + +var WorkflowsStartedTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "workflows_started_total", + Help: "Total number of workflows started", + ConstLabels: constLabels, + }, + []string{"namespace", "workflow_name", "trigger_type"}, +) + +var WorkflowsCompletedTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "workflows_completed_total", + Help: "Total number of workflows completed", + ConstLabels: constLabels, + }, + []string{"namespace", "workflow_name", "status"}, +) + +var WorkflowsRetriedTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "workflows_retried_total", + Help: "Total number of workflow retry attempts", + ConstLabels: constLabels, + }, + []string{"namespace", "workflow_name", "attempt"}, +) + +var WorkflowDurationSeconds = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: "hydra", + Name: "workflow_duration_seconds", + Help: "Time taken to complete workflows", + ConstLabels: constLabels, + Buckets: workflowLatencyBuckets, + }, + []string{"namespace", "workflow_name", "status"}, +) + +var WorkflowQueueTimeSeconds = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: "hydra", + Name: "workflow_queue_time_seconds", + Help: "Time workflow spent queued before execution", + ConstLabels: constLabels, + Buckets: workflowLatencyBuckets, + }, + []string{"namespace", "workflow_name"}, +) + +var WorkflowsActive = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "hydra", + Name: "workflows_active", + Help: "Currently running workflows", + ConstLabels: constLabels, + }, + []string{"namespace", "worker_id"}, +) + +var WorkflowsQueued = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "hydra", + Name: "workflows_queued", + Help: "Workflows waiting to be processed", + ConstLabels: constLabels, + }, + []string{"namespace", "status"}, +) + +var StepsExecutedTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "steps_executed_total", + Help: "Total number of workflow steps executed", + ConstLabels: constLabels, + }, + []string{"namespace", "workflow_name", "step_name", "status"}, +) + +var StepsRetriedTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "steps_retried_total", + Help: "Total number of step retry attempts", + ConstLabels: constLabels, + }, + []string{"namespace", "workflow_name", "step_name"}, +) + +var StepsCachedTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "steps_cached_total", + Help: "Steps skipped due to checkpointing", + ConstLabels: constLabels, + }, + []string{"namespace", "workflow_name", "step_name"}, +) + +var StepDurationSeconds = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: "hydra", + Name: "step_duration_seconds", + Help: "Time taken to execute workflow steps", + ConstLabels: constLabels, + Buckets: stepLatencyBuckets, + }, + []string{"namespace", "workflow_name", "step_name", "status"}, +) + +var WorkerHeartbeatsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "worker_heartbeats_total", + Help: "Total number of worker heartbeat operations", + ConstLabels: constLabels, + }, + []string{"worker_id", "namespace", "status"}, +) + +var LeaseAcquisitionsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "lease_acquisitions_total", + Help: "Total number of lease acquisition attempts", + ConstLabels: constLabels, + }, + []string{"worker_id", "resource_type", "status"}, +) + +var WorkerPollsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "worker_polls_total", + Help: "Total number of worker polling operations", + ConstLabels: constLabels, + }, + []string{"worker_id", "namespace", "status"}, +) + +var WorkerConcurrencyCurrent = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "hydra", + Name: "worker_concurrency_current", + Help: "Current workflow concurrency per worker", + ConstLabels: constLabels, + }, + []string{"worker_id", "namespace"}, +) + +var DbOperationsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "db_operations_total", + Help: "Total number of database operations", + ConstLabels: constLabels, + }, + []string{"operation", "table", "status"}, +) + +var DbOperationDurationSeconds = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: "hydra", + Name: "db_operation_duration_seconds", + Help: "Time taken for database operations", + ConstLabels: constLabels, + Buckets: dbLatencyBuckets, + }, + []string{"operation", "table", "status"}, +) + +var DbConnectionsActive = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "hydra", + Name: "db_connections_active", + Help: "Active database connections", + ConstLabels: constLabels, + }, + []string{"worker_id"}, +) + +var SleepsStartedTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "sleeps_started_total", + Help: "Total number of sleep operations initiated", + ConstLabels: constLabels, + }, + []string{"namespace", "workflow_name"}, +) + +var SleepsResumedTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "sleeps_resumed_total", + Help: "Total number of workflows resumed from sleep", + ConstLabels: constLabels, + }, + []string{"namespace", "workflow_name"}, +) + +var CronTriggersTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "cron_triggers_total", + Help: "Total number of cron-triggered workflows", + ConstLabels: constLabels, + }, + []string{"namespace", "cron_name", "status"}, +) + +var SleepDurationSeconds = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: "hydra", + Name: "sleep_duration_seconds", + Help: "Actual sleep durations", + ConstLabels: constLabels, + Buckets: workflowLatencyBuckets, + }, + []string{"namespace", "workflow_name"}, +) + +var CronExecutionLatencySeconds = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: "hydra", + Name: "cron_execution_latency_seconds", + Help: "Delay between scheduled and actual cron execution", + ConstLabels: constLabels, + Buckets: []float64{0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 300.0}, + }, + []string{"namespace", "cron_name"}, +) + +var WorkflowsSleeping = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: "hydra", + Name: "workflows_sleeping", + Help: "Currently sleeping workflows", + ConstLabels: constLabels, + }, + []string{"namespace"}, +) + +var ErrorsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "errors_total", + Help: "Total number of errors across all components", + ConstLabels: constLabels, + }, + []string{"namespace", "component", "error_type"}, +) + +var PanicsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "panics_total", + Help: "Total number of panic recoveries", + ConstLabels: constLabels, + }, + []string{"worker_id", "component"}, +) + +var TimeoutsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "timeouts_total", + Help: "Total number of operation timeouts", + ConstLabels: constLabels, + }, + []string{"namespace", "operation_type"}, +) + +var PayloadSizeBytes = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: "hydra", + Name: "payload_size_bytes", + Help: "Size of workflow and step payloads", + ConstLabels: constLabels, + Buckets: []float64{100, 500, 1000, 5000, 10000, 50000, 100000, 500000, 1000000}, + }, + []string{"namespace", "workflow_name", "direction"}, +) + +var SerializationErrorsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "hydra", + Name: "serialization_errors_total", + Help: "Total number of payload serialization errors", + ConstLabels: constLabels, + }, + []string{"namespace", "workflow_name", "direction"}, +) + +func ObserveWorkflowDuration(namespace, workflowName, status string, start time.Time) { + duration := time.Since(start) + WorkflowDurationSeconds.WithLabelValues(namespace, workflowName, status).Observe(duration.Seconds()) +} + +func ObserveStepDuration(namespace, workflowName, stepName, status string, start time.Time) { + duration := time.Since(start) + StepDurationSeconds.WithLabelValues(namespace, workflowName, stepName, status).Observe(duration.Seconds()) +} + +func ObserveDbOperation(operation, table, status string, start time.Time) { + duration := time.Since(start) + DbOperationsTotal.WithLabelValues(operation, table, status).Inc() + DbOperationDurationSeconds.WithLabelValues(operation, table, status).Observe(duration.Seconds()) +} + +func RecordError(namespace, component, errorType string) { + ErrorsTotal.WithLabelValues(namespace, component, errorType).Inc() +} + +func RecordPayloadSize(namespace, workflowName, direction string, size int) { + PayloadSizeBytes.WithLabelValues(namespace, workflowName, direction).Observe(float64(size)) +} diff --git a/go/pkg/hydra/minimal-example b/go/pkg/hydra/minimal-example new file mode 100755 index 00000000000..2da22c46f81 Binary files /dev/null and b/go/pkg/hydra/minimal-example differ diff --git a/go/pkg/hydra/models.go b/go/pkg/hydra/models.go new file mode 100644 index 00000000000..f6c46cede03 --- /dev/null +++ b/go/pkg/hydra/models.go @@ -0,0 +1,101 @@ +// Package hydra model types and constants +package hydra + +import "github.com/unkeyed/unkey/go/pkg/hydra/store" + +// WorkflowExecution represents a single workflow execution instance. +// It contains all metadata about a workflow run including status, +// timing, retry configuration, and trigger information. +type WorkflowExecution = store.WorkflowExecution + +// WorkflowStep represents a single step within a workflow execution. +// Steps track their execution state, results, and retry attempts +// to enable checkpointing and exactly-once execution. +type WorkflowStep = store.WorkflowStep + +// CronJob represents a scheduled workflow with cron-style timing. +// Cron jobs automatically create new workflow executions based +// on their schedule configuration. +type CronJob = store.CronJob + +// Lease represents an exclusive lock on a workflow or step. +// Leases prevent multiple workers from executing the same +// work simultaneously and include heartbeat mechanisms. +type Lease = store.Lease + +// WorkflowStatus represents the current state of a workflow execution. +type WorkflowStatus = store.WorkflowStatus + +// StepStatus represents the current state of a workflow step. +type StepStatus = store.StepStatus + +// LeaseKind identifies the type of resource being leased. +type LeaseKind = store.LeaseKind + +// TriggerType identifies how a workflow execution was initiated. +type TriggerType = store.TriggerType + +const ( + // Workflow status constants + + // WorkflowStatusPending indicates a workflow is waiting to be picked up by a worker. + WorkflowStatusPending = store.WorkflowStatusPending + + // WorkflowStatusRunning indicates a workflow is currently being executed by a worker. + WorkflowStatusRunning = store.WorkflowStatusRunning + + // WorkflowStatusSleeping indicates a workflow is suspended and waiting for a timer or external event. + WorkflowStatusSleeping = store.WorkflowStatusSleeping + + // WorkflowStatusCompleted indicates a workflow has finished successfully. + WorkflowStatusCompleted = store.WorkflowStatusCompleted + + // WorkflowStatusFailed indicates a workflow has failed and will not be retried. + WorkflowStatusFailed = store.WorkflowStatusFailed + + // Step status constants + + // StepStatusPending indicates a step is waiting to be executed. + StepStatusPending = store.StepStatusPending + + // StepStatusRunning indicates a step is currently being executed. + StepStatusRunning = store.StepStatusRunning + + // StepStatusCompleted indicates a step has finished successfully and its result is cached. + StepStatusCompleted = store.StepStatusCompleted + + // StepStatusFailed indicates a step has failed and may be retried. + StepStatusFailed = store.StepStatusFailed + + // Lease kind constants + + // LeaseKindWorkflow indicates a lease on an entire workflow execution. + LeaseKindWorkflow = store.LeaseKindWorkflow + + // LeaseKindStep indicates a lease on a specific workflow step. + LeaseKindStep = store.LeaseKindStep + + // LeaseKindCronJob indicates a lease on a cron job execution. + LeaseKindCronJob = store.LeaseKindCronJob + + // Trigger type constants + + // TriggerTypeManual indicates a workflow was started manually by an operator. + TriggerTypeManual = store.TriggerTypeManual + + // TriggerTypeCron indicates a workflow was started by a cron schedule. + TriggerTypeCron = store.TriggerTypeCron + + // TriggerTypeEvent indicates a workflow was started by an external event. + TriggerTypeEvent = store.TriggerTypeEvent + + // TriggerTypeAPI indicates a workflow was started via the API. + TriggerTypeAPI = store.TriggerTypeAPI +) + +// RawPayload wraps raw byte data for workflow payloads. +// This type is used internally when the payload type is not known +// at compile time or when deserializing from the database. +type RawPayload struct { + Data []byte `json:"data"` +} diff --git a/go/pkg/hydra/simple_consistency_test.go b/go/pkg/hydra/simple_consistency_test.go new file mode 100644 index 00000000000..9d195025142 --- /dev/null +++ b/go/pkg/hydra/simple_consistency_test.go @@ -0,0 +1,264 @@ +package hydra + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/unkeyed/unkey/go/pkg/clock" + "github.com/unkeyed/unkey/go/pkg/hydra/store" + "github.com/unkeyed/unkey/go/pkg/hydra/testharness" +) + +// TestSimpleDataConsistency tests basic data consistency using event collection +func TestSimpleDataConsistency(t *testing.T) { + realClock := clock.New() + engine := newTestEngineWithClock(t, realClock) + + const numWorkflows = 10 + + // Create event collector + eventCollector := testharness.NewEventCollector() + + // Create event-aware workflow + workflow := &eventTrackingWorkflow{ + engine: engine, + name: "simple-consistency-workflow", + collector: eventCollector, + } + + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second) + defer cancel() + + // Start a single worker + worker, err := NewWorker(engine, WorkerConfig{ + WorkerID: "simple-consistency-worker", + Concurrency: 2, + PollInterval: 100 * time.Millisecond, + HeartbeatInterval: 1 * time.Second, + ClaimTimeout: 5 * time.Second, + }) + require.NoError(t, err) + + err = RegisterWorkflow(worker, workflow) + require.NoError(t, err) + + err = worker.Start(ctx) + require.NoError(t, err) + defer worker.Shutdown(ctx) + + // Submit workflows + workflowIDs := make([]string, numWorkflows) + for i := 0; i < numWorkflows; i++ { + workflowID, startErr := workflow.Start(ctx, fmt.Sprintf("payload-%d", i)) + require.NoError(t, startErr) + workflowIDs[i] = workflowID + } + + // Wait for all workflows to finish using event collection + require.Eventually(t, func() bool { + completedEvents := eventCollector.Count(testharness.WorkflowCompleted) + failedEvents := eventCollector.Count(testharness.WorkflowFailed) + totalFinished := completedEvents + failedEvents + + return totalFinished == numWorkflows + }, 10*time.Second, 200*time.Millisecond, "All workflows should finish") + + // Verify exactly-once execution using events + for _, workflowID := range workflowIDs { + // Verify exactly one started event + startedEvents := eventCollector.FilterWithData(testharness.WorkflowStarted, "execution_id", workflowID) + require.Len(t, startedEvents, 1, "Workflow %s should start exactly once", workflowID) + + // Verify exactly one completion event + completedEvents := eventCollector.FilterWithData(testharness.WorkflowCompleted, "execution_id", workflowID) + failedEvents := eventCollector.FilterWithData(testharness.WorkflowFailed, "execution_id", workflowID) + totalCompletions := len(completedEvents) + len(failedEvents) + require.Equal(t, 1, totalCompletions, "Workflow %s should complete exactly once", workflowID) + + // Verify exactly one step execution + stepExecutingEvents := eventCollector.FilterWithData(testharness.StepExecuting, "execution_id", workflowID) + require.Len(t, stepExecutingEvents, 1, "Workflow %s should have exactly one step execution", workflowID) + } + + // Verify database consistency + allWorkflows, err := engine.store.GetAllWorkflows(context.Background(), engine.GetNamespace()) + require.NoError(t, err) + + completedInDB := 0 + for _, wf := range allWorkflows { + if wf.Status == store.WorkflowStatusCompleted { + completedInDB++ + } + } + + completedEventsCount := eventCollector.Count(testharness.WorkflowCompleted) + require.Equal(t, completedEventsCount, completedInDB, + "Database completed count should match completed events") + +} + +// TestConcurrentWorkerConsistency tests consistency with multiple workers using events +func TestConcurrentWorkerConsistency(t *testing.T) { + realClock := clock.New() + engine := newTestEngineWithClock(t, realClock) + + const ( + numWorkers = 3 + numWorkflows = 15 + ) + + // Create event collector + eventCollector := testharness.NewEventCollector() + + // Create event-aware workflow + workflow := &eventTrackingWorkflow{ + engine: engine, + name: "concurrent-consistency-workflow", + collector: eventCollector, + } + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + // Start multiple workers + workers := make([]Worker, numWorkers) + for i := 0; i < numWorkers; i++ { + worker, err := NewWorker(engine, WorkerConfig{ + WorkerID: fmt.Sprintf("concurrent-worker-%d", i), + Concurrency: 2, + PollInterval: 50 * time.Millisecond, + HeartbeatInterval: 1 * time.Second, + ClaimTimeout: 5 * time.Second, + }) + require.NoError(t, err) + + err = RegisterWorkflow(worker, workflow) + require.NoError(t, err) + + err = worker.Start(ctx) + require.NoError(t, err) + defer worker.Shutdown(ctx) + + workers[i] = worker + } + + // Submit workflows + workflowIDs := make([]string, numWorkflows) + for i := 0; i < numWorkflows; i++ { + workflowID, err := workflow.Start(ctx, fmt.Sprintf("concurrent-payload-%d", i)) + require.NoError(t, err) + workflowIDs[i] = workflowID + } + + // Wait for workflows to finish using event collection + require.Eventually(t, func() bool { + completedEvents := eventCollector.Count(testharness.WorkflowCompleted) + failedEvents := eventCollector.Count(testharness.WorkflowFailed) + totalFinished := completedEvents + failedEvents + + return totalFinished == numWorkflows + }, 12*time.Second, 300*time.Millisecond, "All concurrent workflows should finish") + + // Verify exactly-once execution for each workflow + duplicateExecutions := 0 + duplicateCompletions := 0 + + for _, workflowID := range workflowIDs { + // Check for duplicate workflow executions + startedEvents := eventCollector.FilterWithData(testharness.WorkflowStarted, "execution_id", workflowID) + if len(startedEvents) > 1 { + duplicateExecutions++ + t.Errorf("DUPLICATE EXECUTION: Workflow %s started %d times", workflowID, len(startedEvents)) + } + require.Len(t, startedEvents, 1, "Workflow %s should start exactly once", workflowID) + + // Check for duplicate completions + completedEvents := eventCollector.FilterWithData(testharness.WorkflowCompleted, "execution_id", workflowID) + failedEvents := eventCollector.FilterWithData(testharness.WorkflowFailed, "execution_id", workflowID) + totalCompletions := len(completedEvents) + len(failedEvents) + + if totalCompletions > 1 { + duplicateCompletions++ + t.Errorf("DUPLICATE COMPLETION: Workflow %s completed %d times (%d completed + %d failed)", + workflowID, totalCompletions, len(completedEvents), len(failedEvents)) + } + require.Equal(t, 1, totalCompletions, "Workflow %s should complete exactly once", workflowID) + + // Verify exactly one step execution + stepExecutingEvents := eventCollector.FilterWithData(testharness.StepExecuting, "execution_id", workflowID) + require.Len(t, stepExecutingEvents, 1, "Workflow %s should have exactly one step execution", workflowID) + } + + // Assert no duplicates were found + require.Equal(t, 0, duplicateExecutions, "Should have zero duplicate workflow executions") + require.Equal(t, 0, duplicateCompletions, "Should have zero duplicate workflow completions") + + // Verify database consistency + allWorkflows, err := engine.store.GetAllWorkflows(context.Background(), engine.GetNamespace()) + require.NoError(t, err) + + completedInDB := 0 + for _, wf := range allWorkflows { + if wf.Status == store.WorkflowStatusCompleted { + completedInDB++ + } + } + + completedEventsCount := eventCollector.Count(testharness.WorkflowCompleted) + require.Equal(t, completedEventsCount, completedInDB, + "Database completed count should match completed events") + +} + +// eventTrackingWorkflow emits events during execution for testing +type eventTrackingWorkflow struct { + engine *Engine + name string + collector *testharness.EventCollector +} + +func (w *eventTrackingWorkflow) Name() string { + return w.name +} + +func (w *eventTrackingWorkflow) Run(ctx WorkflowContext, req any) error { + // Emit workflow started event + w.collector.Emit(ctx, testharness.WorkflowStarted, "Workflow execution started") + + // Emit step executing event + w.collector.Emit(ctx, testharness.StepExecuting, "Step execution started", "step_name", "consistency-step") + + // Execute the step + result, err := Step(ctx, "consistency-step", func(stepCtx context.Context) (string, error) { + // Simulate some work + time.Sleep(20 * time.Millisecond) + return "step-completed", nil + }) + + if err != nil { + // Emit step failed event + w.collector.Emit(ctx, testharness.StepFailed, "Step execution failed", + "step_name", "consistency-step", "error", err.Error()) + + // Emit workflow failed event + w.collector.Emit(ctx, testharness.WorkflowFailed, "Workflow execution failed", "error", err.Error()) + + return err + } + + // Emit step executed event + w.collector.Emit(ctx, testharness.StepExecuted, "Step execution completed", + "step_name", "consistency-step", "result", result) + + // Emit workflow completed event + w.collector.Emit(ctx, testharness.WorkflowCompleted, "Workflow execution completed") + + return nil +} + +func (w *eventTrackingWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} diff --git a/go/pkg/hydra/sleep.go b/go/pkg/hydra/sleep.go new file mode 100644 index 00000000000..b73e5b483ff --- /dev/null +++ b/go/pkg/hydra/sleep.go @@ -0,0 +1,93 @@ +package hydra + +import ( + "fmt" + "time" + + "github.com/unkeyed/unkey/go/pkg/ptr" +) + +// Sleep suspends workflow execution for the specified duration. +// +// This function allows workflows to pause execution and resume after +// a specified time period. The workflow will be marked as sleeping +// and workers will not attempt to execute it until the sleep duration +// has elapsed. +// +// Sleep is useful for: +// - Time-based coordination (e.g., waiting for settlement periods) +// - Human approval workflows (e.g., waiting for manual intervention) +// - Rate limiting and backoff strategies +// - Scheduled processing windows +// +// The sleep duration is durable - if the worker crashes or restarts +// during the sleep period, the workflow will still resume at the +// correct time. +// +// Example usage: +// +// // Sleep for 24 hours for manual approval +// err = hydra.Sleep(ctx, 24*time.Hour) +// if err != nil { +// return err +// } +// +// // Continue with post-approval processing +// result, err := hydra.Step(ctx, "post-approval", func(stepCtx context.Context) (string, error) { +// return processApprovedRequest(stepCtx) +// }) +// +// Note: Sleep creates an internal step to track the sleep state. +// The step name is generated automatically based on the duration. +// +// Metrics recorded: +// - hydra_sleeps_started_total (counter) +// - hydra_workflows_sleeping (gauge) +func Sleep(ctx WorkflowContext, duration time.Duration) error { + wctx, ok := ctx.(*workflowContext) + if !ok { + return fmt.Errorf("invalid workflow context") + } + + stepName := fmt.Sprintf("sleep-%d", duration.Milliseconds()) + + existing, err := wctx.getCompletedStep(stepName) + if err == nil && existing != nil { + return nil + } + + now := time.Now().UnixMilli() + existingStep, err := wctx.getAnyStep(stepName) + if err == nil && existingStep != nil && existingStep.StartedAt != nil { + sleepUntil := *existingStep.StartedAt + duration.Milliseconds() + + if sleepUntil <= now { + return wctx.markStepCompleted(existingStep.ID, []byte("{}")) + } + return wctx.suspendWorkflowForSleep(sleepUntil) + } + + sleepUntil := now + duration.Milliseconds() + + step := &WorkflowStep{ + ID: "", + ExecutionID: wctx.ExecutionID(), + StepName: stepName, + StepOrder: wctx.getNextStepOrder(), + Status: StepStatusRunning, + Namespace: wctx.namespace, + StartedAt: ptr.P(now), + OutputData: nil, + ErrorMessage: "", + MaxAttempts: 1, // Sleep doesn't need retries + RemainingAttempts: 1, + CompletedAt: nil, + } + + err = wctx.store.CreateStep(wctx.ctx, step) + if err != nil { + return fmt.Errorf("failed to create sleep step: %w", err) + } + + return wctx.suspendWorkflowForSleep(sleepUntil) +} diff --git a/go/pkg/hydra/step.go b/go/pkg/hydra/step.go new file mode 100644 index 00000000000..6ff7fcfc932 --- /dev/null +++ b/go/pkg/hydra/step.go @@ -0,0 +1,180 @@ +package hydra + +import ( + "context" + "fmt" + "reflect" + "time" + + "github.com/unkeyed/unkey/go/pkg/hydra/metrics" + "github.com/unkeyed/unkey/go/pkg/ptr" +) + +// Step executes a named step within a workflow with automatic checkpointing and retry logic. +// +// Steps are the fundamental units of work in Hydra workflows. They provide: +// - Exactly-once execution guarantees +// - Automatic result caching (checkpointing) +// - Built-in retry logic for transient failures +// - Comprehensive metrics and observability +// +// Parameters: +// - ctx: The workflow context from the workflow's Run() method +// - stepName: A unique name for this step within the workflow +// - fn: The function to execute, which should be idempotent +// +// The stepName must be unique within the workflow and should remain stable +// across deployments. If a step has already completed successfully, its +// cached result will be returned without re-executing the function. +// +// The function fn receives a standard Go context and should: +// - Be idempotent (safe to run multiple times) +// - Handle context cancellation gracefully +// - Return consistent results for the same inputs +// - Use the provided context for any I/O operations +// +// Example usage: +// +// // Simple step with string result +// result, err := hydra.Step(ctx, "fetch-user", func(stepCtx context.Context) (string, error) { +// user, err := userService.GetUser(stepCtx, userID) +// if err != nil { +// return "", err +// } +// return user.Name, nil +// }) +// +// // Step with complex result type +// order, err := hydra.Step(ctx, "create-order", func(stepCtx context.Context) (*Order, error) { +// return orderService.CreateOrder(stepCtx, &CreateOrderRequest{ +// CustomerID: customerID, +// Items: items, +// }) +// }) +// +// Metrics recorded: +// - hydra_steps_executed_total (counter with status) +// - hydra_step_duration_seconds (histogram) +// - hydra_steps_cached_total (counter for cache hits) +// - hydra_steps_retried_total (counter for retry attempts) +// +// Returns the result of the function execution or the cached result if the +// step has already completed successfully. +func Step[TResponse any](ctx WorkflowContext, stepName string, fn func(context.Context) (TResponse, error)) (TResponse, error) { + var zero TResponse + + wctx, ok := ctx.(*workflowContext) + if !ok { + return zero, fmt.Errorf("invalid workflow context") + } + + existing, err := wctx.getCompletedStep(stepName) + if err == nil && existing != nil { + // Record cached step hit + metrics.StepsCachedTotal.WithLabelValues(wctx.namespace, wctx.workflowName, stepName).Inc() + + responseType := reflect.TypeOf((*TResponse)(nil)).Elem() + var response TResponse + + if responseType.Kind() == reflect.Ptr { + responseValue := reflect.New(responseType.Elem()) + var ok bool + response, ok = responseValue.Interface().(TResponse) + if !ok { + return zero, fmt.Errorf("failed to convert response to expected type") + } + } + + if len(existing.OutputData) > 0 { + err = wctx.marshaller.Unmarshal(existing.OutputData, &response) + if err != nil { + metrics.RecordError(wctx.namespace, "step", "unmarshal_cached_result_failed") + return zero, fmt.Errorf("failed to unmarshal cached step result: %w", err) + } + } + + return response, nil + } + + existingStep, err := wctx.getAnyStep(stepName) + var stepToUse *WorkflowStep + shouldCreateNewStep := true + + if err == nil && existingStep != nil { + stepToUse = existingStep + shouldCreateNewStep = false + } + + stepStartTime := time.Now() + + if shouldCreateNewStep { + stepToUse = &WorkflowStep{ + ID: "", + ExecutionID: wctx.ExecutionID(), + StepName: stepName, + StepOrder: wctx.getNextStepOrder(), + Status: StepStatusRunning, + Namespace: wctx.namespace, + StartedAt: ptr.P(stepStartTime.UnixMilli()), + OutputData: nil, + ErrorMessage: "", + MaxAttempts: wctx.stepMaxAttempts, + RemainingAttempts: wctx.stepMaxAttempts, + CompletedAt: nil, + } + + err = wctx.store.CreateStep(wctx.ctx, stepToUse) + if err != nil { + metrics.RecordError(wctx.namespace, "step", "create_step_failed") + return zero, fmt.Errorf("failed to create step: %w", err) + } + } else { + stepToUse.Status = StepStatusRunning + stepToUse.StartedAt = ptr.P(stepStartTime.UnixMilli()) + stepToUse.ErrorMessage = "" + stepToUse.CompletedAt = nil + + err = wctx.store.UpdateStepStatus(wctx.ctx, wctx.namespace, wctx.executionID, stepName, StepStatusRunning, nil, "") + if err != nil { + metrics.RecordError(wctx.namespace, "step", "update_step_failed") + return zero, fmt.Errorf("failed to update step: %w", err) + } + + // Record step retry + if stepToUse.RemainingAttempts < stepToUse.MaxAttempts { + metrics.StepsRetriedTotal.WithLabelValues(wctx.namespace, wctx.workflowName, stepName).Inc() + } + } + + response, err := fn(wctx.ctx) + if err != nil { + if markErr := wctx.markStepFailed(stepName, err.Error()); markErr != nil { + metrics.RecordError(wctx.namespace, "step", "mark_step_failed_error") + } + metrics.ObserveStepDuration(wctx.namespace, wctx.workflowName, stepName, "failed", stepStartTime) + metrics.StepsExecutedTotal.WithLabelValues(wctx.namespace, wctx.workflowName, stepName, "failed").Inc() + return zero, fmt.Errorf("step execution failed: %w", err) + } + + respData, err := wctx.marshaller.Marshal(response) + if err != nil { + if markErr := wctx.markStepFailed(stepName, err.Error()); markErr != nil { + metrics.RecordError(wctx.namespace, "step", "mark_step_failed_error") + } + metrics.ObserveStepDuration(wctx.namespace, wctx.workflowName, stepName, "failed", stepStartTime) + metrics.StepsExecutedTotal.WithLabelValues(wctx.namespace, wctx.workflowName, stepName, "failed").Inc() + metrics.RecordError(wctx.namespace, "step", "marshal_response_failed") + return zero, fmt.Errorf("failed to marshal response: %w", err) + } + + err = wctx.markStepCompleted(stepName, respData) + if err != nil { + metrics.RecordError(wctx.namespace, "step", "mark_completed_failed") + return zero, fmt.Errorf("failed to mark step completed: %w", err) + } + + metrics.ObserveStepDuration(wctx.namespace, wctx.workflowName, stepName, "completed", stepStartTime) + metrics.StepsExecutedTotal.WithLabelValues(wctx.namespace, wctx.workflowName, stepName, "completed").Inc() + + return response, nil +} diff --git a/go/pkg/hydra/step_atomicity_test.go b/go/pkg/hydra/step_atomicity_test.go new file mode 100644 index 00000000000..5a335fa7dd0 --- /dev/null +++ b/go/pkg/hydra/step_atomicity_test.go @@ -0,0 +1,207 @@ +package hydra + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/unkeyed/unkey/go/pkg/clock" + "github.com/unkeyed/unkey/go/pkg/uid" +) + +// TestStepExecutionAtomicity ensures that step execution is atomic: +// either a step fully completes (executes + status update) or it doesn't execute at all. +// This prevents duplicate side effects when status updates fail after step execution. +func TestStepExecutionAtomicity(t *testing.T) { + // Arrange: Create engine with test clock and a workflow that tracks execution attempts + testClock := clock.NewTestClock() + engine := newTestEngineWithClock(t, testClock) + + var stepExecutionCount int64 + var sideEffectsCount int64 // Track side effects that should only happen once + + // Create a workflow with a step that has side effects + workflow := &atomicityTestWorkflow{ + engine: engine, + name: "atomicity-test-workflow", + stepFunc: func(ctx context.Context) (string, error) { + // This represents the step execution with side effects + _ = atomic.AddInt64(&stepExecutionCount, 1) + + // Simulate important side effects (e.g., sending email, charging payment, etc.) + atomic.AddInt64(&sideEffectsCount, 1) + + return "step-result", nil + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Act: Start workflow + executionID, err := workflow.Start(ctx, struct{}{}) + require.NoError(t, err) + + // Start worker + worker, err := NewWorker(engine, WorkerConfig{ + Concurrency: 1, + PollInterval: 100 * time.Millisecond, + HeartbeatInterval: 1 * time.Second, + ClaimTimeout: 5 * time.Second, + }) + require.NoError(t, err) + + err = RegisterWorkflow(worker, workflow) + require.NoError(t, err) + + err = worker.Start(ctx) + require.NoError(t, err) + defer worker.Shutdown(ctx) + + // Trigger workflow execution + require.Eventually(t, func() bool { + testClock.Tick(200 * time.Millisecond) + time.Sleep(10 * time.Millisecond) + + // Check if workflow completed + currentStatus, getErr := engine.store.GetWorkflow(ctx, engine.GetNamespace(), executionID) + if getErr != nil { + return false + } + return currentStatus.Status == WorkflowStatusCompleted + }, 5*time.Second, 50*time.Millisecond, "Workflow should complete") + + // Assert: Step should execute exactly once despite any potential failures + finalExecutionCount := atomic.LoadInt64(&stepExecutionCount) + finalSideEffectsCount := atomic.LoadInt64(&sideEffectsCount) + + require.Equal(t, int64(1), finalExecutionCount, + "ATOMICITY VIOLATION: Step executed %d times instead of 1. "+ + "This indicates non-atomic step execution where the step ran multiple times.", finalExecutionCount) + + require.Equal(t, int64(1), finalSideEffectsCount, + "SIDE EFFECT DUPLICATION: Side effects occurred %d times instead of 1. "+ + "This could mean duplicate emails sent, multiple payments charged, etc.", finalSideEffectsCount) + + // Verify the workflow completed successfully + finalWorkflow, err := engine.store.GetWorkflow(ctx, engine.GetNamespace(), executionID) + require.NoError(t, err) + require.Equal(t, WorkflowStatusCompleted, finalWorkflow.Status, + "Workflow should complete successfully") + +} + +// TestStepExecutionAtomicityWithFailures tests atomicity when database operations fail +func TestStepExecutionAtomicityWithFailures(t *testing.T) { + // This test would be more complex and would require mocking the store + // to simulate failures during status updates after step execution. + // For now, we'll focus on the basic atomicity test above. + t.Skip("TODO: Implement test with simulated database failures during status updates") +} + +// TestConcurrentStepExecution tests that multiple workers don't execute the same step +func TestConcurrentStepExecution(t *testing.T) { + // Arrange: Create engine with test clock + testClock := clock.NewTestClock() + engine := newTestEngineWithClock(t, testClock) + + var stepExecutionCount int64 + + // Create a workflow with a step that takes time to execute + workflow := &atomicityTestWorkflow{ + engine: engine, + name: "concurrent-test-workflow", + stepFunc: func(ctx context.Context) (string, error) { + _ = atomic.AddInt64(&stepExecutionCount, 1) + + // Simulate some work time + time.Sleep(100 * time.Millisecond) + + return "concurrent-result", nil + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Start workflow + executionID, err := workflow.Start(ctx, struct{}{}) + require.NoError(t, err) + + // Start multiple workers that might try to process the same workflow + worker1ID := uid.New(uid.WorkerPrefix) + worker2ID := uid.New(uid.WorkerPrefix) + + worker1, err := NewWorker(engine, WorkerConfig{ + WorkerID: worker1ID, + Concurrency: 1, + PollInterval: 50 * time.Millisecond, + HeartbeatInterval: 1 * time.Second, + ClaimTimeout: 5 * time.Second, + }) + require.NoError(t, err) + + worker2, err := NewWorker(engine, WorkerConfig{ + WorkerID: worker2ID, + Concurrency: 1, + PollInterval: 50 * time.Millisecond, + HeartbeatInterval: 1 * time.Second, + ClaimTimeout: 5 * time.Second, + }) + require.NoError(t, err) + + err = RegisterWorkflow(worker1, workflow) + require.NoError(t, err) + err = RegisterWorkflow(worker2, workflow) + require.NoError(t, err) + + err = worker1.Start(ctx) + require.NoError(t, err) + defer worker1.Shutdown(ctx) + + err = worker2.Start(ctx) + require.NoError(t, err) + defer worker2.Shutdown(ctx) + + // Trigger both workers to poll simultaneously + require.Eventually(t, func() bool { + testClock.Tick(100 * time.Millisecond) + time.Sleep(20 * time.Millisecond) + + // Check if workflow completed + currentStatus, err := engine.store.GetWorkflow(ctx, engine.GetNamespace(), executionID) + if err != nil { + return false + } + return currentStatus.Status == WorkflowStatusCompleted + }, 5*time.Second, 50*time.Millisecond, "Workflow should complete with concurrent workers") + + // Assert: Step should execute exactly once even with multiple workers + finalExecutionCount := atomic.LoadInt64(&stepExecutionCount) + require.Equal(t, int64(1), finalExecutionCount, + "CONCURRENCY VIOLATION: Step executed %d times instead of 1. "+ + "Multiple workers executed the same step, violating exactly-once guarantees.", finalExecutionCount) + +} + +// atomicityTestWorkflow is a test workflow for testing step execution atomicity +type atomicityTestWorkflow struct { + engine *Engine + name string + stepFunc func(ctx context.Context) (string, error) +} + +func (w *atomicityTestWorkflow) Name() string { + return w.name +} + +func (w *atomicityTestWorkflow) Run(ctx WorkflowContext, req any) error { + _, err := Step(ctx, "atomic-step", w.stepFunc) + return err +} + +func (w *atomicityTestWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} diff --git a/go/pkg/hydra/step_idempotency_test.go b/go/pkg/hydra/step_idempotency_test.go new file mode 100644 index 00000000000..2ecc83249c4 --- /dev/null +++ b/go/pkg/hydra/step_idempotency_test.go @@ -0,0 +1,158 @@ +package hydra + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/unkeyed/unkey/go/pkg/clock" +) + +// TestStepIdempotencyDuringWorkerFailure guarantees that workflow steps are idempotent +// and execute exactly once, even when workers fail and other workers resume the workflow. +// +// This prevents duplicate side effects like sending emails twice, processing payments +// multiple times, or creating duplicate database records during worker handoffs. +func TestStepIdempotencyDuringWorkerFailure(t *testing.T) { + // Arrange: Create engine with test clock for deterministic timing + testClock := clock.NewTestClock() + engine := newTestEngineWithClock(t, testClock) + + var stepExecutionCount int64 + + // Create a workflow with one step that takes time to execute + workflow := &testWorkflow{ + engine: engine, + name: "idempotency-test-workflow", + stepFunc: func(ctx context.Context) (string, error) { + // This should only execute once, but the bug causes it to execute multiple times + atomic.AddInt64(&stepExecutionCount, 1) + + // Step executes instantly - we'll control timing via test clock and worker coordination + return "step-completed", nil + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Act: Start workflow using the preferred pattern + executionID, err := workflow.Start(ctx, struct{}{}) + require.NoError(t, err) + + // Start first worker to begin processing + worker1, err := NewWorker(engine, WorkerConfig{ + Concurrency: 1, + PollInterval: 50 * time.Millisecond, + HeartbeatInterval: 1 * time.Second, // Short heartbeat for faster cleanup + ClaimTimeout: 2 * time.Second, // Short timeout to simulate crash + }) + require.NoError(t, err) + + err = RegisterWorkflow(worker1, workflow) + require.NoError(t, err) + + err = worker1.Start(ctx) + require.NoError(t, err) + + // Let worker1 start processing - advance test clock to trigger polling + testClock.Tick(100 * time.Millisecond) // Trigger initial poll + + // Give a brief moment for worker1 to process (this is unavoidable for goroutine coordination) + time.Sleep(50 * time.Millisecond) + + // Keep triggering polls until workflow is picked up + for i := 0; i < 10; i++ { + testClock.Tick(100 * time.Millisecond) + time.Sleep(10 * time.Millisecond) + + // Check if workflow has been picked up + currentStatus, getErr := engine.store.GetWorkflow(ctx, engine.GetNamespace(), executionID) + require.NoError(t, getErr) + if currentStatus.Status != WorkflowStatusPending { + break + } + } + + // Check that workflow is being processed + _, err = engine.store.GetWorkflow(ctx, engine.GetNamespace(), executionID) + require.NoError(t, err) + + // Simulate worker1 crash by shutting it down + err = worker1.Shutdown(context.Background()) + require.NoError(t, err) + + // Advance time to expire the lease and trigger cleanup + // Leases expire after ClaimTimeout (2 seconds), cleanup runs every HeartbeatInterval * 2 (2 seconds) + testClock.Tick(3 * time.Second) // Advance past lease expiration + cleanup interval + + // Start worker2 to take over the workflow + worker2, err := NewWorker(engine, WorkerConfig{ + Concurrency: 1, + PollInterval: 50 * time.Millisecond, + HeartbeatInterval: 1 * time.Second, // Short heartbeat for faster cleanup + ClaimTimeout: 5 * time.Second, + }) + require.NoError(t, err) + + err = RegisterWorkflow(worker2, workflow) + require.NoError(t, err) + + err = worker2.Start(ctx) + require.NoError(t, err) + defer worker2.Shutdown(ctx) + + // Advance time to trigger worker2 polling and cleanup detection + testClock.Tick(200 * time.Millisecond) // Trigger worker2 polling + + // Keep triggering polls and cleanup until workflow is picked up by worker2 + for i := 0; i < 20; i++ { + testClock.Tick(200 * time.Millisecond) // Trigger polling and cleanup + time.Sleep(10 * time.Millisecond) + + // Check if workflow has been picked up + currentStatus, err := engine.store.GetWorkflow(ctx, engine.GetNamespace(), executionID) + require.NoError(t, err) + if currentStatus.Status != WorkflowStatusPending { + break + } + } + + // Wait for worker2 to complete the workflow + finalResult := waitForWorkflowCompletion(t, engine, executionID, 3*time.Second) + // Check final step execution count + finalCount := atomic.LoadInt64(&stepExecutionCount) + + // Assert: Step idempotency - should execute exactly once despite worker failure + require.Equal(t, int64(1), finalCount, + "STEP IDEMPOTENCY VIOLATION: Step executed %d times instead of 1. "+ + "This could cause duplicate side effects like sending emails twice, "+ + "processing payments multiple times, or creating duplicate records.", finalCount) + + // Verify workflow completed successfully + require.Equal(t, WorkflowStatusCompleted, finalResult.Status, "Workflow should complete successfully despite worker crash") +} + +// testWorkflow is a minimal workflow for testing step idempotency +type testWorkflow struct { + engine *Engine + name string + stepFunc func(ctx context.Context) (string, error) +} + +func (w *testWorkflow) Name() string { + return w.name +} + +func (w *testWorkflow) Run(ctx WorkflowContext, req any) error { + _, err := Step(ctx, "test-step", w.stepFunc) + return err +} + +// Start is a convenience method that starts this workflow using the embedded engine +// This encourages a cleaner API pattern: workflow.Start() instead of engine.StartWorkflow() +func (w *testWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} diff --git a/go/pkg/hydra/store.go b/go/pkg/hydra/store.go new file mode 100644 index 00000000000..16cf00fe33c --- /dev/null +++ b/go/pkg/hydra/store.go @@ -0,0 +1,48 @@ +package hydra + +import ( + "github.com/unkeyed/unkey/go/pkg/clock" + "github.com/unkeyed/unkey/go/pkg/hydra/store" + "github.com/unkeyed/unkey/go/pkg/hydra/store/gorm" + gormDriver "gorm.io/gorm" +) + +// Store defines the persistence interface for workflow state and metadata. +// This interface abstracts the underlying storage implementation to allow +// for different database backends while maintaining the same API. +type Store = store.Store + +// StoreFactory creates Store instances for testing and dependency injection. +// This is primarily used in testing scenarios where multiple store instances +// may be needed or when using dependency injection frameworks. +type StoreFactory = store.StoreFactory + +// NewGORMStore creates a new Store implementation using GORM and the provided database. +// +// This is the primary store implementation for production use. It supports: +// - MySQL, PostgreSQL, and SQLite databases through GORM +// - Automatic schema migration +// - Connection pooling and transaction management +// - Optimized queries with proper indexing +// +// The clock parameter is used for testing with controllable time and should +// typically be clock.New() for production use. +// +// Example: +// +// db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{}) +// if err != nil { +// return err +// } +// +// store := hydra.NewGORMStore(db, clock.New()) +// engine := hydra.New(hydra.Config{ +// Store: store, +// // ... other config +// }) +// +// The database connection should be configured with appropriate timeouts, +// connection limits, and retry logic before passing to this function. +func NewGORMStore(db *gormDriver.DB, clk clock.Clock) Store { + return gorm.NewGORMStore(db, clk) +} diff --git a/go/pkg/hydra/store/gorm/gorm.go b/go/pkg/hydra/store/gorm/gorm.go new file mode 100644 index 00000000000..4c965b2d474 --- /dev/null +++ b/go/pkg/hydra/store/gorm/gorm.go @@ -0,0 +1,589 @@ +package gorm + +import ( + "context" + "errors" + "strings" + "time" + + "github.com/unkeyed/unkey/go/pkg/clock" + "github.com/unkeyed/unkey/go/pkg/hydra/store" + "gorm.io/driver/mysql" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +var ( + ErrStepNotFound = errors.New("step not found") + ErrLeaseNotFound = errors.New("lease not found") + ErrCronJobNotFound = errors.New("cron job not found") + + // Empty structs for GORM Model() calls + emptyWorkflowExecution = &store.WorkflowExecution{ + ID: "", WorkflowName: "", Status: "", InputData: nil, OutputData: nil, + ErrorMessage: "", CreatedAt: 0, StartedAt: nil, CompletedAt: nil, + MaxAttempts: 0, RemainingAttempts: 0, NextRetryAt: nil, Namespace: "", + TriggerType: "", TriggerSource: nil, SleepUntil: nil, TraceID: "", + } + emptyWorkflowStep = &store.WorkflowStep{ + ID: "", ExecutionID: "", StepName: "", StepOrder: 0, Status: "", + OutputData: nil, ErrorMessage: "", StartedAt: nil, CompletedAt: nil, + MaxAttempts: 0, RemainingAttempts: 0, Namespace: "", + } + emptyCronJob = &store.CronJob{ + ID: "", Name: "", CronSpec: "", Namespace: "", WorkflowName: "", + Enabled: false, CreatedAt: 0, UpdatedAt: 0, LastRunAt: nil, NextRunAt: 0, + } + emptyLease = &store.Lease{ + ResourceID: "", Kind: "", Namespace: "", WorkerID: "", + AcquiredAt: 0, ExpiresAt: 0, HeartbeatAt: 0, + } +) + +type gormStore struct { + db *gorm.DB + clock clock.Clock +} + +func NewGORMStore(db *gorm.DB, clk clock.Clock) store.Store { + if clk == nil { + clk = clock.New() // Default to real clock + } + return &gormStore{db: db, clock: clk} +} + +func NewSQLiteStore(dsn string, clk clock.Clock) (store.Store, error) { + if dsn == "" { + dsn = ":memory:" // Default to in-memory + } + + db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + return nil, err + } + + // Auto-migrate the schema + if err := db.AutoMigrate(emptyWorkflowExecution, emptyWorkflowStep, emptyCronJob, emptyLease); err != nil { + return nil, err + } + + return NewGORMStore(db, clk), nil +} + +func NewMySQLStore(dsn string, clk clock.Clock) (store.Store, error) { + if dsn == "" { + return nil, errors.New("MySQL DSN is required") + } + + db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + return nil, err + } + + // Auto-migrate the schema + if err := db.AutoMigrate(emptyWorkflowExecution, emptyWorkflowStep, emptyCronJob, emptyLease); err != nil { + return nil, err + } + + return NewGORMStore(db, clk), nil +} + +func (s *gormStore) CreateWorkflow(ctx context.Context, workflow *store.WorkflowExecution) error { + if workflow.CreatedAt == 0 { + workflow.CreatedAt = s.clock.Now().UnixMilli() + } + return s.db.WithContext(ctx).Create(workflow).Error +} + +func (s *gormStore) GetWorkflow(ctx context.Context, namespace, id string) (*store.WorkflowExecution, error) { + var workflow store.WorkflowExecution + err := s.db.WithContext(ctx). + Where("id = ? AND namespace = ?", id, namespace). + First(&workflow).Error + + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, errors.New("workflow not found") + } + return &workflow, err +} + +func (s *gormStore) GetPendingWorkflows(ctx context.Context, namespace string, limit int, workflowNames []string) ([]store.WorkflowExecution, error) { + return s.GetPendingWorkflowsWithOffset(ctx, namespace, limit, 0, workflowNames) +} + +func (s *gormStore) GetPendingWorkflowsWithOffset(ctx context.Context, namespace string, limit int, offset int, workflowNames []string) ([]store.WorkflowExecution, error) { + now := time.Now().UnixMilli() + var workflows []store.WorkflowExecution + + query := s.db.WithContext(ctx). + Where("namespace = ? AND (status = ? OR (status = ? AND next_retry_at <= ?) OR (status = ? AND sleep_until <= ?))", + namespace, + store.WorkflowStatusPending, + store.WorkflowStatusFailed, + now, + store.WorkflowStatusSleeping, + now, + ) + + if len(workflowNames) > 0 { + query = query.Where("workflow_name IN ?", workflowNames) + } + + err := query. + Order("created_at ASC"). + Offset(offset). + Limit(limit). + Find(&workflows).Error + + return workflows, err +} + +func (s *gormStore) AcquireWorkflowLease(ctx context.Context, workflowID, namespace, workerID string, leaseDuration time.Duration) error { + now := time.Now().UnixMilli() + expiresAt := now + leaseDuration.Milliseconds() + + return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + // First, check if workflow is still available for leasing + var workflow store.WorkflowExecution + err := tx.Where("id = ? AND namespace = ?", workflowID, namespace).First(&workflow).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return errors.New("workflow not found") + } + return err + } + + // Check if workflow is in a leasable state + if workflow.Status != store.WorkflowStatusPending && + workflow.Status != store.WorkflowStatusFailed && + workflow.Status != store.WorkflowStatusSleeping { + return errors.New("workflow not available for acquisition") + } + + // For failed workflows, check if retry time has passed + if workflow.Status == store.WorkflowStatusFailed && workflow.NextRetryAt != nil && *workflow.NextRetryAt > now { + return errors.New("workflow not ready for retry yet") + } + + // For sleeping workflows, check if sleep time has passed + if workflow.Status == store.WorkflowStatusSleeping && workflow.SleepUntil != nil && *workflow.SleepUntil > now { + return errors.New("workflow still sleeping") + } + + // Now check for existing lease + var existingLease store.Lease + err = tx.Where("resource_id = ? AND kind = ?", workflowID, "workflow").First(&existingLease).Error + + switch { + case err == nil: + if existingLease.ExpiresAt > now { + if existingLease.WorkerID != workerID { + return errors.New("workflow already leased by another worker") + } + // Renew existing lease + existingLease.AcquiredAt = now + existingLease.ExpiresAt = expiresAt + existingLease.HeartbeatAt = now + err = tx.Save(&existingLease).Error + if err != nil { + return err + } + } else { + // Take over expired lease + existingLease.WorkerID = workerID + existingLease.AcquiredAt = now + existingLease.ExpiresAt = expiresAt + existingLease.HeartbeatAt = now + err = tx.Save(&existingLease).Error + if err != nil { + return err + } + } + case errors.Is(err, gorm.ErrRecordNotFound): + // Create new lease + lease := &store.Lease{ + ResourceID: workflowID, + Kind: "workflow", + Namespace: namespace, + WorkerID: workerID, + AcquiredAt: now, + ExpiresAt: expiresAt, + HeartbeatAt: now, + } + + createErr := tx.Create(lease).Error + if createErr != nil { + if isDuplicateKeyError(createErr) { + return errors.New("workflow already leased by another worker") + } + return createErr + } + default: + return err + } + + // Update workflow status to running + result := tx.Model(emptyWorkflowExecution). + Where("id = ? AND namespace = ?", workflowID, namespace). + Updates(map[string]any{ + "status": store.WorkflowStatusRunning, + "started_at": gorm.Expr("CASE WHEN started_at IS NULL THEN ? ELSE started_at END", now), + "sleep_until": nil, // Clear sleep_until when waking up + }) + + if result.Error != nil { + return result.Error + } + + return nil + }) +} + +func (s *gormStore) UpdateWorkflowStatus(ctx context.Context, namespace, id string, status store.WorkflowStatus, errorMsg string) error { + updates := map[string]any{ + "status": status, + } + + if errorMsg != "" { + updates["error_message"] = errorMsg + } + + return s.db.WithContext(ctx). + Model(emptyWorkflowExecution). + Where("id = ? AND namespace = ?", id, namespace). + Updates(updates).Error +} + +func (s *gormStore) CompleteWorkflow(ctx context.Context, namespace, id string, outputData []byte) error { + now := time.Now().UnixMilli() + updates := map[string]any{ + "status": store.WorkflowStatusCompleted, + "completed_at": now, + } + + if outputData != nil { + updates["output_data"] = outputData + } + + return s.db.WithContext(ctx). + Model(emptyWorkflowExecution). + Where("id = ? AND namespace = ?", id, namespace). + Updates(updates).Error +} + +func (s *gormStore) FailWorkflow(ctx context.Context, namespace, id string, errorMsg string, isFinal bool) error { + var workflow store.WorkflowExecution + err := s.db.WithContext(ctx). + Where("id = ? AND namespace = ?", id, namespace). + First(&workflow).Error + if err != nil { + return err + } + + now := time.Now().UnixMilli() + workflow.ErrorMessage = errorMsg + workflow.RemainingAttempts-- + + updates := map[string]any{ + "error_message": errorMsg, + "remaining_attempts": workflow.RemainingAttempts, + } + + if isFinal || workflow.RemainingAttempts <= 0 { + updates["status"] = store.WorkflowStatusFailed + updates["completed_at"] = now + updates["next_retry_at"] = nil + } else { + updates["status"] = store.WorkflowStatusFailed + attemptsUsed := workflow.MaxAttempts - workflow.RemainingAttempts + backoffSeconds := int64(1 << attemptsUsed) + nextRetry := now + (backoffSeconds * 1000) + updates["next_retry_at"] = nextRetry + } + + return s.db.WithContext(ctx). + Model(emptyWorkflowExecution). + Where("id = ? AND namespace = ?", id, namespace). + Updates(updates).Error +} + +func (s *gormStore) SleepWorkflow(ctx context.Context, namespace, id string, sleepUntil int64) error { + updates := map[string]any{ + "status": store.WorkflowStatusSleeping, + "sleep_until": sleepUntil, + } + + return s.db.WithContext(ctx). + Model(emptyWorkflowExecution). + Where("id = ? AND namespace = ?", id, namespace). + Updates(updates).Error +} + +func (s *gormStore) GetSleepingWorkflows(ctx context.Context, namespace string, beforeTime int64) ([]store.WorkflowExecution, error) { + var workflows []store.WorkflowExecution + + err := s.db.WithContext(ctx). + Where("namespace = ? AND status = ? AND sleep_until <= ?", + namespace, + store.WorkflowStatusSleeping, + beforeTime, + ). + Order("sleep_until ASC"). + Find(&workflows).Error + + return workflows, err +} + +func (s *gormStore) CreateStep(ctx context.Context, step *store.WorkflowStep) error { + if step.ID == "" { + step.ID = step.ExecutionID + "-" + step.StepName + } + return s.db.WithContext(ctx).Create(step).Error +} + +func (s *gormStore) GetStep(ctx context.Context, namespace, executionID, stepName string) (*store.WorkflowStep, error) { + var step store.WorkflowStep + err := s.db.WithContext(ctx). + Where("namespace = ? AND execution_id = ? AND step_name = ?", + namespace, executionID, stepName). + First(&step).Error + + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrStepNotFound + } + return &step, err +} + +func (s *gormStore) GetCompletedStep(ctx context.Context, namespace, executionID, stepName string) (*store.WorkflowStep, error) { + var step store.WorkflowStep + err := s.db.WithContext(ctx). + Where("namespace = ? AND execution_id = ? AND step_name = ? AND status = ?", + namespace, executionID, stepName, store.StepStatusCompleted). + First(&step).Error + + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrStepNotFound + } + return &step, err +} + +func (s *gormStore) UpdateStepStatus(ctx context.Context, namespace, executionID, stepName string, status store.StepStatus, outputData []byte, errorMsg string) error { + now := time.Now().UnixMilli() + + var step store.WorkflowStep + err := s.db.WithContext(ctx). + Where("namespace = ? AND execution_id = ? AND step_name = ?", namespace, executionID, stepName). + First(&step).Error + if err != nil { + return err + } + + updates := map[string]any{ + "status": status, + "completed_at": now, + } + + if outputData != nil { + updates["output_data"] = outputData + } + + if errorMsg != "" { + updates["error_message"] = errorMsg + } + + return s.db.WithContext(ctx). + Model(emptyWorkflowStep). + Where("namespace = ? AND execution_id = ? AND step_name = ?", namespace, executionID, stepName). + Updates(updates).Error +} + +func (s *gormStore) AcquireLease(ctx context.Context, lease *store.Lease) error { + result := s.db.WithContext(ctx).Create(lease) + if result.Error != nil { + if isDuplicateKeyError(result.Error) { + return errors.New("lease already held by another worker") + } + return result.Error + } + return nil +} + +func (s *gormStore) HeartbeatLease(ctx context.Context, resourceID, workerID string, expiresAt int64) error { + now := time.Now().UnixMilli() + + result := s.db.WithContext(ctx). + Model(emptyLease). + Where("resource_id = ? AND worker_id = ?", resourceID, workerID). + Updates(map[string]any{ + "heartbeat_at": now, + "expires_at": expiresAt, + }) + + if result.Error != nil { + return result.Error + } + + if result.RowsAffected == 0 { + return errors.New("lease not found or not owned by worker") + } + + return nil +} + +func (s *gormStore) ReleaseLease(ctx context.Context, resourceID, workerID string) error { + result := s.db.WithContext(ctx). + Where("resource_id = ? AND worker_id = ?", resourceID, workerID). + Delete(emptyLease) + + if result.Error != nil { + return result.Error + } + + if result.RowsAffected == 0 { + return errors.New("lease not found or not owned by worker") + } + + return nil +} + +func (s *gormStore) GetLease(ctx context.Context, resourceID string) (*store.Lease, error) { + var lease store.Lease + err := s.db.WithContext(ctx). + Where("resource_id = ?", resourceID). + First(&lease).Error + + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrLeaseNotFound + } + + return &lease, err +} + +func (s *gormStore) CleanupExpiredLeases(ctx context.Context, namespace string) error { + now := time.Now().UnixMilli() + + return s.db.WithContext(ctx). + Where("namespace = ? AND expires_at < ?", namespace, now). + Delete(emptyLease).Error +} + +func (s *gormStore) GetExpiredLeases(ctx context.Context, namespace string) ([]store.Lease, error) { + now := time.Now().UnixMilli() + var leases []store.Lease + + err := s.db.WithContext(ctx). + Where("namespace = ? AND expires_at < ?", namespace, now). + Find(&leases).Error + + return leases, err +} + +func (s *gormStore) ResetOrphanedWorkflows(ctx context.Context, namespace string) error { + return s.db.WithContext(ctx).Exec(` + UPDATE workflow_executions + SET status = 'pending' + WHERE namespace = ? + AND status = 'running' + AND id NOT IN ( + SELECT resource_id + FROM leases + WHERE kind = 'workflow' AND namespace = ? + ) + `, namespace, namespace).Error +} + +func (s *gormStore) UpsertCronJob(ctx context.Context, cronJob *store.CronJob) error { + var existing store.CronJob + err := s.db.WithContext(ctx). + Where("namespace = ? AND name = ?", cronJob.Namespace, cronJob.Name). + First(&existing).Error + + if err == nil { + cronJob.ID = existing.ID + cronJob.CreatedAt = existing.CreatedAt + cronJob.UpdatedAt = time.Now().UnixMilli() + return s.db.WithContext(ctx).Save(cronJob).Error + } else if errors.Is(err, gorm.ErrRecordNotFound) { + return s.db.WithContext(ctx).Create(cronJob).Error + } + + return err +} + +func (s *gormStore) GetCronJob(ctx context.Context, namespace, name string) (*store.CronJob, error) { + var cronJob store.CronJob + err := s.db.WithContext(ctx). + Where("namespace = ? AND name = ?", namespace, name). + First(&cronJob).Error + + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrCronJobNotFound + } + + return &cronJob, err +} + +func (s *gormStore) GetCronJobs(ctx context.Context, namespace string) ([]store.CronJob, error) { + var cronJobs []store.CronJob + err := s.db.WithContext(ctx). + Where("namespace = ? AND enabled = ?", namespace, true). + Find(&cronJobs).Error + + return cronJobs, err +} + +func (s *gormStore) GetDueCronJobs(ctx context.Context, namespace string, beforeTime int64) ([]store.CronJob, error) { + var cronJobs []store.CronJob + err := s.db.WithContext(ctx). + Where("namespace = ? AND enabled = ? AND next_run_at <= ?", namespace, true, beforeTime). + Find(&cronJobs).Error + + return cronJobs, err +} + +func (s *gormStore) UpdateCronJobLastRun(ctx context.Context, namespace, cronJobID string, lastRunAt, nextRunAt int64) error { + return s.db.WithContext(ctx). + Model(emptyCronJob). + Where("id = ? AND namespace = ?", cronJobID, namespace). + Updates(map[string]any{ + "last_run_at": lastRunAt, + "next_run_at": nextRunAt, + "updated_at": time.Now().UnixMilli(), + }).Error +} + +func (s *gormStore) WithTx(ctx context.Context, fn func(store.Store) error) error { + return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + txStore := &gormStore{db: tx, clock: s.clock} + return fn(txStore) + }) +} + +func isDuplicateKeyError(err error) bool { + errStr := err.Error() + return strings.Contains(errStr, "duplicate") || + strings.Contains(errStr, "UNIQUE constraint") || + strings.Contains(errStr, "PRIMARY KEY constraint") +} + +// Testing helpers +func (s *gormStore) GetAllWorkflows(ctx context.Context, namespace string) ([]store.WorkflowExecution, error) { + var workflows []store.WorkflowExecution + err := s.db.WithContext(ctx). + Where("namespace = ?", namespace). + Find(&workflows).Error + return workflows, err +} + +func (s *gormStore) GetAllSteps(ctx context.Context, namespace string) ([]store.WorkflowStep, error) { + var steps []store.WorkflowStep + err := s.db.WithContext(ctx). + Where("namespace = ?", namespace). + Find(&steps).Error + return steps, err +} diff --git a/go/pkg/hydra/store/store.go b/go/pkg/hydra/store/store.go new file mode 100644 index 00000000000..820faf7d8b6 --- /dev/null +++ b/go/pkg/hydra/store/store.go @@ -0,0 +1,70 @@ +package store + +import ( + "context" + "time" +) + +type Store interface { + CreateWorkflow(ctx context.Context, workflow *WorkflowExecution) error + + GetWorkflow(ctx context.Context, namespace, id string) (*WorkflowExecution, error) + + GetPendingWorkflows(ctx context.Context, namespace string, limit int, workflowNames []string) ([]WorkflowExecution, error) + + GetPendingWorkflowsWithOffset(ctx context.Context, namespace string, limit int, offset int, workflowNames []string) ([]WorkflowExecution, error) + + AcquireWorkflowLease(ctx context.Context, workflowID, namespace, workerID string, leaseDuration time.Duration) error + + UpdateWorkflowStatus(ctx context.Context, namespace, id string, status WorkflowStatus, errorMsg string) error + + CompleteWorkflow(ctx context.Context, namespace, id string, outputData []byte) error + + FailWorkflow(ctx context.Context, namespace, id string, errorMsg string, isFinal bool) error + + SleepWorkflow(ctx context.Context, namespace, id string, sleepUntil int64) error + + GetSleepingWorkflows(ctx context.Context, namespace string, beforeTime int64) ([]WorkflowExecution, error) + + CreateStep(ctx context.Context, step *WorkflowStep) error + + GetStep(ctx context.Context, namespace, executionID, stepName string) (*WorkflowStep, error) + + GetCompletedStep(ctx context.Context, namespace, executionID, stepName string) (*WorkflowStep, error) + + UpdateStepStatus(ctx context.Context, namespace, executionID, stepName string, status StepStatus, outputData []byte, errorMsg string) error + + UpsertCronJob(ctx context.Context, cronJob *CronJob) error + + GetCronJob(ctx context.Context, namespace, name string) (*CronJob, error) + + GetCronJobs(ctx context.Context, namespace string) ([]CronJob, error) + + GetDueCronJobs(ctx context.Context, namespace string, beforeTime int64) ([]CronJob, error) + + UpdateCronJobLastRun(ctx context.Context, namespace, cronJobID string, lastRunAt, nextRunAt int64) error + + AcquireLease(ctx context.Context, lease *Lease) error + + HeartbeatLease(ctx context.Context, resourceID, workerID string, expiresAt int64) error + + ReleaseLease(ctx context.Context, resourceID, workerID string) error + + GetLease(ctx context.Context, resourceID string) (*Lease, error) + + CleanupExpiredLeases(ctx context.Context, namespace string) error + + GetExpiredLeases(ctx context.Context, namespace string) ([]Lease, error) + + ResetOrphanedWorkflows(ctx context.Context, namespace string) error + + WithTx(ctx context.Context, fn func(Store) error) error + + // Testing helpers + GetAllWorkflows(ctx context.Context, namespace string) ([]WorkflowExecution, error) + GetAllSteps(ctx context.Context, namespace string) ([]WorkflowStep, error) +} + +type StoreFactory interface { + NewStore() Store +} diff --git a/go/pkg/hydra/store/types.go b/go/pkg/hydra/store/types.go new file mode 100644 index 00000000000..a7bca8779ca --- /dev/null +++ b/go/pkg/hydra/store/types.go @@ -0,0 +1,175 @@ +package store + +type WorkflowExecution struct { + ID string `gorm:"primaryKey"` + WorkflowName string `gorm:"index:idx_workflow_namespace_name"` + Status WorkflowStatus `gorm:"index:idx_workflow_namespace_status;index:idx_workflow_status_retry;index:idx_workflow_status_sleep"` + InputData []byte + OutputData []byte + ErrorMessage string + + CreatedAt int64 `gorm:"index:idx_workflow_namespace_status"` + StartedAt *int64 + CompletedAt *int64 + MaxAttempts int32 + RemainingAttempts int32 + NextRetryAt *int64 `gorm:"index:idx_workflow_status_retry"` + + Namespace string `gorm:"index:idx_workflow_namespace_status;index:idx_workflow_namespace_name;index:idx_workflow_status_retry;index:idx_workflow_status_sleep"` + + TriggerType TriggerType + TriggerSource *string + + SleepUntil *int64 `gorm:"index:idx_workflow_status_sleep"` + + TraceID string +} + +func (WorkflowExecution) TableName() string { + return "workflow_executions" +} + +type WorkflowStep struct { + ID string `gorm:"primaryKey"` + ExecutionID string `gorm:"index:idx_workflow_step_unique,unique;index:idx_step_execution_status"` + StepName string `gorm:"index:idx_workflow_step_unique,unique"` + StepOrder int32 + Status StepStatus `gorm:"index:idx_step_execution_status"` + OutputData []byte + ErrorMessage string + + StartedAt *int64 + CompletedAt *int64 + + MaxAttempts int32 + + RemainingAttempts int32 + + Namespace string `gorm:"index:idx_workflow_step_unique,unique;index:idx_step_execution_status"` +} + +func (WorkflowStep) TableName() string { + return "workflow_steps" +} + +type CronJob struct { + ID string `gorm:"primaryKey"` + + Name string + + CronSpec string + + Namespace string `gorm:"index:idx_cron_namespace_enabled_next"` + + WorkflowName string + + Enabled bool `gorm:"index:idx_cron_namespace_enabled_next"` + + CreatedAt int64 + + UpdatedAt int64 + + LastRunAt *int64 + + NextRunAt int64 `gorm:"index:idx_cron_namespace_enabled_next"` +} + +func (CronJob) TableName() string { + return "cron_jobs" +} + +type Lease struct { + ResourceID string `gorm:"primaryKey"` + + Kind string `gorm:"index:idx_lease_resource_kind"` + + Namespace string `gorm:"index:idx_lease_namespace_expires"` + + WorkerID string + + AcquiredAt int64 + + ExpiresAt int64 `gorm:"index:idx_lease_namespace_expires"` + + HeartbeatAt int64 +} + +func (Lease) TableName() string { + return "leases" +} + +type WorkflowStatus string + +const ( + WorkflowStatusPending WorkflowStatus = "pending" + WorkflowStatusRunning WorkflowStatus = "running" + WorkflowStatusSleeping WorkflowStatus = "sleeping" + WorkflowStatusCompleted WorkflowStatus = "completed" + WorkflowStatusFailed WorkflowStatus = "failed" +) + +// IsValid validates if the WorkflowStatus is one of the defined constants +func (ws WorkflowStatus) IsValid() bool { + switch ws { + case WorkflowStatusPending, WorkflowStatusRunning, WorkflowStatusSleeping, WorkflowStatusCompleted, WorkflowStatusFailed: + return true + default: + return false + } +} + +type StepStatus string + +const ( + StepStatusPending StepStatus = "pending" + StepStatusRunning StepStatus = "running" + StepStatusCompleted StepStatus = "completed" + StepStatusFailed StepStatus = "failed" +) + +// IsValid validates if the StepStatus is one of the defined constants +func (ss StepStatus) IsValid() bool { + switch ss { + case StepStatusPending, StepStatusRunning, StepStatusCompleted, StepStatusFailed: + return true + default: + return false + } +} + +type LeaseKind string + +const ( + LeaseKindWorkflow LeaseKind = "workflow" + LeaseKindStep LeaseKind = "step" + LeaseKindCronJob LeaseKind = "cron_job" +) + +// IsValid validates if the LeaseKind is one of the defined constants +func (lk LeaseKind) IsValid() bool { + switch lk { + case LeaseKindWorkflow, LeaseKindStep, LeaseKindCronJob: + return true + default: + return false + } +} + +type TriggerType string + +const ( + TriggerTypeManual TriggerType = "manual" + TriggerTypeCron TriggerType = "cron" + TriggerTypeEvent TriggerType = "event" + TriggerTypeAPI TriggerType = "api" +) + +// IsValid validates if the TriggerType is one of the defined constants +func (tt TriggerType) IsValid() bool { + switch tt { + case TriggerTypeManual, TriggerTypeCron, TriggerTypeEvent, TriggerTypeAPI: + return true + default: + return false + } +} diff --git a/go/pkg/hydra/store_test.go b/go/pkg/hydra/store_test.go new file mode 100644 index 00000000000..e56fe9134d0 --- /dev/null +++ b/go/pkg/hydra/store_test.go @@ -0,0 +1,257 @@ +package hydra + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/unkeyed/unkey/go/pkg/hydra/store" + "github.com/unkeyed/unkey/go/pkg/hydra/store/gorm" +) + +// Store unit tests + +func newTestStore(t *testing.T) store.Store { + s, err := gorm.NewSQLiteStore("", nil) + require.NoError(t, err) + return s +} + +func TestGetPendingWorkflows(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + + // Given: A pending workflow in the database + workflow := &WorkflowExecution{ + ID: "wf_test123", + WorkflowName: "test-workflow", + Status: WorkflowStatusPending, + Namespace: "default", + CreatedAt: time.Now().UnixMilli(), + InputData: []byte(`{}`), + MaxAttempts: 3, + RemainingAttempts: 3, + TriggerType: TriggerTypeManual, + } + + err := store.CreateWorkflow(ctx, workflow) + require.NoError(t, err) + + // When: Getting pending workflows + pending, err := store.GetPendingWorkflows(ctx, "default", 10, []string{"test-workflow"}) + require.NoError(t, err) + + // Then: Should find the pending workflow + require.Len(t, pending, 1, "Should find 1 pending workflow") + require.Equal(t, "wf_test123", pending[0].ID) + require.Equal(t, "test-workflow", pending[0].WorkflowName) + require.Equal(t, WorkflowStatusPending, pending[0].Status) +} + +func TestGetPendingWorkflows_FiltersByWorkflowName(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + + // Given: Two pending workflows with different names + workflow1 := &WorkflowExecution{ + ID: "wf_test1", + WorkflowName: "workflow-a", + Status: WorkflowStatusPending, + Namespace: "default", + CreatedAt: time.Now().UnixMilli(), + InputData: []byte(`{}`), + MaxAttempts: 3, + RemainingAttempts: 3, + TriggerType: TriggerTypeManual, + } + + workflow2 := &WorkflowExecution{ + ID: "wf_test2", + WorkflowName: "workflow-b", + Status: WorkflowStatusPending, + Namespace: "default", + CreatedAt: time.Now().UnixMilli(), + InputData: []byte(`{}`), + MaxAttempts: 3, + RemainingAttempts: 3, + TriggerType: TriggerTypeManual, + } + + require.NoError(t, store.CreateWorkflow(ctx, workflow1)) + require.NoError(t, store.CreateWorkflow(ctx, workflow2)) + + // When: Getting pending workflows for only "workflow-a" + pending, err := store.GetPendingWorkflows(ctx, "default", 10, []string{"workflow-a"}) + require.NoError(t, err) + + // Then: Should only find workflow-a + require.Len(t, pending, 1, "Should find only 1 workflow") + require.Equal(t, "workflow-a", pending[0].WorkflowName) +} + +func TestGetPendingWorkflows_ExcludesNonPendingWorkflows(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + + // Given: Workflows in different states + pendingWorkflow := &WorkflowExecution{ + ID: "wf_pending", + WorkflowName: "test-workflow", + Status: WorkflowStatusPending, + Namespace: "default", + CreatedAt: time.Now().UnixMilli(), + InputData: []byte(`{}`), + MaxAttempts: 3, + RemainingAttempts: 3, + TriggerType: TriggerTypeManual, + } + + completedWorkflow := &WorkflowExecution{ + ID: "wf_completed", + WorkflowName: "test-workflow", + Status: WorkflowStatusCompleted, + Namespace: "default", + CreatedAt: time.Now().UnixMilli(), + InputData: []byte(`{}`), + MaxAttempts: 3, + RemainingAttempts: 3, + TriggerType: TriggerTypeManual, + } + + require.NoError(t, store.CreateWorkflow(ctx, pendingWorkflow)) + require.NoError(t, store.CreateWorkflow(ctx, completedWorkflow)) + + // When: Getting pending workflows + pending, err := store.GetPendingWorkflows(ctx, "default", 10, []string{"test-workflow"}) + require.NoError(t, err) + + // Then: Should only find the pending workflow + require.Len(t, pending, 1, "Should find only pending workflow") + require.Equal(t, "wf_pending", pending[0].ID) +} + +func TestAcquireWorkflowLease_NewLease(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + + // Given: A pending workflow and no existing lease + workflowID := "wf_test123" + workerID := "worker-123" + leaseDuration := 5 * time.Minute + + // Create a pending workflow first + workflow := &WorkflowExecution{ + ID: workflowID, + WorkflowName: "test-workflow", + Status: WorkflowStatusPending, + Namespace: "default", + CreatedAt: time.Now().UnixMilli(), + InputData: []byte(`{}`), + MaxAttempts: 3, + RemainingAttempts: 3, + TriggerType: TriggerTypeManual, + } + err := store.CreateWorkflow(ctx, workflow) + require.NoError(t, err) + + // When: Acquiring lease + err = store.AcquireWorkflowLease(ctx, workflowID, "default", workerID, leaseDuration) + + // Then: Should succeed + require.NoError(t, err) + + // And: Lease should be created + lease, err := store.GetLease(ctx, workflowID) + require.NoError(t, err) + require.Equal(t, workflowID, lease.ResourceID) + require.Equal(t, "workflow", lease.Kind) + require.Equal(t, workerID, lease.WorkerID) + require.Equal(t, "default", lease.Namespace) +} + +func TestAcquireWorkflowLease_ExistingExpiredLease(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + + // Given: A pending workflow and an expired lease exists + workflowID := "wf_test123" + oldWorkerID := "worker-old" + newWorkerID := "worker-new" + + // Create a pending workflow first + workflow := &WorkflowExecution{ + ID: workflowID, + WorkflowName: "test-workflow", + Status: WorkflowStatusPending, + Namespace: "default", + CreatedAt: time.Now().UnixMilli(), + InputData: []byte(`{}`), + MaxAttempts: 3, + RemainingAttempts: 3, + TriggerType: TriggerTypeManual, + } + err := store.CreateWorkflow(ctx, workflow) + require.NoError(t, err) + + // Create expired lease + expiredLease := &Lease{ + ResourceID: workflowID, + Kind: "workflow", + Namespace: "default", + WorkerID: oldWorkerID, + AcquiredAt: time.Now().Add(-10 * time.Minute).UnixMilli(), + ExpiresAt: time.Now().Add(-5 * time.Minute).UnixMilli(), // Expired + HeartbeatAt: time.Now().Add(-10 * time.Minute).UnixMilli(), + } + + err = store.AcquireLease(ctx, expiredLease) + require.NoError(t, err) + + // When: New worker tries to acquire lease + err = store.AcquireWorkflowLease(ctx, workflowID, "default", newWorkerID, 5*time.Minute) + + // Then: Should succeed (take over expired lease) + require.NoError(t, err) + + // And: Lease should be owned by new worker + lease, err := store.GetLease(ctx, workflowID) + require.NoError(t, err) + require.Equal(t, newWorkerID, lease.WorkerID) +} + +func TestAcquireWorkflowLease_ExistingActiveLease(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + + // Given: A pending workflow exists + workflowID := "wf_test123" + workerID1 := "worker-1" + workerID2 := "worker-2" + + // Create a pending workflow first + workflow := &WorkflowExecution{ + ID: workflowID, + WorkflowName: "test-workflow", + Status: WorkflowStatusPending, + Namespace: "default", + CreatedAt: time.Now().UnixMilli(), + InputData: []byte(`{}`), + MaxAttempts: 3, + RemainingAttempts: 3, + TriggerType: TriggerTypeManual, + } + err := store.CreateWorkflow(ctx, workflow) + require.NoError(t, err) + + // Worker 1 acquires lease + err = store.AcquireWorkflowLease(ctx, workflowID, "default", workerID1, 5*time.Minute) + require.NoError(t, err) + + // When: Worker 2 tries to acquire same lease + err = store.AcquireWorkflowLease(ctx, workflowID, "default", workerID2, 5*time.Minute) + + // Then: Should fail + require.Error(t, err) + require.Contains(t, err.Error(), "not available for acquisition") +} diff --git a/go/pkg/hydra/test_helpers.go b/go/pkg/hydra/test_helpers.go new file mode 100644 index 00000000000..11361c54f52 --- /dev/null +++ b/go/pkg/hydra/test_helpers.go @@ -0,0 +1,106 @@ +package hydra + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/unkeyed/unkey/go/pkg/clock" + "github.com/unkeyed/unkey/go/pkg/hydra/store" + "github.com/unkeyed/unkey/go/pkg/hydra/store/gorm" + "github.com/unkeyed/unkey/go/pkg/testutil/containers" + "github.com/unkeyed/unkey/go/pkg/uid" + "gorm.io/driver/mysql" + gormlib "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +// Test helper functions + +// newTestEngine creates an engine instance for testing with MySQL +func newTestEngine(t *testing.T) *Engine { + return newTestEngineWithClock(t, clock.New()) +} + +// newTestEngineWithClock creates an engine instance with a controllable test clock and MySQL +func newTestEngineWithClock(t *testing.T, clk clock.Clock) *Engine { + t.Helper() + + // Use MySQL container for testing + containersClient := containers.New(t) + hostDsn, _ := containersClient.RunMySQL() + + // Open MySQL database with GORM + db, err := gormlib.Open(mysql.Open(hostDsn), &gormlib.Config{ + Logger: logger.Discard, + }) + require.NoError(t, err) + + // Auto-migrate the hydra schema + err = db.AutoMigrate( + &store.WorkflowExecution{ + ID: "", WorkflowName: "", Status: "", InputData: nil, OutputData: nil, + ErrorMessage: "", CreatedAt: 0, StartedAt: nil, CompletedAt: nil, + MaxAttempts: 0, RemainingAttempts: 0, NextRetryAt: nil, Namespace: "", + TriggerType: "", TriggerSource: nil, SleepUntil: nil, TraceID: "", + }, + &store.WorkflowStep{ + ID: "", ExecutionID: "", StepName: "", StepOrder: 0, Status: "", + OutputData: nil, ErrorMessage: "", StartedAt: nil, CompletedAt: nil, + MaxAttempts: 0, RemainingAttempts: 0, Namespace: "", + }, + &store.CronJob{ + ID: "", Name: "", CronSpec: "", Namespace: "", WorkflowName: "", + Enabled: false, CreatedAt: 0, UpdatedAt: 0, LastRunAt: nil, NextRunAt: 0, + }, + &store.Lease{ + ResourceID: "", Kind: "", Namespace: "", WorkerID: "", + AcquiredAt: 0, ExpiresAt: 0, HeartbeatAt: 0, + }, + ) + require.NoError(t, err) + + // Create the store + gormStore := gorm.NewGORMStore(db, clk) + + // Create engine with unique namespace for test isolation + testNamespace := uid.New(uid.TestPrefix) + + engine := New(Config{ + Store: gormStore, + Namespace: testNamespace, + Clock: clk, + Logger: nil, + Marshaller: nil, + }) + + return engine +} + +// waitForWorkflowCompletion waits until a workflow reaches a terminal state (completed or failed) +func waitForWorkflowCompletion(t *testing.T, e *Engine, executionID string, timeout time.Duration) *store.WorkflowExecution { + t.Helper() + + var workflow *store.WorkflowExecution + + require.Eventually(t, func() bool { + var err error + workflow, err = e.GetStore().GetWorkflow(context.Background(), e.GetNamespace(), executionID) + if err != nil { + t.Logf("Error getting workflow %s: %v", executionID, err) + return false + } + + isComplete := workflow.Status == store.WorkflowStatusCompleted || + workflow.Status == store.WorkflowStatusFailed + + if !isComplete { + t.Logf("Workflow %s status: %s", executionID, workflow.Status) + } + + return isComplete + }, timeout, 100*time.Millisecond, "Workflow should complete within timeout") + + return workflow +} diff --git a/go/pkg/hydra/testharness/events.go b/go/pkg/hydra/testharness/events.go new file mode 100644 index 00000000000..922286d6af1 --- /dev/null +++ b/go/pkg/hydra/testharness/events.go @@ -0,0 +1,179 @@ +package testharness + +import ( + "sync" + "time" +) + +// WorkflowContext interface for extracting metadata (avoid import cycle) +type WorkflowContext interface { + ExecutionID() string + WorkflowName() string +} + +// EventType represents the type of event that occurred +type EventType string + +const ( + WorkflowStarted EventType = "workflow_started" + WorkflowCompleted EventType = "workflow_completed" + WorkflowFailed EventType = "workflow_failed" + StepExecuting EventType = "step_executing" + StepExecuted EventType = "step_executed" + StepFailed EventType = "step_failed" +) + +// EventRecord represents something that happened during test execution +type EventRecord struct { + Type EventType `json:"type"` + Message string `json:"message"` + Timestamp time.Time `json:"timestamp"` + Data map[string]interface{} `json:"data"` +} + +// EventCollector captures events during test execution +type EventCollector struct { + mu sync.RWMutex + events []EventRecord +} + +// NewEventCollector creates a new event collector +func NewEventCollector() *EventCollector { + return &EventCollector{ + mu: sync.RWMutex{}, + events: make([]EventRecord, 0), + } +} + +// Emit records an event with workflow context metadata automatically included +func (e *EventCollector) Emit(ctx WorkflowContext, eventType EventType, message string, extraData ...interface{}) { + e.mu.Lock() + defer e.mu.Unlock() + + // Start with context metadata + data := map[string]interface{}{ + "execution_id": ctx.ExecutionID(), + "workflow_name": ctx.WorkflowName(), + } + + // Add extra data as key-value pairs + for i := 0; i < len(extraData); i += 2 { + if i+1 < len(extraData) { + if key, ok := extraData[i].(string); ok { + data[key] = extraData[i+1] + } + } + } + + event := EventRecord{ + Type: eventType, + Message: message, + Timestamp: time.Now(), + Data: data, + } + + e.events = append(e.events, event) +} + +// Events returns all collected events +func (e *EventCollector) Events() []EventRecord { + e.mu.RLock() + defer e.mu.RUnlock() + + // Return a copy to prevent race conditions + events := make([]EventRecord, len(e.events)) + copy(events, e.events) + return events +} + +// Filter returns events that match the given criteria +func (e *EventCollector) Filter(eventType EventType) []EventRecord { + e.mu.RLock() + defer e.mu.RUnlock() + + var filtered []EventRecord + for _, event := range e.events { + if event.Type == eventType { + filtered = append(filtered, event) + } + } + return filtered +} + +// FilterWithData returns events that match the type and have specific data values +func (e *EventCollector) FilterWithData(eventType EventType, key string, value interface{}) []EventRecord { + e.mu.RLock() + defer e.mu.RUnlock() + + var filtered []EventRecord + for _, event := range e.events { + if event.Type == eventType { + if eventValue, exists := event.Data[key]; exists && eventValue == value { + filtered = append(filtered, event) + } + } + } + return filtered +} + +// Count returns the number of events of a specific type +func (e *EventCollector) Count(eventType EventType) int { + return len(e.Filter(eventType)) +} + +// CountWithData returns the number of events that match type and data criteria +func (e *EventCollector) CountWithData(eventType EventType, key string, value interface{}) int { + return len(e.FilterWithData(eventType, key, value)) +} + +// Clear removes all collected events +func (e *EventCollector) Clear() { + e.mu.Lock() + defer e.mu.Unlock() + e.events = e.events[:0] +} + +// GetLatest returns the most recent event of a given type, or nil if none found +func (e *EventCollector) GetLatest(eventType EventType) *EventRecord { + events := e.Filter(eventType) + if len(events) == 0 { + return nil + } + return &events[len(events)-1] +} + +// GetFirst returns the first event of a given type, or nil if none found +func (e *EventCollector) GetFirst(eventType EventType) *EventRecord { + events := e.Filter(eventType) + if len(events) == 0 { + return nil + } + return &events[0] +} + +// EventsBetween returns events that occurred between start and end times (inclusive) +func (e *EventCollector) EventsBetween(start, end time.Time) []EventRecord { + e.mu.RLock() + defer e.mu.RUnlock() + + var filtered []EventRecord + for _, event := range e.events { + if (event.Timestamp.Equal(start) || event.Timestamp.After(start)) && + (event.Timestamp.Equal(end) || event.Timestamp.Before(end)) { + filtered = append(filtered, event) + } + } + return filtered +} + +// Summary returns a summary of all event types and their counts +func (e *EventCollector) Summary() map[string]int { + e.mu.RLock() + defer e.mu.RUnlock() + + summary := make(map[string]int) + for _, event := range e.events { + summary[string(event.Type)]++ + } + return summary +} diff --git a/go/pkg/hydra/worker.go b/go/pkg/hydra/worker.go new file mode 100644 index 00000000000..b0c05e2e327 --- /dev/null +++ b/go/pkg/hydra/worker.go @@ -0,0 +1,615 @@ +package hydra + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/unkeyed/unkey/go/pkg/circuitbreaker" + "github.com/unkeyed/unkey/go/pkg/clock" + "github.com/unkeyed/unkey/go/pkg/hydra/metrics" + "github.com/unkeyed/unkey/go/pkg/hydra/store" + "github.com/unkeyed/unkey/go/pkg/uid" +) + +// Worker represents a workflow worker that can start, run, and shutdown. +// +// Workers are responsible for: +// - Polling the database for pending workflows +// - Acquiring exclusive leases on workflows to prevent duplicate execution +// - Executing workflow logic by calling registered workflow handlers +// - Sending periodic heartbeats to maintain lease ownership +// - Processing scheduled cron jobs +// - Recording metrics for observability +// +// Workers are designed to be run as long-lived processes and can safely +// handle network failures, database outages, and graceful shutdowns. +type Worker interface { + // Start begins the worker's main execution loop. + // This method blocks until the context is cancelled or an error occurs. + Start(ctx context.Context) error + + // Shutdown gracefully stops the worker and waits for active workflows to complete. + // This method should be called during application shutdown to ensure clean termination. + Shutdown(ctx context.Context) error +} + +// WorkerConfig holds the configuration for a worker instance. +// +// All fields are optional and will use sensible defaults if not specified. +type WorkerConfig struct { + // WorkerID uniquely identifies this worker instance. + // If not provided, a random ID will be generated. + WorkerID string + + // Concurrency controls how many workflows can execute simultaneously. + // Defaults to 10 if not specified. + Concurrency int + + // PollInterval controls how frequently the worker checks for new workflows. + // Shorter intervals provide lower latency but increase database load. + // Defaults to 5 seconds if not specified. + PollInterval time.Duration + + // HeartbeatInterval controls how frequently the worker sends lease heartbeats. + // This should be significantly shorter than ClaimTimeout to prevent lease expiration. + // Defaults to 30 seconds if not specified. + HeartbeatInterval time.Duration + + // ClaimTimeout controls how long a worker can hold a workflow lease. + // Expired leases are automatically released, allowing other workers to take over. + // Defaults to 5 minutes if not specified. + ClaimTimeout time.Duration + + // CronInterval controls how frequently the worker checks for due cron jobs. + // Defaults to 1 minute if not specified. + CronInterval time.Duration +} + +type worker struct { + engine *Engine + config WorkerConfig + workflows map[string]Workflow[any] + clock clock.Clock + shutdownC chan struct{} + doneC chan struct{} + wg sync.WaitGroup + activeLeases map[string]bool // Track workflow IDs we have leases for + activeLeasesM sync.RWMutex // Protect the activeLeases map + queryCircuitBreaker circuitbreaker.CircuitBreaker[[]store.WorkflowExecution] // Protect query operations + leaseCircuitBreaker circuitbreaker.CircuitBreaker[any] // Protect lease operations + workflowQueue chan store.WorkflowExecution // Queue of workflows to process +} + +// NewWorker creates a new worker instance with the provided configuration. +// +// The worker will be associated with the given engine and inherit its +// namespace and storage configuration. Missing configuration values +// will be populated with sensible defaults. +// +// The worker must have workflows registered using RegisterWorkflow() +// before calling Start(). +// +// Example: +// +// worker, err := hydra.NewWorker(engine, hydra.WorkerConfig{ +// WorkerID: "worker-1", +// Concurrency: 20, +// PollInterval: 100 * time.Millisecond, +// HeartbeatInterval: 30 * time.Second, +// ClaimTimeout: 5 * time.Minute, +// }) +// if err != nil { +// return err +// } +// +// The worker includes built-in circuit breakers to protect against +// database overload and automatic retry logic for transient failures. +func NewWorker(e *Engine, config WorkerConfig) (Worker, error) { + if config.WorkerID == "" { + config.WorkerID = uid.New(uid.WorkerPrefix) + } + if config.Concurrency <= 0 { + config.Concurrency = 10 + } + if config.PollInterval <= 0 { + config.PollInterval = 5 * time.Second + } + if config.HeartbeatInterval <= 0 { + config.HeartbeatInterval = 30 * time.Second + } + if config.ClaimTimeout <= 0 { + config.ClaimTimeout = 5 * time.Minute + } + if config.CronInterval <= 0 { + config.CronInterval = 1 * time.Minute + } + + // Initialize circuit breakers for different database operations + queryCircuitBreaker := circuitbreaker.New[[]store.WorkflowExecution]("hydra-query") + leaseCircuitBreaker := circuitbreaker.New[any]("hydra-lease") + + // Create workflow queue with capacity based on concurrency + queueSize := config.Concurrency * 10 + if queueSize < 50 { + queueSize = 50 // Minimum queue size + } + + worker := &worker{ + engine: e, + config: config, + workflows: make(map[string]Workflow[any]), + clock: e.clock, + shutdownC: make(chan struct{}), + doneC: make(chan struct{}), + wg: sync.WaitGroup{}, + activeLeases: make(map[string]bool), + activeLeasesM: sync.RWMutex{}, + queryCircuitBreaker: queryCircuitBreaker, + leaseCircuitBreaker: leaseCircuitBreaker, + workflowQueue: make(chan store.WorkflowExecution, queueSize), + } + + return worker, nil +} + +func (w *worker) run(ctx context.Context) { + defer close(w.doneC) + + // Start workflow processors + for i := 0; i < w.config.Concurrency; i++ { + w.wg.Add(1) + go w.processWorkflows(ctx) + } + + w.wg.Add(4) + go w.pollForWorkflows(ctx) + go w.sendHeartbeats(ctx) + go w.cleanupExpiredLeases(ctx) + go w.processCronJobs(ctx) + + select { + case <-w.shutdownC: + case <-ctx.Done(): + } + + // Don't close the queue immediately - let processors drain it first + w.wg.Wait() +} + +func (w *worker) pollForWorkflows(ctx context.Context) { + defer w.wg.Done() + + ticker := w.clock.NewTicker(w.config.PollInterval) + defer ticker.Stop() + tickerC := ticker.C() + + for { + select { + case <-tickerC: + w.pollOnce(ctx) + + case <-w.shutdownC: + return + + case <-ctx.Done(): + return + } + } +} + +func (w *worker) pollOnce(ctx context.Context) { + workflowNames := make([]string, 0, len(w.workflows)) + for name := range w.workflows { + workflowNames = append(workflowNames, name) + } + + // Use a more conservative fetch limit to reduce contention + fetchLimit := w.config.Concurrency * 2 // Fetch less to reduce contention + if fetchLimit < 10 { + fetchLimit = 10 // Minimum fetch size + } + + workflows, err := w.queryCircuitBreaker.Do(ctx, func(ctx context.Context) ([]store.WorkflowExecution, error) { + return w.engine.store.GetPendingWorkflows(ctx, w.engine.namespace, fetchLimit, workflowNames) + }) + + // Record polling metrics + if err != nil { + metrics.WorkerPollsTotal.WithLabelValues(w.config.WorkerID, w.engine.namespace, "error").Inc() + return + } + + // Record successful poll with found work status + status := "no_work" + if len(workflows) > 0 { + status = "found_work" + } + metrics.WorkerPollsTotal.WithLabelValues(w.config.WorkerID, w.engine.namespace, status).Inc() + + // Queue workflows - let polling goroutine block if needed + for _, workflow := range workflows { + w.workflowQueue <- workflow + } +} + +func (w *worker) processWorkflows(ctx context.Context) { + defer w.wg.Done() + + for { + select { + case workflow := <-w.workflowQueue: + // Try to acquire lease with direct store call (no circuit breaker to avoid blocking) + err := w.engine.store.AcquireWorkflowLease(ctx, workflow.ID, w.engine.namespace, w.config.WorkerID, w.config.ClaimTimeout) + if err != nil { + // Another worker got it or error, skip this workflow + metrics.LeaseAcquisitionsTotal.WithLabelValues(w.config.WorkerID, "workflow", "failed").Inc() + continue + } + + // Record successful lease acquisition + metrics.LeaseAcquisitionsTotal.WithLabelValues(w.config.WorkerID, "workflow", "success").Inc() + + // Track this lease for heartbeats + w.addActiveLease(workflow.ID) + + // Update active workflows gauge + metrics.WorkflowsActive.WithLabelValues(w.engine.namespace, w.config.WorkerID).Inc() + + // Execute the workflow + w.executeWorkflow(ctx, &workflow) + + // Release the lease and stop tracking it + if err := w.engine.store.ReleaseLease(ctx, workflow.ID, w.config.WorkerID); err != nil { + w.engine.logger.Error("Failed to release workflow lease", + "workflow_id", workflow.ID, + "worker_id", w.config.WorkerID, + "error", err.Error(), + ) + } + w.removeActiveLease(workflow.ID) + + // Update active workflows gauge + metrics.WorkflowsActive.WithLabelValues(w.engine.namespace, w.config.WorkerID).Dec() + + case <-w.shutdownC: + return + case <-ctx.Done(): + return + } + } +} + +func (w *worker) executeWorkflow(ctx context.Context, e *WorkflowExecution) { + startTime := w.clock.Now() + + // Calculate queue time (time from creation to execution start) + queueTime := time.Duration(startTime.UnixMilli()-e.CreatedAt) * time.Millisecond + metrics.WorkflowQueueTimeSeconds.WithLabelValues(e.Namespace, e.WorkflowName).Observe(queueTime.Seconds()) + + err := w.engine.store.UpdateWorkflowStatus(ctx, e.Namespace, e.ID, WorkflowStatusRunning, "") + if err != nil { + metrics.RecordError(e.Namespace, "worker", "status_update_failed") + return + } + + wf, exists := w.workflows[e.WorkflowName] + if !exists { + noHandlerErr := fmt.Errorf("no handler registered for workflow %s", e.WorkflowName) + if failErr := w.engine.store.FailWorkflow(ctx, e.Namespace, e.ID, noHandlerErr.Error(), true); failErr != nil { + w.engine.logger.Error("Failed to mark workflow as failed", + "workflow_id", e.ID, + "workflow_name", e.WorkflowName, + "namespace", e.Namespace, + "error", failErr.Error(), + ) + } + metrics.ObserveWorkflowDuration(e.Namespace, e.WorkflowName, "failed", startTime) + metrics.WorkflowsCompletedTotal.WithLabelValues(e.Namespace, e.WorkflowName, "failed").Inc() + metrics.RecordError(e.Namespace, "worker", "no_handler_registered") + return + } + + payload := &RawPayload{Data: e.InputData} + + wctx := &workflowContext{ + ctx: ctx, + executionID: e.ID, + workflowName: e.WorkflowName, + namespace: e.Namespace, + workerID: w.config.WorkerID, + store: w.engine.store, + marshaller: w.engine.marshaller, + stepTimeout: 5 * time.Minute, // Default step timeout + stepMaxAttempts: 3, // Default step max attempts + stepOrder: 0, + } + + err = wf.Run(wctx, payload) + + if err != nil { + if suspendErr, ok := err.(*WorkflowSuspendedError); ok { + if sleepErr := w.engine.store.SleepWorkflow(ctx, e.Namespace, e.ID, suspendErr.ResumeTime); sleepErr != nil { + w.engine.logger.Error("Failed to suspend workflow", + "workflow_id", e.ID, + "workflow_name", e.WorkflowName, + "namespace", e.Namespace, + "resume_time", suspendErr.ResumeTime, + "error", sleepErr.Error(), + ) + } + metrics.SleepsStartedTotal.WithLabelValues(e.Namespace, e.WorkflowName).Inc() + return + } + + isFinal := e.RemainingAttempts <= 1 + if failErr := w.engine.store.FailWorkflow(ctx, e.Namespace, e.ID, err.Error(), isFinal); failErr != nil { + w.engine.logger.Error("Failed to mark workflow as failed", + "workflow_id", e.ID, + "workflow_name", e.WorkflowName, + "namespace", e.Namespace, + "is_final", isFinal, + "original_error", err.Error(), + "fail_error", failErr.Error(), + ) + } + + if !isFinal { + metrics.WorkflowsRetriedTotal.WithLabelValues(e.Namespace, e.WorkflowName, fmt.Sprintf("%d", e.MaxAttempts-e.RemainingAttempts+1)).Inc() + } + + metrics.ObserveWorkflowDuration(e.Namespace, e.WorkflowName, "failed", startTime) + metrics.WorkflowsCompletedTotal.WithLabelValues(e.Namespace, e.WorkflowName, "failed").Inc() + return + } + + if err := w.engine.store.CompleteWorkflow(ctx, e.Namespace, e.ID, nil); err != nil { // No output data for now + w.engine.logger.Error("Failed to mark workflow as completed", + "workflow_id", e.ID, + "workflow_name", e.WorkflowName, + "namespace", e.Namespace, + "error", err.Error(), + ) + } + metrics.ObserveWorkflowDuration(e.Namespace, e.WorkflowName, "completed", startTime) + metrics.WorkflowsCompletedTotal.WithLabelValues(e.Namespace, e.WorkflowName, "completed").Inc() +} + +func (w *worker) sendHeartbeats(ctx context.Context) { + defer w.wg.Done() + + ticker := w.clock.NewTicker(w.config.HeartbeatInterval) + defer ticker.Stop() + tickerC := ticker.C() + + for { + select { + case <-tickerC: + w.sendHeartbeatsForActiveLeases(ctx) + + case <-w.shutdownC: + return + case <-ctx.Done(): + return + } + } +} + +// addActiveLease tracks a workflow lease for heartbeat sending +func (w *worker) addActiveLease(workflowID string) { + w.activeLeasesM.Lock() + defer w.activeLeasesM.Unlock() + w.activeLeases[workflowID] = true +} + +// removeActiveLease stops tracking a workflow lease +func (w *worker) removeActiveLease(workflowID string) { + w.activeLeasesM.Lock() + defer w.activeLeasesM.Unlock() + delete(w.activeLeases, workflowID) +} + +// sendHeartbeatsForActiveLeases sends heartbeats for all workflows this worker has leases for +func (w *worker) sendHeartbeatsForActiveLeases(ctx context.Context) { + w.activeLeasesM.RLock() + // Copy the map to avoid holding the lock while sending heartbeats + leaseIDs := make([]string, 0, len(w.activeLeases)) + for workflowID := range w.activeLeases { + leaseIDs = append(leaseIDs, workflowID) + } + w.activeLeasesM.RUnlock() + + // Send heartbeats for each active lease + now := w.clock.Now().UnixMilli() + newExpiresAt := now + w.config.ClaimTimeout.Milliseconds() + + for _, workflowID := range leaseIDs { + // Protect heartbeat with circuit breaker + _, err := w.leaseCircuitBreaker.Do(ctx, func(ctx context.Context) (any, error) { + return nil, w.engine.store.HeartbeatLease(ctx, workflowID, w.config.WorkerID, newExpiresAt) + }) + if err != nil { + // Record failed heartbeat + metrics.WorkerHeartbeatsTotal.WithLabelValues(w.config.WorkerID, w.engine.namespace, "failed").Inc() + continue + } + + // Record successful heartbeat + metrics.WorkerHeartbeatsTotal.WithLabelValues(w.config.WorkerID, w.engine.namespace, "success").Inc() + } +} + +func (w *worker) cleanupExpiredLeases(ctx context.Context) { + defer w.wg.Done() + + ticker := w.clock.NewTicker(w.config.HeartbeatInterval * 2) // Clean up less frequently than heartbeats + defer ticker.Stop() + tickerC := ticker.C() + + for { + select { + case <-tickerC: + // Clean up expired leases first + err := w.engine.store.CleanupExpiredLeases(ctx, w.engine.namespace) + if err != nil { + w.engine.logger.Warn("Failed to cleanup expired leases", "error", err.Error()) + } + + // Then reset orphaned workflows back to pending so they can be picked up again + err = w.engine.store.ResetOrphanedWorkflows(ctx, w.engine.namespace) + if err != nil { + w.engine.logger.Warn("Failed to reset orphaned workflows", "error", err.Error()) + } + + case <-w.shutdownC: + return + case <-ctx.Done(): + return + } + } +} + +func (w *worker) processCronJobs(ctx context.Context) { + defer w.wg.Done() + + ticker := w.clock.NewTicker(w.config.CronInterval) + defer ticker.Stop() + tickerC := ticker.C() + + for { + select { + case <-tickerC: + w.processDueCronJobs(ctx) + + case <-w.shutdownC: + return + case <-ctx.Done(): + return + } + } +} + +func (w *worker) processDueCronJobs(ctx context.Context) { + + now := w.engine.clock.Now().UnixMilli() + + dueCrons, err := w.engine.store.GetDueCronJobs(ctx, w.engine.namespace, now) + if err != nil { + return + } + + if len(dueCrons) == 0 { + return + } + + for _, cronJob := range dueCrons { + var canHandle bool + if cronJob.WorkflowName != "" { + _, canHandle = w.workflows[cronJob.WorkflowName] + } else { + _, canHandle = w.engine.cronHandlers[cronJob.Name] + } + + if !canHandle { + continue + } + + lease := &Lease{ + ResourceID: cronJob.ID, + Kind: string(LeaseKindCronJob), + Namespace: w.engine.namespace, + WorkerID: w.config.WorkerID, + AcquiredAt: now, + ExpiresAt: now + (5 * time.Minute).Milliseconds(), // 5 minute lease for cron execution + HeartbeatAt: now, + } + + err := w.engine.store.AcquireLease(ctx, lease) + if err != nil { + continue + } + + w.executeCronJob(ctx, cronJob) + + if err := w.engine.store.ReleaseLease(ctx, cronJob.ID, w.config.WorkerID); err != nil { + w.engine.logger.Error("Failed to release cron job lease", + "cron_job_id", cronJob.ID, + "cron_name", cronJob.Name, + "worker_id", w.config.WorkerID, + "error", err.Error(), + ) + } + } +} + +func (w *worker) executeCronJob(ctx context.Context, cronJob CronJob) { + + now := w.engine.clock.Now().UnixMilli() + + payload := &CronPayload{ + CronJobID: cronJob.ID, + CronName: cronJob.Name, + ScheduledAt: cronJob.NextRunAt, + ActualRunAt: now, + Namespace: cronJob.Namespace, + } + + handler, exists := w.engine.cronHandlers[cronJob.Name] + if !exists { + return + } + + // Execute cron handler with panic recovery + func() { + defer func() { + if r := recover(); r != nil { + w.engine.logger.Error("Cron handler panicked", + "cron_job_id", cronJob.ID, + "cron_name", cronJob.Name, + "panic", r, + ) + } + }() + if err := handler(ctx, *payload); err != nil { + w.engine.logger.Error("Cron handler execution failed", + "cron_job_id", cronJob.ID, + "cron_name", cronJob.Name, + "error", err.Error(), + ) + } + }() + + nextRun := calculateNextRun(cronJob.CronSpec, w.engine.clock.Now()) + if err := w.engine.store.UpdateCronJobLastRun(ctx, w.engine.namespace, cronJob.ID, now, nextRun); err != nil { + w.engine.logger.Error("Failed to update cron job last run time", + "cron_job_id", cronJob.ID, + "cron_name", cronJob.Name, + "namespace", w.engine.namespace, + "last_run", now, + "next_run", nextRun, + "error", err.Error(), + ) + } + +} + +func (w *worker) Start(ctx context.Context) error { + go w.run(ctx) + return nil +} + +func (w *worker) Shutdown(ctx context.Context) error { + select { + case <-w.shutdownC: + default: + close(w.shutdownC) + } + + select { + case <-w.doneC: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/go/pkg/hydra/worker_heartbeat_test.go b/go/pkg/hydra/worker_heartbeat_test.go new file mode 100644 index 00000000000..20e45b14604 --- /dev/null +++ b/go/pkg/hydra/worker_heartbeat_test.go @@ -0,0 +1,125 @@ +package hydra + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/unkeyed/unkey/go/pkg/clock" + "github.com/unkeyed/unkey/go/pkg/uid" +) + +// TestWorkerHeartbeatFunctionality ensures that workers send heartbeats to maintain their leases +// and prevent workflows from being incorrectly marked as orphaned when workers are healthy. +func TestWorkerHeartbeatFunctionality(t *testing.T) { + // Arrange: Create engine with test clock for deterministic timing + testClock := clock.NewTestClock() + engine := newTestEngineWithClock(t, testClock) + + // Create a workflow that will run for a while to give us time to test heartbeats + workflow := &longRunningWorkflow{ + engine: engine, + name: "heartbeat-test-workflow", + executeTime: 5 * time.Second, // Run longer than heartbeat interval + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Start workflow + executionID, err := workflow.Start(ctx, struct{}{}) + require.NoError(t, err) + + // Start worker with short heartbeat interval for faster testing + workerID := uid.New(uid.WorkerPrefix) + worker, err := NewWorker(engine, WorkerConfig{ + WorkerID: workerID, + Concurrency: 1, + PollInterval: 100 * time.Millisecond, + HeartbeatInterval: 1 * time.Second, // Send heartbeats frequently + ClaimTimeout: 10 * time.Second, // Long enough for multiple heartbeats + }) + require.NoError(t, err) + + err = RegisterWorkflow(worker, workflow) + require.NoError(t, err) + + err = worker.Start(ctx) + require.NoError(t, err) + defer worker.Shutdown(ctx) + + // Act: Let worker pick up workflow and start sending heartbeats + testClock.Tick(200 * time.Millisecond) // Trigger initial poll + time.Sleep(50 * time.Millisecond) // Let worker pick up the workflow + + // Keep triggering polls until workflow is picked up + require.Eventually(t, func() bool { + testClock.Tick(200 * time.Millisecond) + time.Sleep(10 * time.Millisecond) + + // Check if workflow has been picked up + currentStatus, getErr := engine.store.GetWorkflow(ctx, engine.GetNamespace(), executionID) + if getErr != nil { + return false + } + return currentStatus.Status != WorkflowStatusPending + }, 3*time.Second, 50*time.Millisecond, "Worker should pick up workflow within timeout") + + // Verify workflow is being processed + workflowStatus, err := engine.store.GetWorkflow(ctx, engine.GetNamespace(), executionID) + require.NoError(t, err) + require.Equal(t, WorkflowStatusRunning, workflowStatus.Status, "Workflow should be running") + + // Get initial lease + lease, err := engine.store.GetLease(ctx, executionID) + require.NoError(t, err) + require.Equal(t, workerID, lease.WorkerID, "Lease should be held by our worker") + + initialExpiresAt := lease.ExpiresAt + + // Advance time to trigger first heartbeat + testClock.Tick(1500 * time.Millisecond) // Past first heartbeat interval + time.Sleep(50 * time.Millisecond) // Let heartbeat be processed + + // Verify heartbeat extended the lease + updatedLease, err := engine.store.GetLease(ctx, executionID) + require.NoError(t, err) + require.Equal(t, workerID, updatedLease.WorkerID, "Lease should still be held by our worker") + require.Greater(t, updatedLease.ExpiresAt, initialExpiresAt, + "HEARTBEAT FAILURE: Lease expiration should be extended after heartbeat. "+ + "Initial: %d, Updated: %d. This means the worker is not sending heartbeats properly, "+ + "which could cause healthy workers to lose their leases prematurely.", + initialExpiresAt, updatedLease.ExpiresAt) + require.Greater(t, updatedLease.HeartbeatAt, lease.HeartbeatAt, + "HeartbeatAt timestamp should be updated") + + // The key test: verify heartbeat actually extended the lease + extensionAmount := updatedLease.ExpiresAt - initialExpiresAt + require.Greater(t, extensionAmount, int64(0), + "HEARTBEAT SUCCESS: Lease was extended by %d ms. Heartbeats are working correctly.", extensionAmount) + +} + +// longRunningWorkflow simulates a workflow that takes time to execute, +// giving us opportunity to test heartbeat behavior during execution +type longRunningWorkflow struct { + engine *Engine + name string + executeTime time.Duration +} + +func (w *longRunningWorkflow) Name() string { + return w.name +} + +func (w *longRunningWorkflow) Run(ctx WorkflowContext, req any) error { + // Simulate long-running work by sleeping + // In a real test, this would be actual work that takes time + time.Sleep(w.executeTime) + return nil +} + +func (w *longRunningWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} diff --git a/go/pkg/hydra/worker_polling_test.go b/go/pkg/hydra/worker_polling_test.go new file mode 100644 index 00000000000..1117822682c --- /dev/null +++ b/go/pkg/hydra/worker_polling_test.go @@ -0,0 +1,314 @@ +package hydra + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/unkeyed/unkey/go/pkg/clock" +) + +// TestWorkerPollingEfficiency verifies that workers can handle concurrent load +// without excessive database contention or resource exhaustion +func TestWorkerPollingEfficiency(t *testing.T) { + engine := newTestEngine(t) + + const ( + numWorkers = 10 + numWorkflows = 50 + testDuration = 5 * time.Second + ) + + var completedWorkflows atomic.Int64 + + // Create workflow that tracks completion + pollingWorkflow := &pollingTestWorkflow{ + engine: engine, + name: "polling-test-workflow", + onPoll: func() { + completedWorkflows.Add(1) + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), testDuration) + defer cancel() + + // Start workers + var wg sync.WaitGroup + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + + worker, err := NewWorker(engine, WorkerConfig{ + WorkerID: fmt.Sprintf("polling-worker-%d", workerID), + Concurrency: 5, // Multiple workflows per worker + PollInterval: 100 * time.Millisecond, + HeartbeatInterval: 5 * time.Second, + ClaimTimeout: 30 * time.Second, + }) + require.NoError(t, err) + + err = RegisterWorkflow(worker, pollingWorkflow) + require.NoError(t, err) + + err = worker.Start(ctx) + require.NoError(t, err) + defer worker.Shutdown(ctx) + + <-ctx.Done() + }(i) + } + + // Submit workflows for processing + for i := 0; i < numWorkflows; i++ { + _, err := pollingWorkflow.Start(ctx, fmt.Sprintf("poll-test-%d", i)) + require.NoError(t, err) + } + + // Wait for completion or timeout + require.Eventually(t, func() bool { + return completedWorkflows.Load() >= int64(numWorkflows) + }, testDuration, 100*time.Millisecond, + "Should complete %d workflows within %v", numWorkflows, testDuration) + + wg.Wait() + + // Verify all workflows were processed + finalCompleted := completedWorkflows.Load() + require.GreaterOrEqual(t, finalCompleted, int64(numWorkflows), + "Should have completed at least %d workflows, got %d", numWorkflows, finalCompleted) +} + +// TestWorkerPollingAccuracy tests that workers actually poll at the configured interval +func TestWorkerPollingAccuracy(t *testing.T) { + testClock := clock.NewTestClock() + engine := newTestEngineWithClock(t, testClock) + + const pollInterval = 200 * time.Millisecond + const tolerance = 50 * time.Millisecond // 25% tolerance + + var pollTimes []time.Time + var mu sync.Mutex + + worker, err := NewWorker(engine, WorkerConfig{ + WorkerID: "accuracy-test-worker", + Concurrency: 1, + PollInterval: pollInterval, + HeartbeatInterval: 5 * time.Second, + ClaimTimeout: 30 * time.Second, + }) + require.NoError(t, err) + + pollingWorkflow := &pollingTestWorkflow{ + engine: engine, + name: "accuracy-test-workflow", + onPoll: func() { + mu.Lock() + pollTimes = append(pollTimes, testClock.Now()) + mu.Unlock() + }, + } + + err = RegisterWorkflow(worker, pollingWorkflow) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + err = worker.Start(ctx) + require.NoError(t, err) + defer worker.Shutdown(ctx) + + // Advance clock to trigger multiple polls + for i := 0; i < 10; i++ { + testClock.Tick(pollInterval) + time.Sleep(10 * time.Millisecond) // Allow processing + } + + // Analyze interval accuracy + mu.Lock() + if len(pollTimes) < 2 { + mu.Unlock() + t.Skip("Not enough poll events to analyze intervals") + return + } + + actualIntervals := make([]time.Duration, len(pollTimes)-1) + for i := 1; i < len(pollTimes); i++ { + actualIntervals[i-1] = pollTimes[i].Sub(pollTimes[i-1]) + } + mu.Unlock() + + // Check each interval is within tolerance + accurateIntervals := 0 + for _, interval := range actualIntervals { + diff := interval - pollInterval + if diff < 0 { + diff = -diff + } + + isAccurate := diff <= tolerance + if isAccurate { + accurateIntervals++ + } + + } + + accuracy := float64(accurateIntervals) / float64(len(actualIntervals)) * 100 + + // Performance assertions + require.GreaterOrEqual(t, accuracy, 80.0, + "At least 80%% of polling intervals should be accurate, got %.1f%%", accuracy) + +} + +// TestThunderingHerdPrevention ensures that when many workers start at the same time, +// they don't all poll the database simultaneously causing performance issues +func TestThunderingHerdPrevention(t *testing.T) { + testClock := clock.NewTestClock() + engine := newTestEngineWithClock(t, testClock) + + const ( + numWorkers = 50 // Large number to stress test + pollInterval = 100 * time.Millisecond + ) + + // Track when each worker polls + pollEvents := make(chan time.Time, 1000) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + // Start all workers simultaneously + var wg sync.WaitGroup + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + + worker, err := NewWorker(engine, WorkerConfig{ + WorkerID: fmt.Sprintf("herd-worker-%d", workerID), + Concurrency: 1, + PollInterval: pollInterval, + HeartbeatInterval: 5 * time.Second, + ClaimTimeout: 30 * time.Second, + }) + require.NoError(t, err) + + pollingWorkflow := &pollingTestWorkflow{ + engine: engine, + name: "herd-test-workflow", + onPoll: func() { + select { + case pollEvents <- testClock.Now(): + default: + // Channel full, skip + } + }, + } + + err = RegisterWorkflow(worker, pollingWorkflow) + require.NoError(t, err) + + err = worker.Start(ctx) + require.NoError(t, err) + defer worker.Shutdown(ctx) + + <-ctx.Done() + }(i) + } + + // Advance time to trigger polling + go func() { + for { + select { + case <-ctx.Done(): + return + default: + testClock.Tick(pollInterval / 4) + time.Sleep(5 * time.Millisecond) + } + } + }() + + wg.Wait() + close(pollEvents) + + // Analyze thundering herd behavior + pollTimes := make([]time.Time, 0) + for pollTime := range pollEvents { + pollTimes = append(pollTimes, pollTime) + } + + // Check for clustering (thundering herd indicator) + clustering := analyzePollingClustering(pollTimes, pollInterval) + + // Performance assertion + require.Less(t, clustering, 0.5, + "Polling clustering should be low to prevent thundering herd, got %.2f", clustering) + +} + +// pollingTestWorkflow is a minimal workflow that tracks when it's polled for +type pollingTestWorkflow struct { + engine *Engine + name string + onPoll func() +} + +func (w *pollingTestWorkflow) Name() string { + return w.name +} + +func (w *pollingTestWorkflow) Run(ctx WorkflowContext, req any) error { + // This is called when the workflow is actually executed + // We use onPoll to track when workers check for pending work + if w.onPoll != nil { + w.onPoll() + } + + _, err := Step(ctx, "polling-step", func(context.Context) (string, error) { + return "polled", nil + }) + return err +} + +func (w *pollingTestWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} + +// Helper function to analyze polling clustering (thundering herd detection) +func analyzePollingClustering(pollTimes []time.Time, pollInterval time.Duration) float64 { + if len(pollTimes) < 2 { + return 0 + } + + // Group polls by time windows + windowSize := pollInterval / 10 // 10% of poll interval + timeWindows := make(map[int64]int) + + baseTime := pollTimes[0] + for _, pollTime := range pollTimes { + windowIndex := pollTime.Sub(baseTime).Nanoseconds() / windowSize.Nanoseconds() + timeWindows[windowIndex]++ + } + + // Calculate clustering factor (higher = more clustered) + totalPolls := len(pollTimes) + maxWindowCount := 0 + + for _, count := range timeWindows { + if count > maxWindowCount { + maxWindowCount = count + } + } + + clustering := float64(maxWindowCount) / float64(totalPolls) + + return clustering +} diff --git a/go/pkg/hydra/workflow.go b/go/pkg/hydra/workflow.go new file mode 100644 index 00000000000..7cdfec4799d --- /dev/null +++ b/go/pkg/hydra/workflow.go @@ -0,0 +1,277 @@ +package hydra + +import ( + "context" + "fmt" + "time" + + "github.com/unkeyed/unkey/go/pkg/hydra/store" +) + +// Workflow defines the interface for typed workflows. +// +// Workflows are the core business logic containers in Hydra. They define +// a series of steps to be executed reliably with exactly-once guarantees. +// +// Workflows must be stateless and deterministic - they can be executed +// multiple times with the same input and produce the same result. State +// is managed by the workflow engine and persisted automatically. +// +// Type parameter TReq defines the input payload type for the workflow. +// Use 'any' for workflows that accept different payload types. +// +// Example implementation: +// +// type OrderWorkflow struct{} +// +// func (w *OrderWorkflow) Name() string { +// return "order-processing" +// } +// +// func (w *OrderWorkflow) Run(ctx hydra.WorkflowContext, req *OrderRequest) error { +// // Execute steps using hydra.Step() +// payment, err := hydra.Step(ctx, "validate-payment", func(stepCtx context.Context) (*Payment, error) { +// return validatePayment(stepCtx, req.PaymentID) +// }) +// if err != nil { +// return err +// } +// +// // Additional steps... +// return nil +// } +type Workflow[TReq any] interface { + // Name returns a unique identifier for this workflow type. + // The name is used to route workflow executions to the correct handler + // and must be consistent across deployments. + Name() string + + // Run executes the workflow logic with the provided context and request. + // This method should be deterministic and idempotent. + // + // The context provides access to workflow execution metadata and + // the Step() function for creating durable execution units. + // + // Returning an error will mark the workflow as failed and trigger + // retry logic if configured. Use hydra.Sleep() to suspend the + // workflow for time-based coordination. + Run(ctx WorkflowContext, req TReq) error +} + +// GenericWorkflow is a type alias for workflows that accept any request type. +// This is useful when registering workflows that handle different payload types +// or when the payload type is not known at compile time. +type GenericWorkflow = Workflow[any] + +// WorkflowContext provides access to workflow execution context and utilities. +// +// The context is passed to workflow Run() methods and provides access to: +// - The underlying Go context for cancellation and timeouts +// - Workflow execution metadata like execution ID and name +// - Step execution utilities through the Step() function +// +// Workflow contexts are created and managed by the workflow engine and +// should not be created manually. +type WorkflowContext interface { + // Context returns the underlying Go context for this workflow execution. + // This context will be cancelled if the workflow is cancelled or times out. + Context() context.Context + + // ExecutionID returns the unique identifier for this workflow execution. + // This ID can be used for logging, tracking, and debugging purposes. + ExecutionID() string + + // WorkflowName returns the name of the workflow being executed. + // This matches the value returned by the workflow's Name() method. + WorkflowName() string +} + +// workflowContext implements WorkflowContext and provides internal workflow utilities +type workflowContext struct { + ctx context.Context + executionID string + workflowName string + namespace string + workerID string + store store.Store + marshaller Marshaller + stepTimeout time.Duration + stepMaxAttempts int32 + stepOrder int +} + +func (w *workflowContext) Context() context.Context { + return w.ctx +} + +func (w *workflowContext) ExecutionID() string { + return w.executionID +} + +func (w *workflowContext) WorkflowName() string { + return w.workflowName +} + +func (w *workflowContext) getNextStepOrder() int32 { + w.stepOrder++ + return int32(w.stepOrder) // nolint:gosec // Overflow is extremely unlikely in practice +} + +func (w *workflowContext) getCompletedStep(stepName string) (*store.WorkflowStep, error) { + return w.store.GetCompletedStep(w.ctx, w.namespace, w.executionID, stepName) +} + +func (w *workflowContext) getAnyStep(stepName string) (*store.WorkflowStep, error) { + return w.store.GetStep(w.ctx, w.namespace, w.executionID, stepName) +} + +func (w *workflowContext) markStepCompleted(stepName string, outputData []byte) error { + return w.store.UpdateStepStatus(w.ctx, w.namespace, w.executionID, stepName, store.StepStatusCompleted, outputData, "") +} + +func (w *workflowContext) markStepFailed(stepName string, errorMsg string) error { + return w.store.UpdateStepStatus(w.ctx, w.namespace, w.executionID, stepName, store.StepStatusFailed, nil, errorMsg) +} + +func (w *workflowContext) suspendWorkflowForSleep(sleepUntil int64) error { + return w.store.SleepWorkflow(w.ctx, w.namespace, w.executionID, sleepUntil) +} + +// RegisterWorkflow registers a typed workflow with a worker. +// +// This function associates a workflow implementation with a worker so that +// the worker can execute workflows of this type. The workflow's Name() method +// is used as the unique identifier for routing workflow executions. +// +// The function handles type conversion transparently, allowing strongly-typed +// workflow implementations to be registered with the generic worker interface. +// +// Parameters: +// - w: The worker that will execute this workflow type +// - workflow: The workflow implementation to register +// +// Example: +// +// type OrderWorkflow struct{} +// +// func (w *OrderWorkflow) Name() string { return "order-processing" } +// func (w *OrderWorkflow) Run(ctx hydra.WorkflowContext, req *OrderRequest) error { +// // workflow implementation +// return nil +// } +// +// orderWorkflow := &OrderWorkflow{} +// err := hydra.RegisterWorkflow(worker, orderWorkflow) +// if err != nil { +// return err +// } +// +// Requirements: +// - The workflow name must be unique within the worker +// - The workflow must implement the Workflow[TReq] interface +// - The worker must be started with Start() after registration +// +// Returns an error if: +// - A workflow with the same name is already registered +// - The worker type is invalid +func RegisterWorkflow[TReq any](w Worker, workflow Workflow[TReq]) error { + worker, ok := w.(*worker) + if !ok { + return fmt.Errorf("invalid worker type") + } + + if _, exists := worker.workflows[workflow.Name()]; exists { + return fmt.Errorf("workflow %q is already registered", workflow.Name()) + } + + // Create a wrapper that handles the type conversion + genericWorkflow := &workflowWrapper[TReq]{ + wrapped: workflow, + } + + worker.workflows[workflow.Name()] = genericWorkflow + return nil +} + +// workflowWrapper wraps a typed workflow to implement GenericWorkflow +type workflowWrapper[TReq any] struct { + wrapped Workflow[TReq] +} + +func (w *workflowWrapper[TReq]) Name() string { + return w.wrapped.Name() +} + +func (w *workflowWrapper[TReq]) Run(ctx WorkflowContext, req any) error { + // Extract the raw payload and unmarshal it to the correct type + rawPayload, ok := req.(*RawPayload) + if !ok { + return fmt.Errorf("expected RawPayload, got %T", req) + } + + var typedReq TReq + wctx, ok := ctx.(*workflowContext) + if !ok { + return fmt.Errorf("invalid context type, expected *workflowContext") + } + if err := wctx.marshaller.Unmarshal(rawPayload.Data, &typedReq); err != nil { + return fmt.Errorf("failed to unmarshal workflow request: %w", err) + } + + return w.wrapped.Run(ctx, typedReq) +} + +// WorkflowOption defines a function that configures workflow execution +type WorkflowOption func(*WorkflowConfig) + +// WorkflowConfig holds the configuration for workflow execution +type WorkflowConfig struct { + MaxAttempts int32 + + TimeoutDuration time.Duration + + RetryBackoff time.Duration + + TriggerType TriggerType + TriggerSource *string +} + +// WithMaxAttempts sets the maximum number of retry attempts for a workflow +func WithMaxAttempts(attempts int32) WorkflowOption { + return func(c *WorkflowConfig) { + c.MaxAttempts = attempts + } +} + +// WithTimeout sets the timeout duration for a workflow +func WithTimeout(timeout time.Duration) WorkflowOption { + return func(c *WorkflowConfig) { + c.TimeoutDuration = timeout + } +} + +// WithRetryBackoff sets the retry backoff duration for a workflow +func WithRetryBackoff(backoff time.Duration) WorkflowOption { + return func(c *WorkflowConfig) { + c.RetryBackoff = backoff + } +} + +// WithTrigger sets the trigger type and source for a workflow +func WithTrigger(triggerType TriggerType, triggerSource *string) WorkflowOption { + return func(c *WorkflowConfig) { + c.TriggerType = triggerType + c.TriggerSource = triggerSource + } +} + +// WorkflowSuspendedError represents an error that suspends workflow execution until a specific time +type WorkflowSuspendedError struct { + Reason string + + ResumeTime int64 +} + +func (e *WorkflowSuspendedError) Error() string { + return fmt.Sprintf("workflow suspended for %s until %d", e.Reason, e.ResumeTime) +} diff --git a/go/pkg/hydra/workflow_performance_test.go b/go/pkg/hydra/workflow_performance_test.go new file mode 100644 index 00000000000..49e3a88b16c --- /dev/null +++ b/go/pkg/hydra/workflow_performance_test.go @@ -0,0 +1,422 @@ +package hydra + +import ( + "context" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/unkeyed/unkey/go/pkg/clock" + "github.com/unkeyed/unkey/go/pkg/hydra/store/gorm" +) + +// TestWorkflowPickupLatencyBaseline measures the baseline latency for a single worker +// to pick up and start executing a single workflow. This establishes our performance +// baseline before testing the 5-second SLA requirement. +func TestWorkflowPickupLatencyBaseline(t *testing.T) { + // Arrange: Create engine with real clock for accurate timing + realClock := clock.New() + engine := newTestEngineWithClock(t, realClock) + + var workflowStartTime atomic.Int64 + + // Create a workflow that records when it actually starts executing + workflow := &latencyTestWorkflow{ + engine: engine, + name: "baseline-latency-workflow", + onStart: func() { + workflowStartTime.Store(time.Now().UnixMilli()) + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Start worker with production-like configuration + worker, err := NewWorker(engine, WorkerConfig{ + Concurrency: 1, + PollInterval: 100 * time.Millisecond, // Realistic poll interval + HeartbeatInterval: 5 * time.Second, + ClaimTimeout: 30 * time.Second, + }) + require.NoError(t, err) + + err = RegisterWorkflow(worker, workflow) + require.NoError(t, err) + + err = worker.Start(ctx) + require.NoError(t, err) + defer worker.Shutdown(ctx) + + // Act: Record submission time and start workflow + _, err = workflow.Start(ctx, struct{}{}) + require.NoError(t, err) + + // Wait for workflow to start executing + require.Eventually(t, func() bool { + return workflowStartTime.Load() != 0 + }, 5*time.Second, 10*time.Millisecond, "Workflow should start executing within 5 seconds") + + // Calculate pickup latency + latency := time.Since(time.UnixMilli(workflowStartTime.Load())) + + require.Less(t, latency, 5*time.Second, "Pickup latency should be less than 5 seconds for baseline test") +} + +// latencyTestWorkflow is a minimal workflow for testing pickup latency +type latencyTestWorkflow struct { + engine *Engine + name string + onStart func() +} + +func (w *latencyTestWorkflow) Name() string { + return w.name +} + +func (w *latencyTestWorkflow) Run(ctx WorkflowContext, req any) error { + // Record when workflow actually starts executing + if w.onStart != nil { + w.onStart() + } + + // Minimal work to complete quickly + _, err := Step(ctx, "timing-step", func(context.Context) (string, error) { + return "completed", nil + }) + return err +} + +func (w *latencyTestWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} + +// TestWorkflowPickupLatencyConcurrent verifies that ALL workflows are picked up within the 5-second SLA +// under concurrent load. This tests the critical requirement that every workflow must be processed +// within the SLA window, not just the average. +func TestWorkflowPickupLatencyConcurrent(t *testing.T) { + // Arrange: Create engine with real clock for accurate timing + realClock := clock.New() + engine := newTestEngineWithClock(t, realClock) + + const numWorkers = 5 // Multiple workers to test concurrent performance + const numWorkflows = 50 // Realistic batch to stress test SLA compliance + + var completedCount atomic.Int64 + var maxLatency atomic.Int64 + var slaViolations atomic.Int64 + + // Create workflow factory that records completion timing + createWorkflow := func(id int) *concurrentLatencyWorkflow { + return &concurrentLatencyWorkflow{ + engine: engine, + name: "concurrent-latency-workflow", + id: id, + onComplete: func(latencyMs int64) { + // Track maximum latency across all workflows + for { + current := maxLatency.Load() + if latencyMs <= current || maxLatency.CompareAndSwap(current, latencyMs) { + break + } + } + + // Count SLA violations (workflows taking >5s) + if latencyMs > 5000 { + slaViolations.Add(1) + t.Errorf("SLA VIOLATION: Workflow %d took %dms (>5000ms) to be picked up", id, latencyMs) + } + + completedCount.Add(1) + }, + } + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Start multiple workers sharing the same database + workers := make([]Worker, numWorkers) + for i := 0; i < numWorkers; i++ { + worker, err := NewWorker(engine, WorkerConfig{ + Concurrency: 10, // Reasonable concurrency per worker + PollInterval: 50 * time.Millisecond, // Fast polling for concurrent load + HeartbeatInterval: 5 * time.Second, + ClaimTimeout: 30 * time.Second, + }) + require.NoError(t, err) + + // Register the workflow type with each worker + err = RegisterWorkflow(worker, createWorkflow(0)) + require.NoError(t, err) + + err = worker.Start(ctx) + require.NoError(t, err) + defer worker.Shutdown(ctx) + + workers[i] = worker + } + + // Act: Submit all workflows as quickly as possible + submissionStart := time.Now() + executionIDs := make([]string, numWorkflows) + + for i := 0; i < numWorkflows; i++ { + workflow := createWorkflow(i) + executionID, err := workflow.Start(ctx, submissionStart.UnixMilli()) + require.NoError(t, err) + executionIDs[i] = executionID + } + + _ = time.Since(submissionStart) // Submission timing not needed for SLA test + + // Wait for all workflows to complete + require.Eventually(t, func() bool { + return completedCount.Load() == numWorkflows + }, 15*time.Second, 100*time.Millisecond, + "All %d workflows should complete within timeout", numWorkflows) + + // Assert SLA compliance: ALL workflows must be picked up within 5 seconds + finalSlaViolations := slaViolations.Load() + finalMaxLatency := maxLatency.Load() + + require.Equal(t, int64(0), finalSlaViolations, + "SLA VIOLATION: %d out of %d workflows took longer than 5 seconds to be picked up", + finalSlaViolations, numWorkflows) + + require.Less(t, finalMaxLatency, int64(5000), + "SLA VIOLATION: Maximum pickup latency was %dms, must be <5000ms for ALL workflows", + finalMaxLatency) + +} + +// concurrentLatencyWorkflow tracks individual workflow latency in concurrent scenarios +type concurrentLatencyWorkflow struct { + engine *Engine + name string + id int + onComplete func(latencyMs int64) +} + +func (w *concurrentLatencyWorkflow) Name() string { + return w.name +} + +func (w *concurrentLatencyWorkflow) Run(ctx WorkflowContext, req any) error { + var submissionTime int64 + switch v := req.(type) { + case int64: + submissionTime = v + case float64: + submissionTime = int64(v) // JSON unmarshaling converts numbers to float64 + default: + return fmt.Errorf("expected int64 or float64 submission time, got %T", req) + } + + // Calculate latency from submission to execution start + latency := time.Now().UnixMilli() - submissionTime + + // Report completion with latency + if w.onComplete != nil { + w.onComplete(latency) + } + + // Minimal work to complete quickly + _, err := Step(ctx, "latency-step", func(context.Context) (string, error) { + return "completed", nil + }) + + return err +} + +func (w *concurrentLatencyWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} + +// BenchmarkWorkflowSubmission measures the rate at which workflows can be submitted +func BenchmarkWorkflowSubmission(b *testing.B) { + engine := newTestEngineBench(b) + + workflow := &benchmarkWorkflow{ + engine: engine, + name: "benchmark-workflow", + } + + ctx := context.Background() + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, err := workflow.Start(ctx, struct{}{}) + if err != nil { + b.Fatal(err) + } + } + }) +} + +// BenchmarkWorkflowThroughput measures end-to-end workflow processing throughput +func BenchmarkWorkflowThroughput(b *testing.B) { + engine := newTestEngineBench(b) + + workflow := &benchmarkWorkflow{ + engine: engine, + name: "throughput-workflow", + } + + // Start a single worker + worker, err := NewWorker(engine, WorkerConfig{ + Concurrency: 10, // Process multiple workflows concurrently + PollInterval: 10 * time.Millisecond, // Fast polling for benchmarks + HeartbeatInterval: 5 * time.Second, + ClaimTimeout: 30 * time.Second, + }) + if err != nil { + b.Fatal(err) + } + + err = RegisterWorkflow(worker, workflow) + if err != nil { + b.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err = worker.Start(ctx) + if err != nil { + b.Fatal(err) + } + defer worker.Shutdown(ctx) + + // Give worker time to start + time.Sleep(50 * time.Millisecond) + + b.ResetTimer() + + // Track completion + var completed atomic.Int64 + workflow.onComplete = func() { + completed.Add(1) + } + + // Submit N workflows as fast as possible + submissionStart := time.Now() + for i := 0; i < b.N; i++ { + _, err := workflow.Start(ctx, struct{}{}) + if err != nil { + b.Fatal(err) + } + } + submissionDuration := time.Since(submissionStart) + + // Wait for all workflows to complete + for completed.Load() < int64(b.N) { + time.Sleep(1 * time.Millisecond) + } + + b.ReportMetric(float64(b.N)/submissionDuration.Seconds(), "submissions/sec") + b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "completions/sec") +} + +// BenchmarkSingleWorkerLatency measures latency with a single worker processing one workflow at a time +func BenchmarkSingleWorkerLatency(b *testing.B) { + engine := newTestEngineBench(b) + + workflow := &benchmarkWorkflow{ + engine: engine, + name: "latency-workflow", + } + + worker, err := NewWorker(engine, WorkerConfig{ + Concurrency: 1, // Single workflow at a time + PollInterval: 1 * time.Millisecond, // Very fast polling + HeartbeatInterval: 5 * time.Second, + ClaimTimeout: 30 * time.Second, + }) + if err != nil { + b.Fatal(err) + } + + err = RegisterWorkflow(worker, workflow) + if err != nil { + b.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err = worker.Start(ctx) + if err != nil { + b.Fatal(err) + } + defer worker.Shutdown(ctx) + + time.Sleep(50 * time.Millisecond) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + completed := make(chan struct{}) + workflow.onComplete = func() { + close(completed) + } + + start := time.Now() + _, err := workflow.Start(ctx, struct{}{}) + if err != nil { + b.Fatal(err) + } + + <-completed + latency := time.Since(start) + + // Report per-operation timing + if i == 0 { + b.ReportMetric(float64(latency.Nanoseconds()), "ns/workflow") + } + } +} + +// benchmarkWorkflow is a minimal workflow for benchmarking +type benchmarkWorkflow struct { + engine *Engine + name string + onComplete func() +} + +func (w *benchmarkWorkflow) Name() string { + return w.name +} + +func (w *benchmarkWorkflow) Run(ctx WorkflowContext, req any) error { + // Minimal work - just complete a simple step + _, err := Step(ctx, "benchmark-step", func(context.Context) (string, error) { + return "done", nil + }) + + if w.onComplete != nil { + w.onComplete() + } + + return err +} + +func (w *benchmarkWorkflow) Start(ctx context.Context, payload any) (string, error) { + return w.engine.StartWorkflow(ctx, w.Name(), payload) +} + +// Helper for benchmarks that need testing.TB interface +func newTestEngineBench(tb testing.TB) *Engine { + store, err := gorm.NewSQLiteStore(tb.TempDir()+"/bench.db", nil) + if err != nil { + tb.Fatal(err) + } + + return New(Config{ + Store: store, + Clock: clock.New(), + }) +} diff --git a/go/pkg/uid/uid.go b/go/pkg/uid/uid.go index ec811785350..cb7347bb993 100644 --- a/go/pkg/uid/uid.go +++ b/go/pkg/uid/uid.go @@ -30,6 +30,8 @@ const ( AuditLogBucketPrefix Prefix = "buk" AuditLogPrefix Prefix = "log" InstancePrefix Prefix = "ins" + WorkerPrefix Prefix = "wkr" + CronJobPrefix Prefix = "cron" KeyEncryptionKeyPrefix Prefix = "kek" OrgPrefix Prefix = "org" )