Repository navigation
feat(router): allow split config polling #2814
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 9 commits
4c16202
24b3add
519a4f3
8d675ea
25b8ec5
7cd3a63
8c3e2df
d0675dd
1dceabe
88953e5
2284071
88ef61f
8159cb4
fa66786
579de46
adf6a89
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,17 +2,27 @@ package jwt | |
|
|
||
| import ( | ||
| "fmt" | ||
| "slices" | ||
|
|
||
| "github.com/golang-jwt/jwt/v5" | ||
| ) | ||
|
|
||
| const ( | ||
| FederatedGraphIDClaim = "federated_graph_id" | ||
| OrganizationIDClaim = "organization_id" | ||
| FederatedGraphIDClaim = "federated_graph_id" | ||
| OrganizationIDClaim = "organization_id" | ||
| FeaturesClaim = "features" | ||
| FeatureSplitConfigLoading = "split-config-loading" | ||
| ) | ||
|
|
||
| type FederatedGraphTokenClaims struct { | ||
| FederatedGraphID string | ||
| OrganizationID string | ||
| Features []string | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: You could also use a hash set here to not use
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think we will ever run into the amount of features where we can feel this |
||
| } | ||
|
|
||
| // HasFeature reports whether the given feature flag is present in the token claims. | ||
| func (c *FederatedGraphTokenClaims) HasFeature(feature string) bool { | ||
| return slices.Contains(c.Features, feature) | ||
| } | ||
|
|
||
| func ExtractFederatedGraphTokenClaims(token string) (*FederatedGraphTokenClaims, error) { | ||
|
|
@@ -44,8 +54,24 @@ func ExtractFederatedGraphTokenClaims(token string) (*FederatedGraphTokenClaims, | |
| return nil, fmt.Errorf("invalid token claims, %q is not a string, it's %T", OrganizationIDClaim, organizationIDValue) | ||
| } | ||
|
|
||
| var features []string | ||
| if featuresValue := claims[FeaturesClaim]; featuresValue != nil { | ||
| featuresSlice, ok := featuresValue.([]any) | ||
| if !ok { | ||
| return nil, fmt.Errorf("invalid token claims, %q is not an array, it's %T", FeaturesClaim, featuresValue) | ||
| } | ||
| for _, v := range featuresSlice { | ||
| s, ok := v.(string) | ||
| if !ok { | ||
| return nil, fmt.Errorf("invalid token claims, %q contains a non-string element %T", FeaturesClaim, v) | ||
| } | ||
| features = append(features, s) | ||
| } | ||
| } | ||
|
|
||
| return &FederatedGraphTokenClaims{ | ||
| FederatedGraphID: federatedGraphID, | ||
| OrganizationID: organizationID, | ||
| Features: features, | ||
| }, nil | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,272 @@ | ||
| package configpoller | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "slices" | ||
| "time" | ||
|
|
||
| "github.com/cespare/xxhash/v2" | ||
| nodev1 "github.com/wundergraph/cosmo/router/gen/proto/wg/cosmo/node/v1" | ||
| "github.com/wundergraph/cosmo/router/pkg/controlplane" | ||
| "github.com/wundergraph/cosmo/router/pkg/routerconfig" | ||
| "go.uber.org/zap" | ||
| "google.golang.org/protobuf/proto" | ||
| ) | ||
|
|
||
| // SplitConfigFetcher fetches individual graph configs from CDN using the split-config strategy. | ||
| type SplitConfigFetcher interface { | ||
| // FetchMapper fetches the mapper file listing all active graphs and their hashes. | ||
| FetchMapper(ctx context.Context) (*nodev1.ActiveGraphs, error) | ||
| // FetchConfig fetches the router config for the given feature flag name. | ||
| // featureFlagName="" fetches the base graph (manifest/latest.json). | ||
| // featureFlagName="X" fetches the feature flag config (manifest/feature-flags/X.json). | ||
| FetchConfig(ctx context.Context, featureFlagName string) (*nodev1.RouterConfig, error) | ||
| } | ||
|
|
||
| // SplitConfigPollerOption configures a splitConfigPoller. | ||
| type SplitConfigPollerOption func(*splitConfigPoller) | ||
|
|
||
| type splitConfigPoller struct { | ||
| logger *zap.Logger | ||
| poller controlplane.Poller | ||
| pollInterval time.Duration | ||
| pollJitter time.Duration | ||
| fetcher SplitConfigFetcher | ||
|
|
||
| // Internal state – not safe for concurrent access. | ||
| knownHashes map[string]string // name -> hash from last successful mapper fetch ("" = base) | ||
| currentConfig *nodev1.RouterConfig // last successfully assembled full config | ||
| latestVersion string // composite hash used for change detection | ||
| } | ||
|
|
||
| // NewSplitConfigPoller creates a ConfigPoller that uses the split-config strategy. | ||
| func NewSplitConfigPoller(fetcher SplitConfigFetcher, opts ...SplitConfigPollerOption) ConfigPoller { | ||
| p := &splitConfigPoller{ | ||
| fetcher: fetcher, | ||
| knownHashes: make(map[string]string), | ||
| } | ||
| for _, opt := range opts { | ||
| opt(p) | ||
| } | ||
| if p.logger == nil { | ||
| p.logger = zap.NewNop() | ||
| } | ||
| p.poller = controlplane.NewPoll(p.pollInterval, p.pollJitter) | ||
| return p | ||
| } | ||
|
|
||
| // WithSplitLogger sets the logger for the split config poller. | ||
| func WithSplitLogger(logger *zap.Logger) SplitConfigPollerOption { | ||
| return func(p *splitConfigPoller) { | ||
| p.logger = logger | ||
| } | ||
| } | ||
|
|
||
| // WithSplitPolling sets the polling interval and jitter for the split config poller. | ||
| func WithSplitPolling(interval time.Duration, jitter time.Duration) SplitConfigPollerOption { | ||
| return func(p *splitConfigPoller) { | ||
| p.pollInterval = interval | ||
| p.pollJitter = jitter | ||
| } | ||
| } | ||
|
|
||
| // computeCompositeVersion returns a deterministic version string derived from all mapper entries. | ||
| func computeCompositeVersion(graphConfigs map[string]string) string { | ||
| keys := make([]string, 0, len(graphConfigs)) | ||
| for k := range graphConfigs { | ||
| keys = append(keys, k) | ||
| } | ||
| slices.Sort(keys) | ||
|
|
||
| h := xxhash.New() | ||
| for _, k := range keys { | ||
| _, _ = h.Write([]byte(k + ":" + graphConfigs[k] + ";")) | ||
| } | ||
| return fmt.Sprintf("split-%x", h.Sum64()) | ||
| } | ||
|
|
||
| // fetchAndAssembleAll fetches every config listed in activeGraphs and assembles a full RouterConfig. | ||
| func (p *splitConfigPoller) fetchAndAssembleAll(ctx context.Context, activeGraphs *nodev1.ActiveGraphs) (*nodev1.RouterConfig, error) { | ||
| // Fetch base graph. | ||
| baseConfig, err := p.fetcher.FetchConfig(ctx, "") | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to fetch base config: %w", err) | ||
| } | ||
|
|
||
| assembled := &nodev1.RouterConfig{ | ||
| EngineConfig: baseConfig.EngineConfig, | ||
| Version: baseConfig.Version, | ||
| Subgraphs: baseConfig.Subgraphs, | ||
| CompatibilityVersion: baseConfig.CompatibilityVersion, | ||
| } | ||
|
|
||
| // Fetch feature flag configs. | ||
| for name := range activeGraphs.GetGraphConfigs() { | ||
| if name == "" { | ||
| continue // base graph already handled above | ||
| } | ||
| ffConfig, err := p.fetcher.FetchConfig(ctx, name) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to fetch config for feature flag %q: %w", name, err) | ||
| } | ||
| if assembled.FeatureFlagConfigs == nil { | ||
| assembled.FeatureFlagConfigs = &nodev1.FeatureFlagRouterExecutionConfigs{ | ||
| ConfigByFeatureFlagName: make(map[string]*nodev1.FeatureFlagRouterExecutionConfig), | ||
| } | ||
| } | ||
| assembled.FeatureFlagConfigs.ConfigByFeatureFlagName[name] = &nodev1.FeatureFlagRouterExecutionConfig{ | ||
| EngineConfig: ffConfig.EngineConfig, | ||
| Version: ffConfig.Version, | ||
| Subgraphs: ffConfig.Subgraphs, | ||
| } | ||
| } | ||
|
|
||
| return assembled, nil | ||
| } | ||
|
|
||
| // GetRouterConfig performs the initial fetch: mapper + all individual configs. | ||
| func (p *splitConfigPoller) GetRouterConfig(ctx context.Context) (*routerconfig.Response, error) { | ||
| activeGraphs, err := p.fetcher.FetchMapper(ctx) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to fetch mapper: %w", err) | ||
| } | ||
|
|
||
| graphConfigs := activeGraphs.GetGraphConfigs() | ||
| if len(graphConfigs) == 0 { | ||
| return nil, fmt.Errorf("mapper returned empty graph configs") | ||
| } | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
|
|
||
| config, err := p.fetchAndAssembleAll(ctx, activeGraphs) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| p.knownHashes = graphConfigs | ||
| p.currentConfig = config | ||
| p.latestVersion = computeCompositeVersion(graphConfigs) | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
|
|
||
| return &routerconfig.Response{Config: config}, nil | ||
| } | ||
|
|
||
| // Subscribe starts the polling loop and calls handler whenever the assembled config changes. | ||
| func (p *splitConfigPoller) Subscribe(ctx context.Context, handler func(newConfig *nodev1.RouterConfig, oldVersion string) error) { | ||
| p.poller.Subscribe(ctx, func() { | ||
| fetchStart := time.Now() | ||
|
|
||
| activeGraphs, err := p.fetcher.FetchMapper(ctx) | ||
| if err != nil { | ||
| p.logger.Error("Failed to fetch mapper during poll, keeping existing config", zap.Error(err)) | ||
| return | ||
| } | ||
|
|
||
| graphConfigs := activeGraphs.GetGraphConfigs() | ||
| if len(graphConfigs) == 0 { | ||
| p.logger.Warn("Mapper returned empty graph configs, keeping existing config") | ||
| return | ||
| } | ||
|
|
||
| newVersion := computeCompositeVersion(graphConfigs) | ||
| if newVersion == p.latestVersion { | ||
| p.logger.Debug("No changes detected in engine config, keeping existing config") | ||
| return | ||
| } | ||
|
|
||
| p.logger.Info("Router execution config has changed, hot reloading server", | ||
| zap.String("old_version", p.latestVersion), | ||
| zap.String("new_version", newVersion), | ||
| zap.String("fetch_time", time.Since(fetchStart).String()), | ||
| ) | ||
|
|
||
| // Determine what changed, was added, or was removed. | ||
| changed := make(map[string]struct{}) | ||
| added := make(map[string]struct{}) | ||
| removed := make(map[string]struct{}) | ||
|
|
||
| for name, hash := range graphConfigs { | ||
| if oldHash, exists := p.knownHashes[name]; !exists { | ||
| added[name] = struct{}{} | ||
| } else if oldHash != hash { | ||
| changed[name] = struct{}{} | ||
| } | ||
| } | ||
| for name := range p.knownHashes { | ||
| if _, exists := graphConfigs[name]; !exists { | ||
| removed[name] = struct{}{} | ||
| } | ||
| } | ||
|
|
||
| // Clone the in-use config before mutating. | ||
| patched := proto.Clone(p.currentConfig).(*nodev1.RouterConfig) | ||
|
|
||
| // Apply changes and additions. | ||
| toFetch := make(map[string]struct{}, len(changed)+len(added)) | ||
| for name := range changed { | ||
| toFetch[name] = struct{}{} | ||
| } | ||
| for name := range added { | ||
| toFetch[name] = struct{}{} | ||
| } | ||
|
|
||
| for name := range toFetch { | ||
| fetchedConfig, err := p.fetcher.FetchConfig(ctx, name) | ||
| if err != nil { | ||
| p.logger.Error("Failed to fetch config, skipping entire update", | ||
| zap.String("name", name), | ||
| zap.Error(err), | ||
| ) | ||
| return | ||
| } | ||
|
|
||
| if name == "" { | ||
| // Base graph update. | ||
| patched.EngineConfig = fetchedConfig.EngineConfig | ||
| patched.Version = fetchedConfig.Version | ||
| patched.Subgraphs = fetchedConfig.Subgraphs | ||
| patched.CompatibilityVersion = fetchedConfig.CompatibilityVersion | ||
| } else { | ||
| if patched.FeatureFlagConfigs == nil { | ||
| patched.FeatureFlagConfigs = &nodev1.FeatureFlagRouterExecutionConfigs{ | ||
| ConfigByFeatureFlagName: make(map[string]*nodev1.FeatureFlagRouterExecutionConfig), | ||
| } | ||
| } | ||
| patched.FeatureFlagConfigs.ConfigByFeatureFlagName[name] = &nodev1.FeatureFlagRouterExecutionConfig{ | ||
| EngineConfig: fetchedConfig.EngineConfig, | ||
| Version: fetchedConfig.Version, | ||
| Subgraphs: fetchedConfig.Subgraphs, | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // Remove deleted feature flags. | ||
| for name := range removed { | ||
| if name == "" { | ||
| continue // base graph cannot be removed | ||
| } | ||
| if patched.FeatureFlagConfigs != nil { | ||
| delete(patched.FeatureFlagConfigs.ConfigByFeatureFlagName, name) | ||
| if len(patched.FeatureFlagConfigs.ConfigByFeatureFlagName) == 0 { | ||
| patched.FeatureFlagConfigs = nil | ||
| } | ||
| } | ||
| } | ||
|
|
||
| oldVersion := p.latestVersion | ||
|
|
||
| handlerStart := time.Now() | ||
| if err := handler(patched, oldVersion); err != nil { | ||
| p.logger.Error("Error invoking config poll handler", zap.Error(err)) | ||
| return | ||
| } | ||
|
|
||
| p.logger.Debug("New graph server swapped", | ||
| zap.String("duration", time.Since(handlerStart).String()), | ||
| zap.String("config_version", newVersion), | ||
| ) | ||
|
|
||
| // Only update internal state after the handler succeeds. | ||
| p.knownHashes = graphConfigs | ||
| p.currentConfig = patched | ||
| p.latestVersion = newVersion | ||
| }) | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You can swap the
ifthen you can omitelseThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Fixed