diff --git a/src/platform/plugins/shared/workflows_execution_engine/server/filter_contribution.md b/src/platform/plugins/shared/workflows_execution_engine/server/filter_contribution.md index 2f95420eb463a..8ca37525310d3 100644 --- a/src/platform/plugins/shared/workflows_execution_engine/server/filter_contribution.md +++ b/src/platform/plugins/shared/workflows_execution_engine/server/filter_contribution.md @@ -4,10 +4,13 @@ This guide covers the complete process of adding custom filters or tags to the t Adding a new filter/tag requires updates in multiple places: -1. **Engine registration** (server-side) -2. **UI parsing** (frontend validation) -3. **Autocompletion** (UI suggestions) -4. **OSS contribution** (optional, recommended) +1. **Engine registration** (server-side execution) +2. **UI parsing** (frontend validation — prevents editor errors) +3. **UI hover evaluation** (frontend expression evaluator — enables hover tooltips) +4. **Autocompletion** (UI suggestions in Monaco editor) +5. **Foreach schema awareness** (if the filter changes the output type, e.g. object → array) +6. **Tests** (server-side and schema tests) +7. **OSS contribution** (optional, recommended) ## Step 1: Register in the Engine @@ -89,7 +92,29 @@ function getLiquidInstance(): Liquid { **Why?** This prevents parsing errors when users type the filter in the UI editor. -## Step 3: Add to Autocompletion +## Step 3: Register in the UI Hover Expression Evaluator + +The hover tooltip in the YAML editor evaluates template expressions client-side to show live values. +Custom filters must also be registered here, otherwise hover evaluation falls back to path-only +resolution and shows incorrect results (or "undefined"). + +**File:** `src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/template_expression/evaluate_expression.ts` + +```typescript +// Register custom filters that match server-side exactly +liquidEngine.registerFilter('my_custom_filter', (value: unknown): unknown => { + // Full implementation (same as server-side) — this runs during hover evaluation + if (typeof value !== 'string') { + return value; + } + return transformedValue; +}); +``` + +**Why?** Unlike the validation instance (Step 2) which only needs a no-op stub, the hover evaluator +needs the **real implementation** since it evaluates expressions against actual execution data. + +## Step 4: Add to Autocompletion Add your filter to the autocompletion provider: @@ -113,7 +138,30 @@ export const LIQUID_FILTERS = [ - `insertText`: Snippet for autocompletion (use `${1:placeholder}` for arguments) - `example`: Usage example -## Step 4: Write Tests +## Step 5: Update Foreach Schema Awareness (if type-changing) + +If your filter transforms the output type (e.g., `entries` converts an object to an array of `{key, value}` pairs), +the foreach schema resolver needs to know about it. Otherwise, the editor will show "Expected array for foreach +iteration, but got object" when using `| your_filter` in a `foreach:` expression. + +**File:** `src/platform/plugins/shared/workflows_management/public/features/workflow_context/lib/get_foreach_state_schema.ts` + +Add a check for your filter in the `getForeachItemSchema` function, before the final `else` branch +that throws `InvalidForeachParameterError`: + +```typescript +} else if (iterableSchema instanceof z.ZodObject && hasMyFilter) { + // my_filter converts objects to arrays of a known shape + return z.object({ /* your item schema */ }); +} else { + throw new InvalidForeachParameterError(...); +} +``` + +**When is this needed?** Only when the filter changes the type in a way that affects `foreach` iteration +(e.g., object → array). Filters that preserve the type (e.g., `json_parse`, `upcase`) don't need this. + +## Step 6: Write Tests Add tests for your filter: @@ -137,7 +185,7 @@ describe('my_custom_filter', () => { }); ``` -## Step 5: Contribute to OSS (Optional) +## Step 7: Contribute to OSS (Optional) If you want to contribute your filter to the [liquidjs](https://github.com/harttle/liquidjs) project: diff --git a/src/platform/plugins/shared/workflows_execution_engine/server/templating_engine.test.ts b/src/platform/plugins/shared/workflows_execution_engine/server/templating_engine.test.ts index 87a2dda1721ac..27d29d9f47f9e 100644 --- a/src/platform/plugins/shared/workflows_execution_engine/server/templating_engine.test.ts +++ b/src/platform/plugins/shared/workflows_execution_engine/server/templating_engine.test.ts @@ -155,6 +155,79 @@ describe('WorkflowTemplatingEngine', () => { }); }); + describe('custom entries filter', () => { + it('should convert an object to an array of {key, value} pairs', () => { + const template = '${{ data | entries }}'; + const context = { data: { a: 1, b: 2, c: 3 } }; + const result = templatingEngine.render({ out: template }, context); + expect(result.out).toEqual([ + { key: 'a', value: 1 }, + { key: 'b', value: 2 }, + { key: 'c', value: 3 }, + ]); + }); + + it('should handle nested object values', () => { + const template = '${{ data | entries }}'; + const context = { + data: { + 'index-001': { settings: { lifecycle: { name: 'old-policy' } } }, + 'index-002': { settings: { lifecycle: { name: 'new-policy' } } }, + }, + }; + const result = templatingEngine.render({ out: template }, context); + expect(result.out).toEqual([ + { key: 'index-001', value: { settings: { lifecycle: { name: 'old-policy' } } } }, + { key: 'index-002', value: { settings: { lifecycle: { name: 'new-policy' } } } }, + ]); + }); + + it('should return arrays as-is', () => { + const template = '${{ data | entries }}'; + const context = { data: [1, 2, 3] }; + const result = templatingEngine.render({ out: template }, context); + expect(result.out).toEqual([1, 2, 3]); + }); + + it('should return null as-is', () => { + const template = '{{ data | entries }}'; + const context = { data: null }; + const result = templatingEngine.render(template, context); + expect(result).toBe(''); + }); + + it('should return non-object primitives as-is', () => { + const template = '{{ data | entries }}'; + const context = { data: 'hello' }; + const result = templatingEngine.render(template, context); + expect(result).toBe('hello'); + }); + + it('should handle empty objects', () => { + const template = '${{ data | entries }}'; + const context = { data: {} }; + const result = templatingEngine.render({ out: template }, context); + expect(result.out).toEqual([]); + }); + + it('should work with evaluateExpression for foreach usage', () => { + const template = '{{ data | entries }}'; + const context = { data: { x: 10, y: 20 } }; + const result = templatingEngine.evaluateExpression(template, context); + expect(result).toEqual([ + { key: 'x', value: 10 }, + { key: 'y', value: 20 }, + ]); + }); + + it('should chain with other filters', () => { + const template = '${{ data | entries | first }}'; + const context = { data: { a: 1, b: 2 } }; + const result = templatingEngine.render({ out: template }, context); + expect(result.out).toEqual({ key: 'a', value: 1 }); + }); + }); + describe('filter chaining', () => { it('should chain json and json_parse filters', () => { const template = '{{ data | json | json_parse | json }}'; diff --git a/src/platform/plugins/shared/workflows_execution_engine/server/templating_engine.ts b/src/platform/plugins/shared/workflows_execution_engine/server/templating_engine.ts index 7537de170cc6f..310e861c10ecf 100644 --- a/src/platform/plugins/shared/workflows_execution_engine/server/templating_engine.ts +++ b/src/platform/plugins/shared/workflows_execution_engine/server/templating_engine.ts @@ -34,6 +34,14 @@ export class WorkflowTemplatingEngine { return value; } }); + + // register entries filter that converts an object into an array of {key, value} pairs + this.engine.registerFilter('entries', (value: unknown): unknown => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return value; + } + return Object.entries(value).map(([k, v]) => ({ key: k, value: v })); + }); } public render(obj: T, context: Record): T { diff --git a/src/platform/plugins/shared/workflows_management/common/lib/validate_liquid_template.ts b/src/platform/plugins/shared/workflows_management/common/lib/validate_liquid_template.ts index 39ccc0f8c1ba9..98c664e838e3d 100644 --- a/src/platform/plugins/shared/workflows_management/common/lib/validate_liquid_template.ts +++ b/src/platform/plugins/shared/workflows_management/common/lib/validate_liquid_template.ts @@ -31,6 +31,9 @@ function getLiquidInstance(): Liquid { liquidInstance.registerFilter('json_parse', (value: unknown): unknown => { return value; }); + liquidInstance.registerFilter('entries', (value: unknown): unknown => { + return value; + }); } return liquidInstance; } diff --git a/src/platform/plugins/shared/workflows_management/public/features/workflow_context/lib/get_foreach_state_schema.test.ts b/src/platform/plugins/shared/workflows_management/public/features/workflow_context/lib/get_foreach_state_schema.test.ts index 274246541c663..b2e28913c041c 100644 --- a/src/platform/plugins/shared/workflows_management/public/features/workflow_context/lib/get_foreach_state_schema.test.ts +++ b/src/platform/plugins/shared/workflows_management/public/features/workflow_context/lib/get_foreach_state_schema.test.ts @@ -125,6 +125,54 @@ describe('getForeachStateSchema', () => { ); }); + it('should return {key, value} item schema when object has | entries filter', () => { + const stepContext = DynamicStepContextSchema.extend({ + consts: z.object({ + settings: z.object({ name: z.string(), surname: z.string() }), + }), + }); + const foreachStateSchema = getForeachStateSchema(stepContext, { + foreach: '{{consts.settings | entries}}', + type: 'foreach', + name: 'foreach-step', + }); + expectZodSchemaEqual( + foreachStateSchema, + ForEachContextSchema.extend({ + item: z.object({ key: z.string(), value: z.unknown() }), + items: z.array(z.object({ key: z.string(), value: z.unknown() })), + }) + ); + }); + + it('should return {key, value} item schema for deeply nested object with | entries filter and spaces', () => { + const stepContext = DynamicStepContextSchema.extend({ + consts: z.object({ + es_settings_response: z.object({ + '.ds-logs-docker-000001': z.object({ + settings: z.object({ + index: z.object({ + lifecycle: z.object({ name: z.literal('old-policy') }), + }), + }), + }), + }), + }), + }); + const foreachStateSchema = getForeachStateSchema(stepContext, { + foreach: '{{ consts.es_settings_response | entries }}', + type: 'foreach', + name: 'foreach-step', + }); + expectZodSchemaEqual( + foreachStateSchema, + ForEachContextSchema.extend({ + item: z.object({ key: z.string(), value: z.unknown() }), + items: z.array(z.object({ key: z.string(), value: z.unknown() })), + }) + ); + }); + it('should return an unknown schema with a description if the foreach parameter is not a valid JSON', () => { const stepContext = DynamicStepContextSchema.extend({ consts: z.object({ diff --git a/src/platform/plugins/shared/workflows_management/public/features/workflow_context/lib/get_foreach_state_schema.ts b/src/platform/plugins/shared/workflows_management/public/features/workflow_context/lib/get_foreach_state_schema.ts index 6311629793824..a8bc5c64240cf 100644 --- a/src/platform/plugins/shared/workflows_management/public/features/workflow_context/lib/get_foreach_state_schema.ts +++ b/src/platform/plugins/shared/workflows_management/public/features/workflow_context/lib/get_foreach_state_schema.ts @@ -97,12 +97,15 @@ const extractForeachItemSchemaFromJson = (foreachParam: string): z.ZodType => { return extractForeachItemSchemaFromArray(foreachParamParsed); }; +const ENTRIES_FILTER = 'entries'; + export function getForeachItemSchema( stepContextSchema: typeof DynamicStepContextSchema, foreachParam: string ): z.ZodType { const parsedPath = parseVariablePath(foreachParam); const iterateOverPath = parsedPath?.propertyPath; + const hasEntriesFilter = parsedPath?.filters?.includes(ENTRIES_FILTER) ?? false; // If we have a valid variable path syntax (e.g., {{some.path}}) if (parsedPath && !parsedPath.errors && iterateOverPath) { @@ -140,6 +143,8 @@ export function getForeachItemSchema( InvalidForeachParameterErrorCodes.INVALID_UNION ); } + } else if (iterableSchema instanceof z.ZodObject && hasEntriesFilter) { + return z.object({ key: z.string(), value: z.unknown() }); } else { throw new InvalidForeachParameterError( `Expected array for foreach iteration, but got ${getZodTypeName( diff --git a/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/autocomplete/suggestions/liquid/liquid_completions.ts b/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/autocomplete/suggestions/liquid/liquid_completions.ts index 4b2b4ae32937e..6215d8ec75a5b 100644 --- a/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/autocomplete/suggestions/liquid/liquid_completions.ts +++ b/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/autocomplete/suggestions/liquid/liquid_completions.ts @@ -132,6 +132,14 @@ export const LIQUID_FILTERS = [ insertText: 'downcase', example: '{{ "HELLO" | downcase }} => hello', }, + { + name: 'entries', + description: + 'Converts an object into an array of {key, value} pairs, enabling iteration over object keys with foreach', + insertText: 'entries', + example: + '{{ {"a": 1, "b": 2} | entries }} => [{"key": "a", "value": 1}, {"key": "b", "value": 2}]', + }, { name: 'escape', description: 'Escapes a string by replacing characters with escape sequences', diff --git a/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/execution_context/build_execution_context.ts b/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/execution_context/build_execution_context.ts index 01dfd7300ceb5..04945228d4f64 100644 --- a/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/execution_context/build_execution_context.ts +++ b/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/execution_context/build_execution_context.ts @@ -43,17 +43,35 @@ export function buildExecutionContext( return null; } - // Build steps object from step executions + // Build steps object from step executions. + // For foreach steps the server emits multiple executions with the same stepId + // (enter, each iteration advance, exit). Preserve the entry that carries + // state.index so findForeachContext can re-evaluate the foreach expression. const steps: Record = {}; for (const stepExecution of stepExecutions) { - steps[stepExecution.stepId] = { - output: stepExecution.output as JsonValue | undefined, - error: stepExecution.error, - input: stepExecution.input as JsonValue | undefined, - status: stepExecution.status, - state: stepExecution.state as JsonObject | undefined, - }; + const existing = steps[stepExecution.stepId]; + const newState = stepExecution.state as JsonObject | undefined; + + if ( + existing?.state && + typeof existing.state.index === 'number' && + newState && + typeof newState.index !== 'number' + ) { + // Don't overwrite a foreach entry (has state.index) with the exit (no index) + existing.output = existing.output ?? (stepExecution.output as JsonValue | undefined); + existing.error = existing.error ?? stepExecution.error; + existing.status = stepExecution.status; + } else { + steps[stepExecution.stepId] = { + output: stepExecution.output as JsonValue | undefined, + error: stepExecution.error, + input: stepExecution.input as JsonValue | undefined, + status: stepExecution.status, + state: newState, + }; + } } // Build full context diff --git a/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/monaco_providers/unified_hover_provider.ts b/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/monaco_providers/unified_hover_provider.ts index fe591ef7af535..ee4d2e18f8ea1 100644 --- a/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/monaco_providers/unified_hover_provider.ts +++ b/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/monaco_providers/unified_hover_provider.ts @@ -413,6 +413,40 @@ export class UnifiedHoverProvider implements monaco.languages.HoverProvider { return stepData; } + /** + * Ensure foreach step input AND referenced step outputs are loaded. + * The foreach expression (e.g. "{{ steps.X.output | entries }}") needs + * both the foreach step's input (to know the expression) and the + * referenced step's output (to evaluate it). + */ + private async enrichForeachStepInput(context: ExecutionContext): Promise { + let enrichedSteps = context.steps; + let changed = false; + + const enrich = async (sId: string, stepData: StepExecutionData) => { + const fullData = await this.fetchStepDataIfNeeded(stepData, sId); + if (fullData && fullData !== stepData) { + if (!changed) { + enrichedSteps = { ...context.steps }; + changed = true; + } + enrichedSteps[sId] = fullData; + } + }; + + for (const [sId, stepData] of Object.entries(context.steps)) { + const needsInput = + stepData.state && typeof stepData.state.index === 'number' && stepData.input === undefined; + const needsOutput = stepData.output === undefined; + + if (needsInput || needsOutput) { + await enrich(sId, stepData); + } + } + + return changed ? { ...context, steps: enrichedSteps } : context; + } + /** * Handle hover for template expressions {{ }} */ @@ -447,6 +481,12 @@ export class UnifiedHoverProvider implements monaco.languages.HoverProvider { } } + // For foreach.* paths, ensure the foreach step's input is loaded + // so findForeachContext can re-evaluate the foreach expression. + if (templateInfo.pathSegments[0] === 'foreach') { + evalContext = await this.enrichForeachStepInput(evalContext); + } + // Determine what to evaluate let value: JsonValue | undefined; let evaluatedPath: string; diff --git a/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/template_expression/evaluate_expression.ts b/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/template_expression/evaluate_expression.ts index a5754a3b00a70..266c759e4c2b8 100644 --- a/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/template_expression/evaluate_expression.ts +++ b/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/template_expression/evaluate_expression.ts @@ -30,6 +30,13 @@ liquidEngine.registerFilter('json_parse', (value: unknown): unknown => { } }); +liquidEngine.registerFilter('entries', (value: unknown): unknown => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return value; + } + return Object.entries(value).map(([k, v]) => ({ key: k, value: v })); +}); + export interface EvaluateExpressionOptions { /** The full expression including filters (e.g., "steps.search.output.hits | json") */ expression: string; @@ -93,26 +100,79 @@ function buildEnhancedContext( } /** - * Find foreach context by looking for foreach steps in execution - * Returns the first foreach step found with its items + * Find foreach context by looking for foreach steps in execution. + * The server stores only { index, total } in state (not the full items array). + * The foreach expression is stored in input.foreach, so we re-evaluate it + * against the execution context to derive the items array. */ function findForeachContext( context: ExecutionContext ): { item: JsonValue; index: number; total: number; items: JsonArray } | null { - // Look through all steps to find foreach steps for (const [, stepData] of Object.entries(context.steps)) { - // Check if this step has foreach state (items array) - if (stepData.state && 'items' in stepData.state && Array.isArray(stepData.state.items)) { - // This is a foreach step - const items = stepData.state.items as JsonArray; - // For simplicity, use the first item (as requested by user) - return { - item: items.length > 0 ? items[0] : null, - index: 0, - total: items.length, - items, - }; + if (stepData.state && typeof stepData.state.index === 'number') { + const index = stepData.state.index as number; + const total = (stepData.state.total as number) ?? 0; + + // Try to get items from state directly (legacy path) + if ('items' in stepData.state && Array.isArray(stepData.state.items)) { + const items = stepData.state.items as JsonArray; + return { + item: items[index] ?? (items.length > 0 ? items[0] : null), + index, + total, + items, + }; + } + + // Re-evaluate the foreach expression from the step's input to derive items + const foreachExpression = extractForeachExpression(stepData.input); + if (foreachExpression) { + const items = resolveForeachItems(foreachExpression, context); + if (items) { + return { + item: items[index] ?? (items.length > 0 ? items[0] : null), + index, + total, + items, + }; + } + } + } + } + return null; +} + +function extractForeachExpression(input: JsonValue | undefined): string | undefined { + if (input !== null && typeof input === 'object' && !Array.isArray(input)) { + const { foreach: expression } = input as Record; + return typeof expression === 'string' ? expression : undefined; + } + return undefined; +} + +function resolveForeachItems( + foreachExpression: string, + context: ExecutionContext +): JsonArray | null { + try { + let expression = foreachExpression.trim(); + const openIdx = expression.indexOf('{{'); + const closeIdx = expression.lastIndexOf('}}'); + if (openIdx !== -1 && closeIdx !== -1) { + expression = expression.substring(openIdx + 2, closeIdx).trim(); + } + const result = liquidEngine.evalValueSync(expression, context); + if (Array.isArray(result)) { + return result as JsonArray; + } + if (typeof result === 'string') { + const parsed = JSON.parse(result); + if (Array.isArray(parsed)) { + return parsed as JsonArray; + } } + } catch { + // ignore evaluation errors } return null; } diff --git a/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/template_expression/resolve_path_value.ts b/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/template_expression/resolve_path_value.ts index d982753fefc67..38b4f917f4d8f 100644 --- a/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/template_expression/resolve_path_value.ts +++ b/src/platform/plugins/shared/workflows_management/public/widgets/workflow_yaml_editor/lib/template_expression/resolve_path_value.ts @@ -57,7 +57,7 @@ export function truncateForDisplay( currentDepth?: number; } = {} ): JsonValue | string { - const { maxDepth = 5, maxProperties = 15, maxArrayItems = 5, currentDepth = 0 } = options; + const { maxDepth = 8, maxProperties = 15, maxArrayItems = 5, currentDepth = 0 } = options; // Max depth reached if (currentDepth >= maxDepth) {