Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
54 commits
Select commit Hold shift + click to select a range
646d03a
[Workflows] Add sync execution types and Liquid template-string suppo…
jonwalstedt Sep 2, 2026
aca1eee
[Workflows] Address Copilot review comments on withTemplateStringSupp…
jonwalstedt Sep 4, 2026
19a626c
[Workflows] Extract WorkflowExecutionPersistence / StepExecutionPersi…
jonwalstedt Sep 2, 2026
9d1936a
[Workflows] Return defensive copies from InMemoryExecutionPersistence…
jonwalstedt Sep 4, 2026
81e19e4
[Workflows] Prevent identity fields from being overwritten in InMemor…
jonwalstedt Sep 4, 2026
a372d38
[Workflows] Implement sourceIncludes/sourceExcludes in InMemoryExecut…
jonwalstedt Sep 4, 2026
7bbb955
[Workflows] Upgrade defensive copies to structuredClone in InMemoryEx…
jonwalstedt Sep 5, 2026
1170ac7
[Workflows] Extend synchronous execution for inference capabilities
jonwalstedt Sep 2, 2026
730617f
[Workflows] Address sync-execution review comments
jonwalstedt Sep 8, 2026
21fdd6c
[Workflows] Remove WorkflowTriggerMatchOutcome from public server index
jonwalstedt Sep 8, 2026
f1c4d27
[Workflows] Document capabilities threat model and add serialization …
jonwalstedt Sep 8, 2026
7b4b653
[Workflows] Address remaining sync-execution review findings
jonwalstedt Sep 8, 2026
d98dcfe
[Workflows] Fix critic review findings in sync-execution test pass
jonwalstedt Sep 8, 2026
6381b82
[Workflows] Replace hardcoded trigger conflict check with registratio…
jonwalstedt Sep 8, 2026
6a0e61b
[Inference] Add standalone RE2-only PII detection runtime for workflo…
jonwalstedt Sep 2, 2026
9e51e89
[Inference] Add native RegExp fallback to PII detection runtime
jonwalstedt Sep 4, 2026
6e39e7c
[Inference] Fix HMAC input format and add generateEntityToken tests
jonwalstedt Sep 4, 2026
d37ee5b
[Inference] Fix err type narrowing and timeout message in PiiRegexWor…
jonwalstedt Sep 4, 2026
031d123
[Inference] Add PiiRegexWorkerService integration tests
jonwalstedt Sep 4, 2026
df1c217
[Inference] Add anonymization config + decouple from third-effort ano…
jonwalstedt Sep 2, 2026
9b9e075
fix(inference): mark anonymization encryptionKey as sensitive in conf…
jonwalstedt Sep 6, 2026
cd97fea
test(inference): add test for provider registered without synchronous…
jonwalstedt Sep 6, 2026
602f8ff
fix(inference): resolve workflow anonymization options once at startu…
jonwalstedt Sep 6, 2026
4959684
test(inference): add unit tests for sentinel-based capability plumbing
jonwalstedt Sep 6, 2026
fc810e9
fix(inference): decouple workflow encryptionKey from legacy saltPromi…
jonwalstedt Sep 7, 2026
09b8ebf
fix(inference): add maxLength bound to encryptionKey config schema
jonwalstedt Sep 7, 2026
d560f10
feat(inference): add workflowDrivenMinThreads config for operator-con…
jonwalstedt Sep 7, 2026
f2cc3cd
[Inference] Add anonymization contract, workflow steps, and inference…
jonwalstedt Sep 2, 2026
fabe3fc
[Inference] Fix tsconfig.json: add @kbn/workflows-management-plugin d…
jonwalstedt Sep 2, 2026
0e0d34b
[Workflows] Fix config comment: workflowDriven not workflow_driven
jonwalstedt Sep 8, 2026
2d2d524
[Inference] Integrate workflow anonymization pipeline into chat_compl…
jonwalstedt Sep 2, 2026
22a53e7
fix(inference): read sessionId/agentId from workflowAnonymization met…
jonwalstedt Sep 5, 2026
1b982dc
fix(inference): read agentId from metadata and sessionId from request…
jonwalstedt Sep 5, 2026
ccaf12f
fix(inference): decouple workflow encryptionKey from legacy saltPromi…
jonwalstedt Sep 7, 2026
fba545c
fix(inference): mark encryptionKey sensitive and add maxLength bound
jonwalstedt Sep 7, 2026
6eedf23
feat(inference): add workflowDrivenMinThreads config for operator-con…
jonwalstedt Sep 7, 2026
62a86e6
fix(inference): update test to use current sessionId/agentId API shape
jonwalstedt Sep 9, 2026
4df47cf
fix(inference): gate PiiRegexWorkerService on workflowDriven; add pii…
jonwalstedt Sep 9, 2026
2ed1bcb
refactor(inference): split workers config into separate anonymization…
jonwalstedt Sep 10, 2026
1925d1f
[Workflows] Update comment regarding the added supportedExecutionMode…
jonwalstedt Sep 10, 2026
0861c14
[Workflows] Fix type errors from ZodOptional/ZodDefault unwrapping
jonwalstedt Sep 10, 2026
cfd0c2f
[Workflows] Constrain template-string widening to Liquid expressions …
jonwalstedt Sep 10, 2026
419b168
[Workflows] Use safeExtend to handle refined connector paramsSchemas
jonwalstedt Sep 10, 2026
c3f9682
[Workflows] Address self-review: fix template-string widening correct…
jonwalstedt Sep 10, 2026
fbf8130
[Inference] Fix zero-length match test to use a* instead of a+
jonwalstedt Sep 9, 2026
066fccb
feat(inference): enforce RE2-only patterns on sync path in PiiRegexWo…
jonwalstedt Sep 10, 2026
5598fd0
fix(inference): rely on AbortSignal alone for task timeout, drop pool…
jonwalstedt Sep 10, 2026
f884fc6
fix(inference): address code-review findings in PII detection runtime
jonwalstedt Sep 10, 2026
500894a
fix(inference): address code review findings on PII detection runtime
jonwalstedt Sep 10, 2026
e6c8be3
fix(inference): correct allow_unsafe docstring in PiiRegexWorkerServi…
jonwalstedt Sep 10, 2026
d47b403
fix(inference): correct Piscina queue-saturation message match in Pii…
jonwalstedt Sep 10, 2026
7cabaec
fix(inference): rename misleading describe block in execute_regex_rul…
jonwalstedt Sep 10, 2026
d7fd5eb
fix(inference): narrow allow_unsafe blast radius to rule level in Pii…
jonwalstedt Sep 10, 2026
cd63215
docs(inference): update PiiDetectionFailureMode docstring for narrowe…
jonwalstedt Sep 10, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

/**
* Matches a *whole-value* Liquid template expression such as `"${{ event.messages }}"` — the
* only template form whose resolved value keeps its native type (array, object, number)
* instead of being stringified.
*
* The shape is deliberately narrow, and each restriction mirrors the runtime in
* `WorkflowTemplatingEngine.renderValueRecursively`:
*
* - **`$` is required.** The engine returns the raw evaluated value only for strings matching
* `startsWith('${{') && endsWith('}}')`. A bare `{{ expr }}` falls through to `renderString`
* and always comes back as a string, so it can never satisfy an array-typed param.
* - **No leading or trailing whitespace.** That runtime check does not trim.
* - **No inner `}}`.** `evaluateExpression` slices from the first `{{` to the last `}}`, so a
* concatenation like `"${{ a }}-${{ b }}"` would be parsed as the single invalid expression
* `a }}-${{ b` and throw at execution time.
*
* The invariant this encodes is one-directional: everything matched here is type-preserved at
* runtime. The engine itself accepts a superset, and those extra forms stringify or throw —
* which is exactly why callers should validate against this rather than re-deriving the shape.
* `template_expressions_runtime.test.ts` in `workflows_execution_engine` pins the invariant.
*/
export const WHOLE_VALUE_TEMPLATE_EXPRESSION_REGEX = /^\$\{\{(?:(?!\}\})[\s\S])*\}\}$/;

/**
* Upper bound on a whole-value Liquid template expression accepted where a connector param
* declares a non-string type. This is a sanity bound on a single YAML scalar, not a defence
* against large workflow payloads — the branch it widens (the param's own array/object schema)
* carries whatever bounds that connector declared, and this value does not change them.
*/
export const TEMPLATE_EXPRESSION_MAX_LENGTH = 500;

/**
* Returns true when `value` is a whole-value Liquid template expression whose resolved value
* keeps its native type at runtime. See {@link WHOLE_VALUE_TEMPLATE_EXPRESSION_REGEX}.
*/
export const isWholeValueTemplateExpression = (value: string): boolean =>
WHOLE_VALUE_TEMPLATE_EXPRESSION_REGEX.test(value);
1 change: 1 addition & 0 deletions src/platform/packages/shared/kbn-workflows/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ export * from './spec/deprecated_step_metadata';
export * from './types/latest';
export * from './types/utils';
export * from './common/constants';
export * from './common/template_expressions';
export * from './common/validate_step_names';
export * from './common/workflows_events';
export type * from './common/event_trigger_replay';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,39 @@ describe('builtInStepDefinitions', () => {
});
});

describe('supportedExecutionModes', () => {
// These steps depend on Task Manager — to suspend and resume the workflow, or to schedule
// work outside the current execution — so they can never complete inside a single HTTP
// request. `validateSyncWorkflow` refuses to run a workflow containing one of them in sync
// mode; if an annotation here is dropped, that guard silently stops firing and a synchronous
// request hangs instead of failing fast.
const ASYNC_ONLY_IDS = [
'wait',
'waitForInput',
'waitForApproval',
'workflow.execute',
'workflow.executeAsync',
];

it.each(ASYNC_ONLY_IDS)('"%s" is declared async-only', (id) => {
expect(getBuiltInStepDefinition(id)?.supportedExecutionModes).toEqual(['async']);
});

it('lists every async-only built-in — a new Task Manager-dependent step must be added here', () => {
const declaredAsyncOnly = builtInStepDefinitions
.filter((def) => def.supportedExecutionModes?.includes('sync') === false)
.map((def) => def.id);
expect(declaredAsyncOnly.sort()).toEqual([...ASYNC_ONLY_IDS].sort());
});

it.each(['console', 'data.set', 'if', 'foreach'])(
'"%s" leaves the field unset, so it stays runnable in both modes',
(id) => {
expect(getBuiltInStepDefinition(id)?.supportedExecutionModes).toBeUndefined();
}
);
});

describe('getBuiltInStepDefinition', () => {
it('returns the definition for a known id', () => {
const def = getBuiltInStepDefinition('if');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,7 @@ export const builtInStepDefinitions: BaseStepDefinition[] = [
},
{
id: 'wait',
supportedExecutionModes: ['async'],
label: 'Wait',
description: 'Pause execution for a specified duration',
category: StepCategory.FlowControl,
Expand Down Expand Up @@ -267,6 +268,7 @@ export const builtInStepDefinitions: BaseStepDefinition[] = [
},
{
id: 'workflow.execute',
supportedExecutionModes: ['async'],
label: 'Execute Workflow',
description: 'Execute another workflow and wait for it to complete',
category: StepCategory.FlowControl,
Expand All @@ -286,6 +288,7 @@ export const builtInStepDefinitions: BaseStepDefinition[] = [
},
{
id: 'waitForInput',
supportedExecutionModes: ['async'],
label: 'Wait For Input',
description: 'Pause execution until external input is provided (human-in-the-loop)',
category: StepCategory.FlowControl,
Expand Down Expand Up @@ -328,6 +331,7 @@ export const builtInStepDefinitions: BaseStepDefinition[] = [
},
{
id: 'waitForApproval',
supportedExecutionModes: ['async'],
label: 'Wait For Approval',
description: 'Pause execution until approval or rejection is received (human-in-the-loop)',
category: StepCategory.FlowControl,
Expand Down Expand Up @@ -357,6 +361,7 @@ export const builtInStepDefinitions: BaseStepDefinition[] = [
},
{
id: 'workflow.executeAsync',
supportedExecutionModes: ['async'],
label: 'Execute Workflow (Async)',
description: 'Start another workflow and continue without waiting for completion',
category: StepCategory.FlowControl,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
CONNECTOR_ID_MAX_LENGTH,
type ConnectorContractUnion,
generateYamlSchemaFromConnectors,
TEMPLATE_EXPRESSION_MAX_LENGTH,
} from '../..';

const BASE_WORKFLOW = {
Expand Down Expand Up @@ -218,4 +219,199 @@ describe('generateYamlSchemaFromConnectors', () => {
expect(elapsed).toBeLessThan(500);
});
});

describe('withTemplateStringSupport (array field widening)', () => {
const arrayConnector: ConnectorContractUnion = {
summary: 'Notifier',
description: null,
type: 'notify',
paramsSchema: z.object({
recipients: z.array(z.string()),
subject: z.string(),
}),
outputSchema: z.unknown(),
};

const parse = (withValue: unknown) =>
generateYamlSchemaFromConnectors([arrayConnector]).safeParse({
...BASE_WORKFLOW,
steps: [{ name: 'step', type: 'notify', with: withValue }],
});

it('accepts a ${{ expr }} template for an array param', () => {
expect(
parse({ recipients: '${{ workflow.inputs.recipients }}', subject: 'hi' }).success
).toBe(true);
});

// Only `${{ … }}` survives as an array at runtime: WorkflowTemplatingEngine returns the raw
// evaluated value for that form only, and renders everything else to a string. Accepting any
// of the shapes below would let YAML be saved that hands the connector a string — or, for the
// multi-expression forms, throws "The provided expression is invalid" mid-execution, because
// evaluateExpression slices from the first `{{` to the last `}}`.
it.each([
['a plain string', 'not-a-template'],
['a bare {{ expr }} template (renders to a string, not an array)', '{{ recipients }}'],
['text before the expression', 'prefix-${{ expr }}'],
['text after the expression', '${{ expr }}-suffix'],
['two concatenated expressions', '${{ a }}-${{ b }}'],
['an expression with literal text between two others', '${{ a }} literal ${{ b }}'],
['leading whitespace (the runtime check does not trim)', ' ${{ expr }}'],
['trailing whitespace (the runtime check does not trim)', '${{ expr }} '],
])('rejects %s for an array param', (_label, recipients) => {
expect(parse({ recipients, subject: 'hi' }).success).toBe(false);
});

it('rejects a template string exceeding TEMPLATE_EXPRESSION_MAX_LENGTH', () => {
const long = `\${{ ${'x'.repeat(TEMPLATE_EXPRESSION_MAX_LENGTH)} }}`;
expect(parse({ recipients: long, subject: 'hi' }).success).toBe(false);
});

it('does not widen non-array fields — string params remain string-only', () => {
expect(parse({ recipients: ['a@b.com'], subject: '{{ not-widened }}' }).success).toBe(true);
// A real Liquid expression is still valid as a string value, but a plain array is not
expect(parse({ recipients: ['a@b.com'], subject: ['array', 'not', 'ok'] }).success).toBe(
false
);
});

it('widens optional array params and preserves optionality', () => {
const connector: ConnectorContractUnion = {
summary: 'Opt',
description: null,
type: 'opt.step',
paramsSchema: z.object({ tags: z.array(z.string()).optional() }),
outputSchema: z.unknown(),
};
const schema = generateYamlSchemaFromConnectors([connector]);
// template string accepted
expect(
schema.safeParse({
...BASE_WORKFLOW,
steps: [{ name: 's', type: 'opt.step', with: { tags: '${{ workflow.inputs.tags }}' } }],
}).success
).toBe(true);
// omitting the optional field is still valid
expect(
schema.safeParse({
...BASE_WORKFLOW,
steps: [{ name: 's', type: 'opt.step', with: {} }],
}).success
).toBe(true);
});

it('widens default-wrapped array params without making them required', () => {
const connector: ConnectorContractUnion = {
summary: 'Def',
description: null,
type: 'def.step',
// Mirrors a real shipped connector: InferenceRerankParamsSchema declares
// `input: z.array(z.string()).default([])` as a top-level param.
paramsSchema: z.object({ tags: z.array(z.string()).default([]), query: z.string() }),
outputSchema: z.unknown(),
};
const schema = generateYamlSchemaFromConnectors([connector]);
expect(
schema.safeParse({
...BASE_WORKFLOW,
steps: [
{
name: 's',
type: 'def.step',
with: { tags: '${{ workflow.inputs.tags }}', query: 'q' },
},
],
}).success
).toBe(true);

// Widening must not strip `.default()`. If it did, every existing workflow that omits a
// defaulted array param would stop validating — including on update, since this schema
// gates persistence and not just editor feedback.
const omitted = schema.safeParse({
...BASE_WORKFLOW,
steps: [{ name: 's', type: 'def.step', with: { query: 'q' } }],
});
expect(omitted.success).toBe(true);
expect(omitted.data).toMatchObject({ steps: [{ with: { tags: [] } }] });
});

it('widens array params wrapped in both .optional() and .default()', () => {
const connector: ConnectorContractUnion = {
summary: 'Both',
description: null,
type: 'both.step',
paramsSchema: z.object({ tags: z.array(z.string()).optional().default([]) }),
outputSchema: z.unknown(),
};
const schema = generateYamlSchemaFromConnectors([connector]);
// Stacked wrappers must still be unwrapped down to the array, otherwise the field is
// silently skipped and the template string is reported as a type error.
expect(
schema.safeParse({
...BASE_WORKFLOW,
steps: [{ name: 's', type: 'both.step', with: { tags: '${{ workflow.inputs.tags }}' } }],
}).success
).toBe(true);
});

it('preserves the object unknownKeys policy of the original paramsSchema', () => {
// A strict paramsSchema should still reject unknown keys after widening.
const connector: ConnectorContractUnion = {
summary: 'Strict',
description: null,
type: 'strict.step',
paramsSchema: z.strictObject({ ids: z.array(z.string()) }),
outputSchema: z.unknown(),
};
const schema = generateYamlSchemaFromConnectors([connector]);
expect(
schema.safeParse({
...BASE_WORKFLOW,
steps: [
{
name: 's',
type: 'strict.step',
with: { ids: '${{ workflow.inputs.ids }}', unknown_key: 'bad' },
},
],
}).success
).toBe(false);
});

it('does not throw for a connector whose paramsSchema has object-level refinements', () => {
const connector: ConnectorContractUnion = {
summary: 'Refined',
description: null,
type: 'refined.step',
paramsSchema: z
.object({ ids: z.array(z.string()), name: z.string() })
.refine((v) => v.ids.length > 0, 'ids must not be empty'),
outputSchema: z.unknown(),
};
// Schema construction must not throw even though paramsSchema has a refinement.
expect(() => generateYamlSchemaFromConnectors([connector])).not.toThrow();

const schema = generateYamlSchemaFromConnectors([connector]);
// Template string still accepted for the array field.
expect(
schema.safeParse({
...BASE_WORKFLOW,
steps: [
{
name: 's',
type: 'refined.step',
with: { ids: '${{ workflow.inputs.ids }}', name: 'x' },
},
],
}).success
).toBe(true);
// The object-level refinement is preserved: an empty ids array fails.
expect(
schema.safeParse({
...BASE_WORKFLOW,
steps: [{ name: 's', type: 'refined.step', with: { ids: [], name: 'x' } }],
}).success
).toBe(false);
});
});
});
Loading
Loading