From b1356c7004c597f7beb96bf47cac62c8a3b72346 Mon Sep 17 00:00:00 2001 From: Quentin Couvelaire Date: Thu, 30 Oct 2025 10:17:13 -0400 Subject: [PATCH 1/8] add timezone and locale to streamlang date action types --- .../packages/shared/kbn-streamlang/types/processors/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/platform/packages/shared/kbn-streamlang/types/processors/index.ts b/x-pack/platform/packages/shared/kbn-streamlang/types/processors/index.ts index 87373bf4c66ce..2392583218415 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/types/processors/index.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/types/processors/index.ts @@ -121,6 +121,8 @@ export interface DateProcessor extends ProcessorBaseWithWhere { to?: string; formats: string[]; output_format?: string; + timezone?: string; + locale?: string; } export const dateProcessorSchema = processorBaseWithWhereSchema.extend({ @@ -129,6 +131,8 @@ export const dateProcessorSchema = processorBaseWithWhereSchema.extend({ to: z.optional(StreamlangTargetField), formats: z.array(NonEmptyString), output_format: z.optional(NonEmptyString), + timezone: z.optional(NonEmptyString), + locale: z.optional(NonEmptyString), }) satisfies z.Schema; /** From 2da38e9d4e5846cfdcf2e0c46864604d639e56e4 Mon Sep 17 00:00:00 2001 From: Quentin Couvelaire Date: Thu, 30 Oct 2025 14:46:34 -0400 Subject: [PATCH 2/8] extend ingest pipeline and esql transpilers --- .../test/scout/api/tests/esql/date.spec.ts | 24 +++++ .../api/tests/ingest_pipeline/date.spec.ts | 91 +++++++++++++++++++ .../src/transpilers/esql/processors/date.ts | 22 ++++- 3 files changed, 136 insertions(+), 1 deletion(-) diff --git a/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/esql/date.spec.ts b/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/esql/date.spec.ts index 84683fde602ab..45ad8ba506e58 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/esql/date.spec.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/esql/date.spec.ts @@ -185,4 +185,28 @@ apiTest.describe('Streamlang to ES|QL - Date Processor', () => { ); } ); + + apiTest( + 'should parse a date with locale and timezone', + { tag: ['@ess', '@svlOblt'] }, + async ({ testBed, esql }) => { + const indexName = 'stream-e2e-test-locale-timezone'; + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'date', + from: 'log.date', + formats: ['dd MMMM yyyy'], + locale: 'fr', + timezone: 'Europe/Paris', + } as DateProcessor, + ], + }; + const { query } = transpile(streamlangDSL); + const docs = [{ log: { date: '10 septembre 2025' } }]; + await testBed.ingest(indexName, docs); + const esqlResult = await esql.queryOnIndex(indexName, query); + expect(esqlResult.documents[0]['@timestamp']).toBe('2025-09-09T22:00:00.000Z'); + } + ); }); diff --git a/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/ingest_pipeline/date.spec.ts b/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/ingest_pipeline/date.spec.ts index ce1bfbff4ede9..89b3ac01b5291 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/ingest_pipeline/date.spec.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/ingest_pipeline/date.spec.ts @@ -207,4 +207,95 @@ apiTest.describe('Streamlang to Ingest Pipeline - Date Processor', () => { }).toThrow('Mustache template syntax {{ }} or {{{ }}} is not allowed'); }); }); + + apiTest( + 'should parse a date with a specific locale and timezone', + { tag: ['@ess', '@svlOblt'] }, + async ({ testBed }) => { + const indexName = 'stream-e2e-test-locale-timezone'; + + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'date', + from: 'event.created', + to: 'event.created_date', + formats: ['dd MMMM yyyy'], + locale: 'fr', + timezone: 'Europe/Paris', + } as DateProcessor, + ], + }; + + const { processors } = transpile(streamlangDSL); + + const docs = [{ event: { created: '10 septembre 2025' } }]; + await testBed.ingest(indexName, docs, processors); + + const ingestedDocs = await testBed.getDocs(indexName); + + expect(ingestedDocs).toHaveLength(1); + expect(ingestedDocs[0]).toHaveProperty('event.created_date', '2025-09-10T00:00:00.000+02:00'); + } + ); + + apiTest( + 'should fail when locale is not valid', + { tag: ['@ess', '@svlOblt'] }, + async ({ testBed }) => { + const indexName = 'stream-e2e-test-locale-fail'; + + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'date', + from: 'event.created', + to: 'event.created_date', + formats: ['dd MMMM yyyy'], + locale: 'french', + timezone: 'Europe/Paris', + } as DateProcessor, + ], + }; + + const { processors } = transpile(streamlangDSL); + + const docs = [{ event: { created: '10 septembre 2025' } }]; + await testBed.ingest(indexName, docs, processors); + + const { errors } = await testBed.ingest(indexName, docs, processors); + expect(errors[0].reason).toContain('unable to parse date'); + expect(errors[0].caused_by?.reason).toContain('Unknown language: french'); + } + ); + + apiTest( + 'should fail when timezone is not valid', + { tag: ['@ess', '@svlOblt'] }, + async ({ testBed }) => { + const indexName = 'stream-e2e-test-timezone-fail'; + + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'date', + from: 'event.created', + to: 'event.created_date', + formats: ['dd MMMM yyyy'], + locale: 'fr', + timezone: 'France', + } as DateProcessor, + ], + }; + + const { processors } = transpile(streamlangDSL); + + const docs = [{ event: { created: '10 septembre 2025' } }]; + await testBed.ingest(indexName, docs, processors); + + const { errors } = await testBed.ingest(indexName, docs, processors); + expect(errors[0].reason).toContain('unable to parse date'); + expect(errors[0].caused_by?.reason).toContain('Unknown time-zone ID: France'); + } + ); }); diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/date.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/date.ts index 2ef559465c120..53104587d4f11 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/date.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/date.ts @@ -7,6 +7,7 @@ import { Builder } from '@kbn/esql-ast'; import type { ESQLAstCommand } from '@kbn/esql-ast'; +import type { ESQLMapEntry } from '@kbn/esql-ast/src/types'; import type { CommonDatePreset } from '../../../../types/formats'; import type { DateProcessor } from '../../../../types/processors'; import { conditionToESQLAst } from '../condition_to_esql'; @@ -17,6 +18,8 @@ export function convertDateProcessorToESQL(processor: DateProcessor): ESQLAstCom to, formats, // eslint-disable-next-line @typescript-eslint/naming-convention output_format, + timezone, + locale, where, } = processor; const fromColumn = Builder.expression.column(from); @@ -24,6 +27,19 @@ export function convertDateProcessorToESQL(processor: DateProcessor): ESQLAstCom const fromAsString = Builder.expression.func.call('TO_STRING', [fromColumn]); const targetDateField = to ?? '@timestamp'; // As with Ingest Date Processor, default to @timestamp const toColumn = Builder.expression.column(targetDateField); + // TODO - need to use timezone and locale in DATE_PARSE, can probably clean up this logic + const dateParseNamedParamsEntries: ESQLMapEntry[] = []; + if (timezone !== undefined) + dateParseNamedParamsEntries.push( + Builder.expression.entry('time_zone', Builder.expression.literal.string(timezone)) + ); + if (locale !== undefined) + dateParseNamedParamsEntries.push( + Builder.expression.entry('locale', Builder.expression.literal.string(locale)) + ); + const dateParseNamedParams = Builder.expression.map({ + entries: dateParseNamedParamsEntries, + }); const resolvedInputFormats = formats.map((f) => resolveCommonDatePresetsForESQL(f as CommonDatePreset) @@ -33,7 +49,11 @@ export function convertDateProcessorToESQL(processor: DateProcessor): ESQLAstCom : undefined; const dateParseExpressions = resolvedInputFormats.map((f) => - Builder.expression.func.call('DATE_PARSE', [Builder.expression.literal.string(f), fromAsString]) + Builder.expression.func.call('DATE_PARSE', [ + Builder.expression.literal.string(f), + fromAsString, + dateParseNamedParams, + ]) ); const coalesceDateParse = Builder.expression.func.call('COALESCE', dateParseExpressions); From 2ec10686a03985524c85622ef67d5d09a1a6f03a Mon Sep 17 00:00:00 2001 From: Quentin Couvelaire Date: Fri, 31 Oct 2025 08:43:53 -0400 Subject: [PATCH 3/8] add timezone and locale inputs to date processor ui --- .../steps/blocks/action/date/index.tsx | 9 ++++++++- .../data_management/stream_detail_enrichment/utils.ts | 4 +++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/date/index.tsx b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/date/index.tsx index 0fde7c779dcc1..03b11c89868a6 100644 --- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/date/index.tsx +++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/date/index.tsx @@ -19,7 +19,12 @@ import { ProcessorFieldSelector } from '../processor_field_selector'; import { FieldsAccordion } from '../optional_fields_accordion'; import { ProcessorConditionEditor } from '../processor_condition_editor'; import { IgnoreFailureToggle } from '../ignore_toggles'; -import { DateTargetField, DateOutputFormatField } from './date_optional_fields'; +import { + DateTargetField, + DateOutputFormatField, + DateTimezoneField, + DateLocaleField, +} from './date_optional_fields'; import { DateFormatsField } from './date_formats_field'; import { selectPreviewRecords } from '../../../../state_management/simulation_state_machine/selectors'; @@ -148,6 +153,8 @@ export const DateProcessorForm = () => { + + diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/utils.ts b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/utils.ts index b55319dd2e008..52192dff388a1 100644 --- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/utils.ts +++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/utils.ts @@ -311,7 +311,7 @@ export const convertFormStateToProcessor = ( } if (formState.action === 'date') { - const { from, formats, ignore_failure, to, output_format } = formState; + const { from, formats, ignore_failure, to, output_format, timezone, locale } = formState; return { processorDefinition: { @@ -322,6 +322,8 @@ export const convertFormStateToProcessor = ( ignore_failure, to: isEmpty(to) ? undefined : to, output_format: isEmpty(output_format) ? undefined : output_format, + timezone: isEmpty(timezone) ? undefined : timezone, + locale: isEmpty(locale) ? undefined : locale, }, }; } From d95ff70582751fc52f64f68931645f2155ed1c82 Mon Sep 17 00:00:00 2001 From: Quentin Couvelaire Date: Mon, 3 Nov 2025 08:15:37 -0500 Subject: [PATCH 4/8] cleanup date parse object creation logic --- .../src/transpilers/esql/processors/date.ts | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/date.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/date.ts index 53104587d4f11..5bba07e9e0571 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/date.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/date.ts @@ -12,6 +12,19 @@ import type { CommonDatePreset } from '../../../../types/formats'; import type { DateProcessor } from '../../../../types/processors'; import { conditionToESQLAst } from '../condition_to_esql'; +const buildDateParseNamedParamEntries = (timezone?: string, locale?: string): ESQLMapEntry[] => { + const dateParseNamedParamsEntries: ESQLMapEntry[] = []; + if (timezone !== undefined) + dateParseNamedParamsEntries.push( + Builder.expression.entry('time_zone', Builder.expression.literal.string(timezone)) + ); + if (locale !== undefined) + dateParseNamedParamsEntries.push( + Builder.expression.entry('locale', Builder.expression.literal.string(locale)) + ); + return dateParseNamedParamsEntries; +}; + export function convertDateProcessorToESQL(processor: DateProcessor): ESQLAstCommand[] { const { from, @@ -27,18 +40,8 @@ export function convertDateProcessorToESQL(processor: DateProcessor): ESQLAstCom const fromAsString = Builder.expression.func.call('TO_STRING', [fromColumn]); const targetDateField = to ?? '@timestamp'; // As with Ingest Date Processor, default to @timestamp const toColumn = Builder.expression.column(targetDateField); - // TODO - need to use timezone and locale in DATE_PARSE, can probably clean up this logic - const dateParseNamedParamsEntries: ESQLMapEntry[] = []; - if (timezone !== undefined) - dateParseNamedParamsEntries.push( - Builder.expression.entry('time_zone', Builder.expression.literal.string(timezone)) - ); - if (locale !== undefined) - dateParseNamedParamsEntries.push( - Builder.expression.entry('locale', Builder.expression.literal.string(locale)) - ); const dateParseNamedParams = Builder.expression.map({ - entries: dateParseNamedParamsEntries, + entries: buildDateParseNamedParamEntries(timezone, locale), }); const resolvedInputFormats = formats.map((f) => From 768cce8f7b9d4cfba75aa3cb4508a0473f115e24 Mon Sep 17 00:00:00 2001 From: Quentin Couvelaire Date: Mon, 3 Nov 2025 11:41:34 -0500 Subject: [PATCH 5/8] add ui integration test for locale and timezone date processor scenario --- .../ui/fixtures/page_objects/streams_app.ts | 28 ++++++++++++ .../processing_simulation_preview.spec.ts | 43 +++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/x-pack/platform/plugins/shared/streams_app/test/scout/ui/fixtures/page_objects/streams_app.ts b/x-pack/platform/plugins/shared/streams_app/test/scout/ui/fixtures/page_objects/streams_app.ts index d450119a0e025..5a79e73432358 100644 --- a/x-pack/platform/plugins/shared/streams_app/test/scout/ui/fixtures/page_objects/streams_app.ts +++ b/x-pack/platform/plugins/shared/streams_app/test/scout/ui/fixtures/page_objects/streams_app.ts @@ -511,6 +511,34 @@ export class StreamsApp { await this.page.getByTestId('streamsAppPatternExpression').getByRole('textbox').fill(value); } + async fillDateProcessorSourceFieldInput(value: string) { + await this.page.getByLabel('Source Field').fill(value); + } + + async fillDateProcessorFormatInput(value: string) { + await this.page.getByPlaceholder('Type and then hit "Enter"').fill(value); + } + + async fillDateProcessorTargetFieldInput(value: string) { + await this.page.getByLabel('Target field').fill(value); + } + + async fillDateProcessorTimezoneInput(value: string) { + await this.page.getByLabel('Timezone').fill(value); + } + + async fillDateProcessorLocaleInput(value: string) { + await this.page.getByLabel('Locale').fill(value); + } + + async fillDateProcessorOutputFormatInput(value: string) { + await this.page.getByLabel('Output format').fill(value); + } + + async clickDateProcessorAdvancedSettings() { + await this.page.getByRole('button', { name: 'Advanced settings' }).click(); + } + async fillCustomSamplesEditor(value: string) { // Clean previous content await this.page.getByTestId('streamsAppCustomSamplesDataSourceEditor').click(); diff --git a/x-pack/platform/plugins/shared/streams_app/test/scout/ui/tests/data_management/data_processing/processing_simulation_preview.spec.ts b/x-pack/platform/plugins/shared/streams_app/test/scout/ui/tests/data_management/data_processing/processing_simulation_preview.spec.ts index 3c9d04771b3a8..be098084e40e5 100644 --- a/x-pack/platform/plugins/shared/streams_app/test/scout/ui/tests/data_management/data_processing/processing_simulation_preview.spec.ts +++ b/x-pack/platform/plugins/shared/streams_app/test/scout/ui/tests/data_management/data_processing/processing_simulation_preview.spec.ts @@ -188,4 +188,47 @@ test.describe('Stream data processing - simulation preview', { tag: ['@ess', '@s }); } }); + + test('should update the simulation preview with processed dates from another locale/timezone', async ({ + page, + pageObjects, + }) => { + const sourceField = 'french_date'; + const targetField = 'processed_french_date'; + + await pageObjects.streams.clickAddProcessor(); + await pageObjects.streams.selectProcessorType('set'); + await pageObjects.streams.fillProcessorFieldInput(sourceField, { isCustomValue: true }); + await page.locator('input[name="value"]').fill('08 avril 1999'); + await pageObjects.streams.clickSaveProcessor(); + + await pageObjects.streams.clickAddProcessor(); + await pageObjects.streams.selectProcessorType('set'); + await pageObjects.streams.fillProcessorFieldInput(targetField, { + isCustomValue: true, + }); + await page.locator('input[name="value"]').fill('null'); + await pageObjects.streams.clickSaveProcessor(); + + await pageObjects.streams.clickAddProcessor(); + await pageObjects.streams.selectProcessorType('date'); + await pageObjects.streams.fillDateProcessorSourceFieldInput(sourceField); + await pageObjects.streams.fillDateProcessorFormatInput('dd MMMM yyyy'); + await pageObjects.streams.clickDateProcessorAdvancedSettings(); + await pageObjects.streams.fillDateProcessorTargetFieldInput(targetField); + await pageObjects.streams.fillDateProcessorTimezoneInput('Europe/Paris'); + await pageObjects.streams.fillDateProcessorLocaleInput('fr'); + await pageObjects.streams.fillDateProcessorOutputFormatInput('yyyy-MM-dd'); + await pageObjects.streams.clickSaveProcessor(); + + const updatedRows = await pageObjects.streams.getPreviewTableRows(); + expect(updatedRows.length).toBeGreaterThan(0); + for (let rowIndex = 0; rowIndex < updatedRows.length; rowIndex++) { + await pageObjects.streams.expectCellValueContains({ + columnName: targetField, + rowIndex, + value: '1999-04-08', + }); + } + }); }); From 5f683a272d5b93ee197db29a9788b636029546fd Mon Sep 17 00:00:00 2001 From: Quentin Couvelaire Date: Tue, 4 Nov 2025 09:45:35 -0500 Subject: [PATCH 6/8] fix failing tests, remove incorrect ux text from inputs --- .../tests/cross_compatibility/date.spec.ts | 32 ++++++++++++++ .../test/scout/api/tests/esql/date.spec.ts | 24 ---------- .../src/transpilers/esql/processors/date.ts | 44 ++++++++++++++----- .../action/date/date_optional_fields.tsx | 4 +- .../ui/fixtures/page_objects/streams_app.ts | 2 +- .../processing_simulation_preview.spec.ts | 16 +++---- 6 files changed, 75 insertions(+), 47 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/cross_compatibility/date.spec.ts b/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/cross_compatibility/date.spec.ts index 588bd004061c3..9fed7c0f5085e 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/cross_compatibility/date.spec.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/cross_compatibility/date.spec.ts @@ -227,4 +227,36 @@ apiTest.describe('Cross-compatibility - Date Processor', () => { expect(esqlResult.documentsOrdered[0]['log.time']).toBe('01-01-2025'); // Unchanged } ); + + apiTest( + 'should parse a date with locale and timezone', + { tag: ['@ess', '@svlOblt'] }, + async ({ testBed, esql }) => { + const streamlangDSL: StreamlangDSL = { + steps: [ + { + action: 'date', + from: 'log.date', + formats: ['dd MMMM yyyy'], + locale: 'fr', + timezone: 'UTC', + output_format: 'yyyy-MM-dd', + } as DateProcessor, + ], + }; + + const { processors } = transpileIngestPipeline(streamlangDSL); + const { query } = transpileEsql(streamlangDSL); + + const docs = [{ log: { date: '08 avril 1999' } }]; + await testBed.ingest('ingest-date-timezone-locale-format', docs, processors); + const ingestResult = await testBed.getDocsOrdered('ingest-date-timezone-locale-format'); + + await testBed.ingest('esql-date-timezone-locale-format', docs); + const esqlResult = await esql.queryOnIndex('esql-date-timezone-locale-format', query); + + expect(ingestResult[0]['@timestamp']).toBe('1999-04-08'); + expect(esqlResult.documentsOrdered[0]['@timestamp']).toBe('1999-04-08'); + } + ); }); diff --git a/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/esql/date.spec.ts b/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/esql/date.spec.ts index 45ad8ba506e58..84683fde602ab 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/esql/date.spec.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang-tests/test/scout/api/tests/esql/date.spec.ts @@ -185,28 +185,4 @@ apiTest.describe('Streamlang to ES|QL - Date Processor', () => { ); } ); - - apiTest( - 'should parse a date with locale and timezone', - { tag: ['@ess', '@svlOblt'] }, - async ({ testBed, esql }) => { - const indexName = 'stream-e2e-test-locale-timezone'; - const streamlangDSL: StreamlangDSL = { - steps: [ - { - action: 'date', - from: 'log.date', - formats: ['dd MMMM yyyy'], - locale: 'fr', - timezone: 'Europe/Paris', - } as DateProcessor, - ], - }; - const { query } = transpile(streamlangDSL); - const docs = [{ log: { date: '10 septembre 2025' } }]; - await testBed.ingest(indexName, docs); - const esqlResult = await esql.queryOnIndex(indexName, query); - expect(esqlResult.documents[0]['@timestamp']).toBe('2025-09-09T22:00:00.000Z'); - } - ); }); diff --git a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/date.ts b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/date.ts index 5bba07e9e0571..fb49f1208077a 100644 --- a/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/date.ts +++ b/x-pack/platform/packages/shared/kbn-streamlang/src/transpilers/esql/processors/date.ts @@ -7,24 +7,48 @@ import { Builder } from '@kbn/esql-ast'; import type { ESQLAstCommand } from '@kbn/esql-ast'; -import type { ESQLMapEntry } from '@kbn/esql-ast/src/types'; +import type { ESQLFunction, ESQLMapEntry, FunctionSubtype } from '@kbn/esql-ast/src/types'; import type { CommonDatePreset } from '../../../../types/formats'; import type { DateProcessor } from '../../../../types/processors'; import { conditionToESQLAst } from '../condition_to_esql'; const buildDateParseNamedParamEntries = (timezone?: string, locale?: string): ESQLMapEntry[] => { const dateParseNamedParamsEntries: ESQLMapEntry[] = []; - if (timezone !== undefined) + if (timezone) dateParseNamedParamsEntries.push( Builder.expression.entry('time_zone', Builder.expression.literal.string(timezone)) ); - if (locale !== undefined) + if (locale) dateParseNamedParamsEntries.push( Builder.expression.entry('locale', Builder.expression.literal.string(locale)) ); return dateParseNamedParamsEntries; }; +const buildDateParseExpressions = ( + inputFormats: string[], + fromAsString: ESQLFunction, + timezone?: string, + locale?: string +): ESQLFunction[] => { + if (timezone || locale) { + const dateParseNamedParams = Builder.expression.map({ + entries: buildDateParseNamedParamEntries(timezone, locale), + }); + return inputFormats.map((f) => + Builder.expression.func.call('DATE_PARSE', [ + Builder.expression.literal.string(f), + fromAsString, + dateParseNamedParams, + ]) + ); + } + + return inputFormats.map((f) => + Builder.expression.func.call('DATE_PARSE', [Builder.expression.literal.string(f), fromAsString]) + ); +}; + export function convertDateProcessorToESQL(processor: DateProcessor): ESQLAstCommand[] { const { from, @@ -40,9 +64,6 @@ export function convertDateProcessorToESQL(processor: DateProcessor): ESQLAstCom const fromAsString = Builder.expression.func.call('TO_STRING', [fromColumn]); const targetDateField = to ?? '@timestamp'; // As with Ingest Date Processor, default to @timestamp const toColumn = Builder.expression.column(targetDateField); - const dateParseNamedParams = Builder.expression.map({ - entries: buildDateParseNamedParamEntries(timezone, locale), - }); const resolvedInputFormats = formats.map((f) => resolveCommonDatePresetsForESQL(f as CommonDatePreset) @@ -51,12 +72,11 @@ export function convertDateProcessorToESQL(processor: DateProcessor): ESQLAstCom ? resolveCommonDatePresetsForESQL(output_format as CommonDatePreset) : undefined; - const dateParseExpressions = resolvedInputFormats.map((f) => - Builder.expression.func.call('DATE_PARSE', [ - Builder.expression.literal.string(f), - fromAsString, - dateParseNamedParams, - ]) + const dateParseExpressions = buildDateParseExpressions( + resolvedInputFormats, + fromAsString, + timezone, + locale ); const coalesceDateParse = Builder.expression.func.call('COALESCE', dateParseExpressions); diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/date/date_optional_fields.tsx b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/date/date_optional_fields.tsx index 7793b38d0a585..530fb392f36d2 100644 --- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/date/date_optional_fields.tsx +++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/steps/blocks/action/date/date_optional_fields.tsx @@ -48,7 +48,7 @@ export const DateTimezoneField = () => { helpText={ UTC }} /> } @@ -72,7 +72,7 @@ export const DateLocaleField = () => { helpText={ ENGLISH }} /> } diff --git a/x-pack/platform/plugins/shared/streams_app/test/scout/ui/fixtures/page_objects/streams_app.ts b/x-pack/platform/plugins/shared/streams_app/test/scout/ui/fixtures/page_objects/streams_app.ts index 5a79e73432358..8fa0073bfaee1 100644 --- a/x-pack/platform/plugins/shared/streams_app/test/scout/ui/fixtures/page_objects/streams_app.ts +++ b/x-pack/platform/plugins/shared/streams_app/test/scout/ui/fixtures/page_objects/streams_app.ts @@ -489,7 +489,7 @@ export class StreamsApp { await expect(this.getModal()).toBeHidden(); } - async selectProcessorType(value: ProcessorType) { + async selectProcessorType(value: Capitalize) { await this.page.getByTestId('streamsAppProcessorTypeSelector').click(); await this.page.getByRole('dialog').getByRole('option').getByText(value).click(); } diff --git a/x-pack/platform/plugins/shared/streams_app/test/scout/ui/tests/data_management/data_processing/processing_simulation_preview.spec.ts b/x-pack/platform/plugins/shared/streams_app/test/scout/ui/tests/data_management/data_processing/processing_simulation_preview.spec.ts index be098084e40e5..0a595ec2cb81f 100644 --- a/x-pack/platform/plugins/shared/streams_app/test/scout/ui/tests/data_management/data_processing/processing_simulation_preview.spec.ts +++ b/x-pack/platform/plugins/shared/streams_app/test/scout/ui/tests/data_management/data_processing/processing_simulation_preview.spec.ts @@ -53,7 +53,7 @@ test.describe('Stream data processing - simulation preview', { tag: ['@ess', '@s pageObjects, }) => { await pageObjects.streams.clickAddProcessor(); - await pageObjects.streams.selectProcessorType('rename'); + await pageObjects.streams.selectProcessorType('Rename'); await pageObjects.streams.fillProcessorFieldInput('message'); await page.locator('input[name="to"]').fill('message'); @@ -74,7 +74,7 @@ test.describe('Stream data processing - simulation preview', { tag: ['@ess', '@s pageObjects, }) => { await pageObjects.streams.clickAddProcessor(); - await pageObjects.streams.selectProcessorType('rename'); + await pageObjects.streams.selectProcessorType('Rename'); await pageObjects.streams.fillProcessorFieldInput('message'); await page.locator('input[name="to"]').fill('message'); @@ -107,7 +107,7 @@ test.describe('Stream data processing - simulation preview', { tag: ['@ess', '@s pageObjects, }) => { await pageObjects.streams.clickAddProcessor(); - await pageObjects.streams.selectProcessorType('rename'); + await pageObjects.streams.selectProcessorType('Rename'); await pageObjects.streams.fillProcessorFieldInput('message'); await page.locator('input[name="to"]').fill('message'); @@ -142,13 +142,13 @@ test.describe('Stream data processing - simulation preview', { tag: ['@ess', '@s pageObjects, }) => { await pageObjects.streams.clickAddProcessor(); - await pageObjects.streams.selectProcessorType('rename'); + await pageObjects.streams.selectProcessorType('Rename'); await pageObjects.streams.fillProcessorFieldInput('message'); await page.locator('input[name="to"]').fill('message'); await pageObjects.streams.clickSaveProcessor(); await pageObjects.streams.clickAddProcessor(); - await pageObjects.streams.selectProcessorType('set'); + await pageObjects.streams.selectProcessorType('Set'); await pageObjects.streams.fillProcessorFieldInput('custom_threshold', { isCustomValue: true }); await page.locator('input[name="value"]').fill('1024'); await pageObjects.streams.clickSaveProcessor(); @@ -197,13 +197,13 @@ test.describe('Stream data processing - simulation preview', { tag: ['@ess', '@s const targetField = 'processed_french_date'; await pageObjects.streams.clickAddProcessor(); - await pageObjects.streams.selectProcessorType('set'); + await pageObjects.streams.selectProcessorType('Set'); await pageObjects.streams.fillProcessorFieldInput(sourceField, { isCustomValue: true }); await page.locator('input[name="value"]').fill('08 avril 1999'); await pageObjects.streams.clickSaveProcessor(); await pageObjects.streams.clickAddProcessor(); - await pageObjects.streams.selectProcessorType('set'); + await pageObjects.streams.selectProcessorType('Set'); await pageObjects.streams.fillProcessorFieldInput(targetField, { isCustomValue: true, }); @@ -211,7 +211,7 @@ test.describe('Stream data processing - simulation preview', { tag: ['@ess', '@s await pageObjects.streams.clickSaveProcessor(); await pageObjects.streams.clickAddProcessor(); - await pageObjects.streams.selectProcessorType('date'); + await pageObjects.streams.selectProcessorType('Date'); await pageObjects.streams.fillDateProcessorSourceFieldInput(sourceField); await pageObjects.streams.fillDateProcessorFormatInput('dd MMMM yyyy'); await pageObjects.streams.clickDateProcessorAdvancedSettings(); From 0d6b38a4a74390c3496ba5bf9b290ff6546a4eb9 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 4 Nov 2025 15:12:28 +0000 Subject: [PATCH 7/8] Changes from node scripts/capture_oas_snapshot --include-path /api/status --include-path /api/alerting/rule/ --include-path /api/alerting/rules --include-path /api/actions --include-path /api/security/role --include-path /api/spaces --include-path /api/streams --include-path /api/fleet --include-path /api/saved_objects/_import --include-path /api/saved_objects/_export --include-path /api/maintenance_window --include-path /api/agent_builder --update --- oas_docs/bundle.json | 48 +++++++++++++++++++++++++++++++++ oas_docs/bundle.serverless.json | 48 +++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/oas_docs/bundle.json b/oas_docs/bundle.json index 779d6f643a893..5676f359c578a 100644 --- a/oas_docs/bundle.json +++ b/oas_docs/bundle.json @@ -52804,10 +52804,18 @@ "ignore_failure": { "type": "boolean" }, + "locale": { + "minLength": 1, + "type": "string" + }, "output_format": { "minLength": 1, "type": "string" }, + "timezone": { + "minLength": 1, + "type": "string" + }, "to": { "minLength": 1, "type": "string" @@ -57096,10 +57104,18 @@ "ignore_failure": { "type": "boolean" }, + "locale": { + "minLength": 1, + "type": "string" + }, "output_format": { "minLength": 1, "type": "string" }, + "timezone": { + "minLength": 1, + "type": "string" + }, "to": { "minLength": 1, "type": "string" @@ -60981,10 +60997,18 @@ "ignore_failure": { "type": "boolean" }, + "locale": { + "minLength": 1, + "type": "string" + }, "output_format": { "minLength": 1, "type": "string" }, + "timezone": { + "minLength": 1, + "type": "string" + }, "to": { "minLength": 1, "type": "string" @@ -64969,10 +64993,18 @@ "ignore_failure": { "type": "boolean" }, + "locale": { + "minLength": 1, + "type": "string" + }, "output_format": { "minLength": 1, "type": "string" }, + "timezone": { + "minLength": 1, + "type": "string" + }, "to": { "minLength": 1, "type": "string" @@ -69438,10 +69470,18 @@ "ignore_failure": { "type": "boolean" }, + "locale": { + "minLength": 1, + "type": "string" + }, "output_format": { "minLength": 1, "type": "string" }, + "timezone": { + "minLength": 1, + "type": "string" + }, "to": { "minLength": 1, "type": "string" @@ -73295,10 +73335,18 @@ "ignore_failure": { "type": "boolean" }, + "locale": { + "minLength": 1, + "type": "string" + }, "output_format": { "minLength": 1, "type": "string" }, + "timezone": { + "minLength": 1, + "type": "string" + }, "to": { "minLength": 1, "type": "string" diff --git a/oas_docs/bundle.serverless.json b/oas_docs/bundle.serverless.json index df11f5add8a83..9f1d5bbdaf108 100644 --- a/oas_docs/bundle.serverless.json +++ b/oas_docs/bundle.serverless.json @@ -51913,10 +51913,18 @@ "ignore_failure": { "type": "boolean" }, + "locale": { + "minLength": 1, + "type": "string" + }, "output_format": { "minLength": 1, "type": "string" }, + "timezone": { + "minLength": 1, + "type": "string" + }, "to": { "minLength": 1, "type": "string" @@ -56205,10 +56213,18 @@ "ignore_failure": { "type": "boolean" }, + "locale": { + "minLength": 1, + "type": "string" + }, "output_format": { "minLength": 1, "type": "string" }, + "timezone": { + "minLength": 1, + "type": "string" + }, "to": { "minLength": 1, "type": "string" @@ -60090,10 +60106,18 @@ "ignore_failure": { "type": "boolean" }, + "locale": { + "minLength": 1, + "type": "string" + }, "output_format": { "minLength": 1, "type": "string" }, + "timezone": { + "minLength": 1, + "type": "string" + }, "to": { "minLength": 1, "type": "string" @@ -64078,10 +64102,18 @@ "ignore_failure": { "type": "boolean" }, + "locale": { + "minLength": 1, + "type": "string" + }, "output_format": { "minLength": 1, "type": "string" }, + "timezone": { + "minLength": 1, + "type": "string" + }, "to": { "minLength": 1, "type": "string" @@ -68547,10 +68579,18 @@ "ignore_failure": { "type": "boolean" }, + "locale": { + "minLength": 1, + "type": "string" + }, "output_format": { "minLength": 1, "type": "string" }, + "timezone": { + "minLength": 1, + "type": "string" + }, "to": { "minLength": 1, "type": "string" @@ -72404,10 +72444,18 @@ "ignore_failure": { "type": "boolean" }, + "locale": { + "minLength": 1, + "type": "string" + }, "output_format": { "minLength": 1, "type": "string" }, + "timezone": { + "minLength": 1, + "type": "string" + }, "to": { "minLength": 1, "type": "string" From a403a7c80089f70d80fe2603d47689664f7ce589 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 4 Nov 2025 15:26:50 +0000 Subject: [PATCH 8/8] Changes from make api-docs --- oas_docs/output/kibana.serverless.yaml | 36 ++++++++++++++++++++++++++ oas_docs/output/kibana.yaml | 36 ++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index cb3c74692c968..e098db4ffc81a 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -54755,9 +54755,15 @@ paths: type: string ignore_failure: type: boolean + locale: + minLength: 1 + type: string output_format: minLength: 1 type: string + timezone: + minLength: 1 + type: string to: minLength: 1 type: string @@ -56966,9 +56972,15 @@ paths: type: string ignore_failure: type: boolean + locale: + minLength: 1 + type: string output_format: minLength: 1 type: string + timezone: + minLength: 1 + type: string to: minLength: 1 type: string @@ -58973,9 +58985,15 @@ paths: type: string ignore_failure: type: boolean + locale: + minLength: 1 + type: string output_format: minLength: 1 type: string + timezone: + minLength: 1 + type: string to: minLength: 1 type: string @@ -61034,9 +61052,15 @@ paths: type: string ignore_failure: type: boolean + locale: + minLength: 1 + type: string output_format: minLength: 1 type: string + timezone: + minLength: 1 + type: string to: minLength: 1 type: string @@ -63414,9 +63438,15 @@ paths: type: string ignore_failure: type: boolean + locale: + minLength: 1 + type: string output_format: minLength: 1 type: string + timezone: + minLength: 1 + type: string to: minLength: 1 type: string @@ -65398,9 +65428,15 @@ paths: type: string ignore_failure: type: boolean + locale: + minLength: 1 + type: string output_format: minLength: 1 type: string + timezone: + minLength: 1 + type: string to: minLength: 1 type: string diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index 7a5870e0f098a..5dbe91def9398 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -59104,9 +59104,15 @@ paths: type: string ignore_failure: type: boolean + locale: + minLength: 1 + type: string output_format: minLength: 1 type: string + timezone: + minLength: 1 + type: string to: minLength: 1 type: string @@ -61315,9 +61321,15 @@ paths: type: string ignore_failure: type: boolean + locale: + minLength: 1 + type: string output_format: minLength: 1 type: string + timezone: + minLength: 1 + type: string to: minLength: 1 type: string @@ -63322,9 +63334,15 @@ paths: type: string ignore_failure: type: boolean + locale: + minLength: 1 + type: string output_format: minLength: 1 type: string + timezone: + minLength: 1 + type: string to: minLength: 1 type: string @@ -65383,9 +65401,15 @@ paths: type: string ignore_failure: type: boolean + locale: + minLength: 1 + type: string output_format: minLength: 1 type: string + timezone: + minLength: 1 + type: string to: minLength: 1 type: string @@ -67763,9 +67787,15 @@ paths: type: string ignore_failure: type: boolean + locale: + minLength: 1 + type: string output_format: minLength: 1 type: string + timezone: + minLength: 1 + type: string to: minLength: 1 type: string @@ -69747,9 +69777,15 @@ paths: type: string ignore_failure: type: boolean + locale: + minLength: 1 + type: string output_format: minLength: 1 type: string + timezone: + minLength: 1 + type: string to: minLength: 1 type: string