Skip to content

[Entity Store] Make schema properties nullable #264509

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
orouz merged 9 commits into from
Apr 26, 2026
Merged

[Entity Store] Make schema properties nullable #264509

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
6abd6de
Revert "[Entity Store] Reset state error after successful task run (#…
orouz Apr 21, 2026
0c7fd64
[Entity Store] Make schema properties nullable
orouz Apr 21, 2026
218bd51
[Entity Store] Update SO type snapshots for engine-descriptor-v2 v10.5.0
orouz Apr 21, 2026
a69241e
make createMockEngineDescriptor props nullable
orouz Apr 23, 2026
6111144
fix types
orouz Apr 23, 2026
a95b418
remove error schema oneOf changes
orouz Apr 26, 2026
a75d0a8
update SO type snapshots
orouz Apr 26, 2026
2f58c1c
schema fix
orouz Apr 26, 2026
feac50e
Changes from node scripts/jest_integration -u src/core/server/integra…
kibanamachine Apr 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions ...eck-saved-objects-cli/src/migrations/__fixtures__/entity-engine-descriptor-v2/10.5.0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
{
"10.4.0": [
{
"type": "user",
"status": "started",
"logExtractionState": {
"lastExecutionTimestamp": "2026-03-04T17:37:20.000Z"
},
"versionState": {
"version": "2",
"state": "running",
"isMigratedFromV1": true
}
}
],
"10.5.0": [
{
"type": "user",
"status": "started",
"logExtractionState": {
"paginationTimestamp": null,
"paginationId": null,
"lastExecutionTimestamp": "2026-03-04T17:37:20.000Z",
"logsPageCursorStartTimestamp": null,
"logsPageCursorStartId": null,
"logsPageCursorEndTimestamp": null,
"logsPageCursorEndId": null
},
"error": null,
"versionState": {
"version": "2",
"state": "running",
"isMigratedFromV1": true
}
}
]
}
7 changes: 4 additions & 3 deletions src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
"entity-analytics-monitoring-entity-source": "a7307895e5c38accc21ebea852bd46ef98ebb73f7a73d116c8dfcd3932f57899",
"entity-definition": "28cde811da5815b6fdebb0112419b593253fb5053d60c2cf29b50af3be058c67",
"entity-discovery-api-key": "2350e2ce1b6e913f55aef2cd04971b15604b03e049370ef10949c3eeec38c46f",
"entity-engine-descriptor-v2": "eb2f5ab5f8f620f00a2bb7ee8a99da7ee5187905b8e20ec95ce654cf84fe7b78",
"entity-engine-descriptor-v2": "44b60aa3d3d4583082b58500b297f2dddb4ffc14b77ad1d568b70536ca4ae787",
"entity-engine-status": "005903620a00737932aa54ae57817b078810b2f71cc42e7715d1c22c5e5b715e",
"entity-store-global-state": "8581bc65d1b2bf6d0218b693509129a2515599aeff8933d85353a3fb28d52bda",
"epm-packages": "46e4129dba3ac33d4924239672169f12ad75536e9f44f695964220a80ebfeaca",
Expand Down Expand Up @@ -634,6 +634,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
"entity-engine-descriptor-v2|global: ae6bdc1b39c24032ba32bbc8a5d032f63d576bb0",
"entity-engine-descriptor-v2|mappings: c8194b9fb0346df2967cedb78fb4a3c7139f5ff1",
"entity-engine-descriptor-v2|schemas: da39a3ee5e6b4b0d3255bfef95601890afd80709",
"entity-engine-descriptor-v2|10.5.0: 33de56b01a358e06b3e3729f22b7f281e51c9f4bce4dd86067d368462018d258",
"entity-engine-descriptor-v2|10.4.0: 52668e17842853b15deb997ceddb19b551e6d38b18d2440123db05bf844e157e",
"entity-engine-descriptor-v2|10.3.0: 14e40ca4628481c3db33990704f13a4dcef6e9ae2e6e7539606dc58194169c96",
"entity-engine-descriptor-v2|10.2.0: 114c9ccc2679bf611a71b9ba2fc9ee842ab1372e808dc4bf45a061c5ae0e2081",
Expand Down Expand Up @@ -1455,7 +1456,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
"entity-analytics-monitoring-entity-source": "10.2.0",
"entity-definition": "10.4.0",
"entity-discovery-api-key": "10.0.0",
"entity-engine-descriptor-v2": "10.4.0",
"entity-engine-descriptor-v2": "10.5.0",
"entity-engine-status": "10.2.0",
"entity-store-global-state": "10.1.0",
"epm-packages": "10.8.0",
Expand Down Expand Up @@ -1621,7 +1622,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
"entity-analytics-monitoring-entity-source": "10.2.0",
"entity-definition": "10.4.0",
"entity-discovery-api-key": "0.0.0",
"entity-engine-descriptor-v2": "10.4.0",
"entity-engine-descriptor-v2": "10.5.0",
"entity-engine-status": "10.2.0",
"entity-store-global-state": "10.1.0",
"epm-packages": "10.8.0",
Expand Down
153 changes: 65 additions & 88 deletions ...ecurity/plugins/entity_store/server/domain/logs_extraction/logs_extraction_client.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,13 +97,13 @@ function createMockEngineDescriptor(
}>
) {
const logExtractionState = {
paginationTimestamp: overrides?.paginationTimestamp,
paginationId: overrides?.paginationId,
lastExecutionTimestamp: overrides?.lastExecutionTimestamp,
logsPageCursorStartTimestamp: overrides?.logsPageCursorStartTimestamp,
logsPageCursorStartId: overrides?.logsPageCursorStartId,
logsPageCursorEndTimestamp: overrides?.logsPageCursorEndTimestamp,
logsPageCursorEndId: overrides?.logsPageCursorEndId,
paginationTimestamp: overrides?.paginationTimestamp ?? null,
paginationId: overrides?.paginationId ?? null,
lastExecutionTimestamp: overrides?.lastExecutionTimestamp ?? null,
logsPageCursorStartTimestamp: overrides?.logsPageCursorStartTimestamp ?? null,
logsPageCursorStartId: overrides?.logsPageCursorStartId ?? null,
logsPageCursorEndTimestamp: overrides?.logsPageCursorEndTimestamp ?? null,
logsPageCursorEndId: overrides?.logsPageCursorEndId ?? null,
};
return {
type,
Expand Down Expand Up @@ -136,7 +136,7 @@ describe('LogsExtractionClient', () => {
let mockEsClient: jest.Mocked<ElasticsearchClient>;
let mockDataViewsService: jest.Mocked<DataViewsService>;
let mockEngineDescriptorClient: jest.Mocked<
Pick<EngineDescriptorClient, 'findOrThrow' | 'update' | 'updateWith'>
Pick<EngineDescriptorClient, 'findOrThrow' | 'update'>
>;
let mockGlobalStateClient: ReturnType<typeof createMockGlobalStateClient>;
let mockCcsLogsExtractionClient: ReturnType<typeof createMockCcsLogsExtractionClient>;
Expand All @@ -152,7 +152,6 @@ describe('LogsExtractionClient', () => {
mockEngineDescriptorClient = {
findOrThrow: jest.fn(),
update: jest.fn().mockResolvedValue({}),
updateWith: jest.fn().mockResolvedValue({}),
};
mockGlobalStateClient = createMockGlobalStateClient();
mockCcsLogsExtractionClient = createMockCcsLogsExtractionClient();
Expand All @@ -169,17 +168,6 @@ describe('LogsExtractionClient', () => {
});

describe('extractLogs', () => {
function applyLastUpdater(
entityType: EntityType = 'user',
descriptorOverrides?: Partial<Awaited<ReturnType<EngineDescriptorClient['findOrThrow']>>>
) {
const [[, updater]] = mockEngineDescriptorClient.updateWith.mock.calls;
return updater({
...createMockEngineDescriptor(entityType),
...descriptorOverrides,
} as Awaited<ReturnType<EngineDescriptorClient['findOrThrow']>>);
}

it('should successfully extract logs and ingest entities', async () => {
const lastTimestamp = '2024-01-02T12:00:00.000Z';
const mockEsqlResponse: ESQLSearchResponse = {
Expand Down Expand Up @@ -241,22 +229,20 @@ describe('LogsExtractionClient', () => {
abortController: undefined,
});

expect(mockEngineDescriptorClient.updateWith).toHaveBeenCalledWith(
expect(mockEngineDescriptorClient.update).toHaveBeenCalledWith(
'user',
expect.any(Function)
expect.objectContaining({
logExtractionState: expect.objectContaining({
paginationTimestamp: null,
paginationId: null,
logsPageCursorStartTimestamp: null,
logsPageCursorStartId: null,
logsPageCursorEndTimestamp: null,
logsPageCursorEndId: null,
lastExecutionTimestamp: expect.any(String),
}),
})
);
expect(applyLastUpdater()).toMatchObject({
logExtractionState: expect.objectContaining({
paginationTimestamp: undefined,
paginationId: undefined,
logsPageCursorStartTimestamp: undefined,
logsPageCursorStartId: undefined,
logsPageCursorEndTimestamp: undefined,
logsPageCursorEndId: undefined,
lastExecutionTimestamp: expect.any(String),
}),
error: undefined,
});
});

it('should handle empty results from ESQL query', async () => {
Expand All @@ -280,22 +266,20 @@ describe('LogsExtractionClient', () => {

expect(mockExecuteEsqlQuery).toHaveBeenCalledTimes(1);
expect(mockIngestEntities).toHaveBeenCalledTimes(0);
expect(mockEngineDescriptorClient.updateWith).toHaveBeenCalledWith(
expect(mockEngineDescriptorClient.update).toHaveBeenCalledWith(
'user',
expect.any(Function)
expect.objectContaining({
logExtractionState: expect.objectContaining({
paginationTimestamp: null,
paginationId: null,
logsPageCursorStartTimestamp: null,
logsPageCursorStartId: null,
logsPageCursorEndTimestamp: null,
logsPageCursorEndId: null,
lastExecutionTimestamp: expect.any(String),
}),
})
);
expect(applyLastUpdater()).toMatchObject({
logExtractionState: expect.objectContaining({
paginationTimestamp: undefined,
paginationId: undefined,
logsPageCursorStartTimestamp: undefined,
logsPageCursorStartId: undefined,
logsPageCursorEndTimestamp: undefined,
logsPageCursorEndId: undefined,
lastExecutionTimestamp: expect.any(String),
}),
error: undefined,
});
});

it('should compute extraction window from lookbackPeriod and delay when no custom range', async () => {
Expand Down Expand Up @@ -618,7 +602,7 @@ describe('LogsExtractionClient', () => {
esClient: mockEsClient,
query: expect.stringContaining(toDate),
});
expect(mockEngineDescriptorClient.updateWith).not.toHaveBeenCalled();
expect(mockEngineDescriptorClient.update).not.toHaveBeenCalled();
});

it('should not update engine descriptor when specificWindow is provided', async () => {
Expand Down Expand Up @@ -657,7 +641,7 @@ describe('LogsExtractionClient', () => {
expect(result.success && result.count).toBe(2);
expect(mockExecuteEsqlQuery).toHaveBeenCalledTimes(3);
expect(mockIngestEntities).toHaveBeenCalledTimes(1);
expect(mockEngineDescriptorClient.updateWith).not.toHaveBeenCalled();
expect(mockEngineDescriptorClient.update).not.toHaveBeenCalled();
});

it('should handle errors from executeEsqlQuery', async () => {
Expand Down Expand Up @@ -800,48 +784,42 @@ describe('LogsExtractionClient', () => {
expect(mockIngestEntities).toHaveBeenCalledTimes(1);

// CCS error is stored in the saved object
expect(mockEngineDescriptorClient.updateWith).toHaveBeenCalledWith(
expect(mockEngineDescriptorClient.update).toHaveBeenCalledWith(
'user',
expect.any(Function)
expect.objectContaining({
logExtractionState: expect.objectContaining({
paginationTimestamp: null,
paginationId: null,
logsPageCursorStartTimestamp: null,
logsPageCursorStartId: null,
logsPageCursorEndTimestamp: null,
logsPageCursorEndId: null,
lastExecutionTimestamp: expect.any(String),
}),
error: { message: ccsError.message, action: 'extractLogs' },
})
);
expect(applyLastUpdater()).toMatchObject({
logExtractionState: expect.objectContaining({
paginationTimestamp: undefined,
paginationId: undefined,
logsPageCursorStartTimestamp: undefined,
logsPageCursorStartId: undefined,
logsPageCursorEndTimestamp: undefined,
logsPageCursorEndId: undefined,
lastExecutionTimestamp: expect.any(String),
}),
error: { message: ccsError.message, action: 'extractLogs' },
});
});

it('should clear a previous error after a successful extraction', async () => {
const mockDataView = {
getIndexPattern: jest.fn().mockReturnValue('logs-*'),
};

mockEngineDescriptorClient.findOrThrow.mockResolvedValue(
createMockEngineDescriptor('user') as Awaited<
ReturnType<EngineDescriptorClient['findOrThrow']>
>
);
mockEngineDescriptorClient.findOrThrow.mockResolvedValue({
...createMockEngineDescriptor('user'),
Comment thread
orouz marked this conversation as resolved.
error: { message: 'previous error', action: 'extractLogs' as const },
} as Awaited<ReturnType<EngineDescriptorClient['findOrThrow']>>);
mockDataViewsService.get.mockResolvedValue(mockDataView as any);
mockExecuteEsqlQuery.mockResolvedValue(mockLogPaginationCursorProbeEmpty());
mockIngestEntities.mockResolvedValue(undefined);

await client.extractLogs('user');

expect(mockEngineDescriptorClient.updateWith).toHaveBeenCalledWith(
expect(mockEngineDescriptorClient.update).toHaveBeenCalledWith(
'user',
expect.any(Function)
expect.objectContaining({ error: null })
);
expect(
applyLastUpdater('user', { error: { message: 'previous error', action: 'extractLogs' } })
.error
).toBeUndefined();
});

it('should fallback to logs-* when data view is not found', async () => {
Expand Down Expand Up @@ -894,21 +872,20 @@ describe('LogsExtractionClient', () => {
targetIndex: expect.stringContaining('.entities.v2.latest.security_default'),
})
);
expect(mockEngineDescriptorClient.updateWith).toHaveBeenCalledWith(
expect(mockEngineDescriptorClient.update).toHaveBeenCalledWith(
'host',
expect.any(Function)
expect.objectContaining({
logExtractionState: expect.objectContaining({
paginationTimestamp: null,
paginationId: null,
logsPageCursorStartTimestamp: null,
logsPageCursorStartId: null,
logsPageCursorEndTimestamp: null,
logsPageCursorEndId: null,
lastExecutionTimestamp: expect.any(String),
}),
})
);
expect(applyLastUpdater('host')).toMatchObject({
logExtractionState: expect.objectContaining({
paginationTimestamp: undefined,
paginationId: undefined,
logsPageCursorStartTimestamp: undefined,
logsPageCursorStartId: undefined,
logsPageCursorEndTimestamp: undefined,
logsPageCursorEndId: undefined,
lastExecutionTimestamp: expect.any(String),
}),
});
});

it('should return success false when engine is not started', async () => {
Expand Down
Loading
Loading