Skip to content

Commit 1cc5c0f

Browse files
ananzhps48opensearch-changeset-bot[bot]
authored
[Manual backport 2.x] Add support for async ppl to discover (#8706) (#8769)
Backport PR: #8706 From original PR: * add support for async ppl to discover * Changeset file for PR #8706 created/updated * update s3_type test to add PPL as supported lang * fix lint error --------- Signed-off-by: Shenoy Pratik <[email protected]> Co-authored-by: Shenoy Pratik <[email protected]> Co-authored-by: opensearch-changeset-bot[bot] <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com>
1 parent e2aaab4 commit 1cc5c0f

File tree

11 files changed

+139
-18
lines changed

11 files changed

+139
-18
lines changed

‎changelogs/fragments/8706.yml

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
feat:
2+
- Add support for async ppl to discover ([#8706](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8706))

‎src/plugins/query_enhancements/common/constants.ts

+2
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,15 @@ export const SEARCH_STRATEGY = {
1717
PPL_RAW: 'pplraw',
1818
SQL: 'sql',
1919
SQL_ASYNC: 'sqlasync',
20+
PPL_ASYNC: 'pplasync',
2021
};
2122

2223
export const API = {
2324
SEARCH: `${BASE_API}/search`,
2425
PPL_SEARCH: `${BASE_API}/search/${SEARCH_STRATEGY.PPL}`,
2526
SQL_SEARCH: `${BASE_API}/search/${SEARCH_STRATEGY.SQL}`,
2627
SQL_ASYNC_SEARCH: `${BASE_API}/search/${SEARCH_STRATEGY.SQL_ASYNC}`,
28+
PPL_ASYNC_SEARCH: `${BASE_API}/search/${SEARCH_STRATEGY.PPL_ASYNC}`,
2729
QUERY_ASSIST: {
2830
LANGUAGES: `${BASE_API}/assist/languages`,
2931
GENERATE: `${BASE_API}/assist/generate`,

‎src/plugins/query_enhancements/public/datasets/s3_type.test.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -244,9 +244,9 @@ describe('s3TypeConfig', () => {
244244
expect(result[3].type).toBe('number');
245245
});
246246

247-
test('supportedLanguages returns SQL', () => {
247+
test('supportedLanguages returns SQL, PPL', () => {
248248
const mockDataset: Dataset = { id: 'table1', title: 'Table 1', type: 'S3' };
249-
expect(s3TypeConfig.supportedLanguages(mockDataset)).toEqual(['SQL']);
249+
expect(s3TypeConfig.supportedLanguages(mockDataset)).toEqual(['SQL', 'PPL']);
250250
});
251251

252252
describe('castS3FieldTypeToOSDFieldType()', () => {

‎src/plugins/query_enhancements/public/datasets/s3_type.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ export const s3TypeConfig: DatasetTypeConfig = {
118118
},
119119

120120
supportedLanguages: (dataset: Dataset): string[] => {
121-
return ['SQL'];
121+
return ['SQL', 'PPL'];
122122
},
123123

124124
getSampleQueries: (dataset: Dataset, language: string) => {
@@ -129,7 +129,7 @@ export const s3TypeConfig: DatasetTypeConfig = {
129129
title: i18n.translate('queryEnhancements.s3Type.sampleQuery.basicPPLQuery', {
130130
defaultMessage: 'Sample query for PPL',
131131
}),
132-
query: `source = ${dataset.title}`,
132+
query: `source = ${dataset.title} | head 10`,
133133
},
134134
];
135135
case 'SQL':

‎src/plugins/query_enhancements/public/plugin.tsx

+12-7
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,26 @@
44
*/
55
import { i18n } from '@osd/i18n';
66
import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '../../../core/public';
7+
import { DataStorage } from '../../data/common';
8+
import {
9+
createEditor,
10+
DefaultInput,
11+
LanguageConfig,
12+
Query,
13+
SingleLineInput,
14+
} from '../../data/public';
715
import { ConfigSchema } from '../common/config';
8-
import { setData, setStorage } from './services';
16+
import { s3TypeConfig } from './datasets';
917
import { createQueryAssistExtension } from './query_assist';
18+
import { pplLanguageReference, sqlLanguageReference } from './query_editor_extensions';
1019
import { PPLSearchInterceptor, SQLSearchInterceptor } from './search';
20+
import { setData, setStorage } from './services';
1121
import {
1222
QueryEnhancementsPluginSetup,
1323
QueryEnhancementsPluginSetupDependencies,
1424
QueryEnhancementsPluginStart,
1525
QueryEnhancementsPluginStartDependencies,
1626
} from './types';
17-
import { LanguageConfig, Query } from '../../data/public';
18-
import { s3TypeConfig } from './datasets';
19-
import { createEditor, DefaultInput, SingleLineInput } from '../../data/public';
20-
import { DataStorage } from '../../data/common';
21-
import { pplLanguageReference, sqlLanguageReference } from './query_editor_extensions';
2227

2328
export class QueryEnhancementsPlugin
2429
implements
@@ -57,7 +62,7 @@ export class QueryEnhancementsPlugin
5762
startServices: core.getStartServices(),
5863
usageCollector: data.search.usageCollector,
5964
}),
60-
getQueryString: (currentQuery: Query) => `source = ${currentQuery.dataset?.title}`,
65+
getQueryString: (currentQuery: Query) => `source = ${currentQuery.dataset?.title} | head 10`,
6166
fields: { filterable: false, visualizable: false },
6267
docLink: {
6368
title: i18n.translate('queryEnhancements.pplLanguage.docLink', {

‎src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import {
1616
} from '../../../data/public';
1717
import {
1818
API,
19+
DATASET,
1920
EnhancedFetchContext,
2021
fetch,
2122
formatDate,
@@ -60,7 +61,7 @@ export class PPLSearchInterceptor extends SearchInterceptor {
6061
public search(request: IOpenSearchDashboardsSearchRequest, options: ISearchOptions) {
6162
const dataset = this.queryService.queryString.getQuery().dataset;
6263
const datasetType = dataset?.type;
63-
let strategy = SEARCH_STRATEGY.PPL;
64+
let strategy = datasetType === DATASET.S3 ? SEARCH_STRATEGY.PPL_ASYNC : SEARCH_STRATEGY.PPL;
6465

6566
if (datasetType) {
6667
const datasetTypeConfig = this.queryService.queryString

‎src/plugins/query_enhancements/public/search/sql_search_interceptor.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@
44
*/
55

66
import { trimEnd } from 'lodash';
7+
import { CoreStart } from 'opensearch-dashboards/public';
78
import { Observable, throwError } from 'rxjs';
89
import { catchError } from 'rxjs/operators';
9-
import { CoreStart } from 'opensearch-dashboards/public';
1010
import {
1111
DataPublicPluginStart,
1212
IOpenSearchDashboardsSearchRequest,

‎src/plugins/query_enhancements/server/plugin.ts

+10-2
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,11 @@ import { SEARCH_STRATEGY } from '../common';
1717
import { ConfigSchema } from '../common/config';
1818
import { defineRoutes, defineSearchStrategyRouteProvider } from './routes';
1919
import {
20-
pplSearchStrategyProvider,
20+
pplAsyncSearchStrategyProvider,
2121
pplRawSearchStrategyProvider,
22-
sqlSearchStrategyProvider,
22+
pplSearchStrategyProvider,
2323
sqlAsyncSearchStrategyProvider,
24+
sqlSearchStrategyProvider,
2425
} from './search';
2526
import {
2627
QueryEnhancementsPluginSetup,
@@ -58,11 +59,17 @@ export class QueryEnhancementsPlugin
5859
this.logger,
5960
client
6061
);
62+
const pplAsyncSearchStrategy = pplAsyncSearchStrategyProvider(
63+
this.config$,
64+
this.logger,
65+
client
66+
);
6167

6268
data.search.registerSearchStrategy(SEARCH_STRATEGY.PPL, pplSearchStrategy);
6369
data.search.registerSearchStrategy(SEARCH_STRATEGY.PPL_RAW, pplRawSearchStrategy);
6470
data.search.registerSearchStrategy(SEARCH_STRATEGY.SQL, sqlSearchStrategy);
6571
data.search.registerSearchStrategy(SEARCH_STRATEGY.SQL_ASYNC, sqlAsyncSearchStrategy);
72+
data.search.registerSearchStrategy(SEARCH_STRATEGY.PPL_ASYNC, pplAsyncSearchStrategy);
6673

6774
core.http.registerRouteHandlerContext('query_assist', () => ({
6875
logger: this.logger,
@@ -86,6 +93,7 @@ export class QueryEnhancementsPlugin
8693
ppl: pplSearchStrategy,
8794
sql: sqlSearchStrategy,
8895
sqlasync: sqlAsyncSearchStrategy,
96+
pplasync: pplAsyncSearchStrategy,
8997
});
9098

9199
this.logger.info('queryEnhancements: Setup complete');

‎src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
*/
55

66
import { schema } from '@osd/config-schema';
7-
import { IRouter, ILegacyClusterClient } from 'opensearch-dashboards/server';
7+
import { ILegacyClusterClient, IRouter } from 'opensearch-dashboards/server';
88
import { API } from '../../../common';
99

1010
export function registerDataSourceConnectionsRoutes(

‎src/plugins/query_enhancements/server/search/index.ts

+3-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
* SPDX-License-Identifier: Apache-2.0
44
*/
55

6-
export { pplSearchStrategyProvider } from './ppl_search_strategy';
6+
export { pplAsyncSearchStrategyProvider } from './ppl_async_search_strategy';
77
export { pplRawSearchStrategyProvider } from './ppl_raw_search_strategy';
8-
export { sqlSearchStrategyProvider } from './sql_search_strategy';
8+
export { pplSearchStrategyProvider } from './ppl_search_strategy';
99
export { sqlAsyncSearchStrategyProvider } from './sql_async_search_strategy';
10+
export { sqlSearchStrategyProvider } from './sql_search_strategy';

‎src/plugins/query_enhancements/server/search/ppl_async_search_strategy.ts

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
import { ILegacyClusterClient, Logger, SharedGlobalConfig } from 'opensearch-dashboards/server';
7+
import { Observable } from 'rxjs';
8+
import {
9+
createDataFrame,
10+
DATA_FRAME_TYPES,
11+
IDataFrameResponse,
12+
IOpenSearchDashboardsSearchRequest,
13+
Query,
14+
} from '../../../data/common';
15+
import { ISearchStrategy, SearchUsage } from '../../../data/server';
16+
import { buildQueryStatusConfig, getFields, handleFacetError, SEARCH_STRATEGY } from '../../common';
17+
import { Facet } from '../utils';
18+
19+
export const pplAsyncSearchStrategyProvider = (
20+
config$: Observable<SharedGlobalConfig>,
21+
logger: Logger,
22+
client: ILegacyClusterClient,
23+
usage?: SearchUsage
24+
): ISearchStrategy<IOpenSearchDashboardsSearchRequest, IDataFrameResponse> => {
25+
const pplAsyncFacet = new Facet({
26+
client,
27+
logger,
28+
endpoint: 'enhancements.runDirectQuery',
29+
});
30+
const pplAsyncJobsFacet = new Facet({
31+
client,
32+
logger,
33+
endpoint: 'enhancements.getJobStatus',
34+
useJobs: true,
35+
});
36+
37+
return {
38+
search: async (context, request: any, options) => {
39+
try {
40+
const query: Query = request.body.query;
41+
const pollQueryResultsParams = request.body.pollQueryResultsParams;
42+
const inProgressQueryId = pollQueryResultsParams?.queryId;
43+
44+
if (!inProgressQueryId) {
45+
request.body = { ...request.body, lang: SEARCH_STRATEGY.PPL };
46+
const rawResponse: any = await pplAsyncFacet.describeQuery(context, request);
47+
48+
if (!rawResponse.success) handleFacetError(rawResponse);
49+
50+
const statusConfig = buildQueryStatusConfig(rawResponse);
51+
52+
return {
53+
type: DATA_FRAME_TYPES.POLLING,
54+
status: 'started',
55+
body: {
56+
queryStatusConfig: statusConfig,
57+
},
58+
} as IDataFrameResponse;
59+
} else {
60+
request.params = { queryId: inProgressQueryId };
61+
const queryStatusResponse: any = await pplAsyncJobsFacet.describeQuery(context, request);
62+
const queryStatus = queryStatusResponse?.data?.status;
63+
logger.info(`pplAsyncSearchStrategy: JOB: ${inProgressQueryId} - STATUS: ${queryStatus}`);
64+
65+
if (queryStatus?.toUpperCase() === 'SUCCESS') {
66+
const dataFrame = createDataFrame({
67+
name: query.dataset?.id,
68+
schema: queryStatusResponse.data.schema,
69+
meta: { ...pollQueryResultsParams },
70+
fields: getFields(queryStatusResponse),
71+
});
72+
73+
dataFrame.size = queryStatusResponse.data.datarows.length;
74+
75+
return {
76+
type: DATA_FRAME_TYPES.POLLING,
77+
status: 'success',
78+
body: dataFrame,
79+
} as IDataFrameResponse;
80+
} else if (queryStatus?.toUpperCase() === 'FAILED') {
81+
return {
82+
type: DATA_FRAME_TYPES.POLLING,
83+
status: 'failed',
84+
body: {
85+
error: `JOB: ${inProgressQueryId} failed: ${queryStatusResponse.data.error}`,
86+
},
87+
} as IDataFrameResponse;
88+
}
89+
90+
return {
91+
type: DATA_FRAME_TYPES.POLLING,
92+
status: queryStatus,
93+
} as IDataFrameResponse;
94+
}
95+
} catch (e: any) {
96+
logger.error(`pplAsyncSearchStrategy: ${e.message}`);
97+
if (usage) usage.trackError();
98+
throw e;
99+
}
100+
},
101+
};
102+
};

0 commit comments

Comments
 (0)