Skip to content

Commit 970656d

Browse files
committed
Merge branch 'alerting/consumer-based-rbac' of github.com:gmmorris/kibana into alerting/consumer-based-rbac
* 'alerting/consumer-based-rbac' of github.com:gmmorris/kibana: (25 commits) [Lens] Fix broken test (elastic#70117) [SIEM] Import timeline fix (elastic#65448) [SECURITY SOLUTION][INGEST] UX update for ingest manager edit/create datasource for endpoint (elastic#70079) [Telemetry] Collector Schema (elastic#64942) [Endpoint] Add Endpoint empty states for onboarding (elastic#69626) Hide unused resolver buttons (elastic#70112) [Security] `Investigate in Resolver` Timeline Integration (elastic#70111) [Discover] Improve styling of graphs in sidebar (elastic#69440) [Metrics UI] Fix EuiTheme type issue (elastic#69735) skip failing suite (elastic#70104) (elastic#70103) [ENDPOINT] Hide the Timeline Flyout while on the Management Pages (elastic#69998) [SIEM][CASE] Persist callout when dismissed (elastic#68372) [SIEM][Exceptions] - Cleaned up and updated exception list item comment structure (elastic#69532) [Maps] remove indexing state from redux (elastic#69765) Add API integration test for deleting data streams. (elastic#70020) renames SIEM to Security Solution (elastic#70070) Adding saved_objects_page in OSS (elastic#69900) [Lens] Use accordion menus in field list for available and empty fields (elastic#68871) Dynamic uiActions & license support (elastic#68507) [SIEM] Update readme for timeline apis (elastic#67038) ...
2 parents 1a20848 + bd0f78c commit 970656d

File tree

491 files changed

+16916
-4976
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

491 files changed

+16916
-4976
lines changed

.github/CODEOWNERS

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,13 +170,19 @@
170170

171171
# Kibana Telemetry
172172
/packages/kbn-analytics/ @elastic/kibana-telemetry
173+
/packages/kbn-telemetry-tools/ @elastic/kibana-telemetry
173174
/src/plugins/kibana_usage_collection/ @elastic/kibana-telemetry
174175
/src/plugins/newsfeed/ @elastic/kibana-telemetry
175176
/src/plugins/telemetry/ @elastic/kibana-telemetry
176177
/src/plugins/telemetry_collection_manager/ @elastic/kibana-telemetry
177178
/src/plugins/telemetry_management_section/ @elastic/kibana-telemetry
178179
/src/plugins/usage_collection/ @elastic/kibana-telemetry
179180
/x-pack/plugins/telemetry_collection_xpack/ @elastic/kibana-telemetry
181+
/.telemetryrc.json @elastic/kibana-telemetry
182+
/x-pack/.telemetryrc.json @elastic/kibana-telemetry
183+
src/plugins/telemetry/schema/legacy_oss_plugins.json @elastic/kibana-telemetry
184+
src/plugins/telemetry/schema/oss_plugins.json @elastic/kibana-telemetry
185+
x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @elastic/kibana-telemetry
180186

181187
# Kibana Alerting Services
182188
/x-pack/plugins/alerts/ @elastic/kibana-alerting-services

.telemetryrc.json

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
[
2+
{
3+
"output": "src/plugins/telemetry/schema/legacy_oss_plugins.json",
4+
"root": "src/legacy/core_plugins/",
5+
"exclude": [
6+
"src/legacy/core_plugins/testbed",
7+
"src/legacy/core_plugins/elasticsearch",
8+
"src/legacy/core_plugins/tests_bundle"
9+
]
10+
},
11+
{
12+
"output": "src/plugins/telemetry/schema/oss_plugins.json",
13+
"root": "src/plugins/",
14+
"exclude": [
15+
"src/plugins/kibana_react/",
16+
"src/plugins/testbed/",
17+
"src/plugins/kibana_utils/",
18+
"src/plugins/kibana_usage_collection/server/collectors/kibana/kibana_usage_collector.ts",
19+
"src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.ts",
20+
"src/plugins/kibana_usage_collection/server/collectors/management/telemetry_management_collector.ts",
21+
"src/plugins/kibana_usage_collection/server/collectors/ui_metric/telemetry_ui_metric_collector.ts",
22+
"src/plugins/telemetry/server/collectors/usage/telemetry_usage_collector.ts"
23+
]
24+
}
25+
]

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@
139139
"@kbn/babel-preset": "1.0.0",
140140
"@kbn/config-schema": "1.0.0",
141141
"@kbn/i18n": "1.0.0",
142+
"@kbn/telemetry-tools": "1.0.0",
142143
"@kbn/interpreter": "1.0.0",
143144
"@kbn/pm": "1.0.0",
144145
"@kbn/test-subj-selector": "0.2.1",

packages/kbn-monaco/src/xjson/grammar.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -200,12 +200,13 @@ export const createParser = () => {
200200

201201
try {
202202
value();
203+
white();
203204
} catch (e) {
204205
errored = true;
205206
annos.push({ type: AnnoTypes.error, at: e.at - 1, text: e.message });
206207
}
207208
if (!errored && ch) {
208-
error('Syntax error');
209+
annos.push({ type: AnnoTypes.error, at: at, text: 'Syntax Error' });
209210
}
210211
return { annotations: annos };
211212
}

packages/kbn-monaco/src/xjson/language.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,10 @@ export const registerGrammarChecker = (editor: monaco.editor.IEditor) => {
5252

5353
const updateAnnos = async () => {
5454
const { annotations } = await wps.getAnnos();
55-
const model = editor.getModel() as monaco.editor.ITextModel;
55+
const model = editor.getModel() as monaco.editor.ITextModel | null;
56+
if (!model) {
57+
return;
58+
}
5659
monaco.editor.setModelMarkers(
5760
model,
5861
OWNER,
Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
# Telemetry Tools
2+
3+
## Schema extraction tool
4+
5+
### Description
6+
7+
The tool is used to extract telemetry collectors schema from all `*.{ts}` files in provided plugins directories to JSON files. The tool looks for `.telemetryrc.json` files in the root of the project and in the `x-pack` dir for its runtime configurations.
8+
9+
It uses typescript parser to build an AST for each file. The tool is able to validate, extract and match collector schemas.
10+
11+
### Examples and restrictions
12+
13+
**Global restrictions**:
14+
15+
The `id` can be only a string literal, it cannot be a template literals w/o expressions or string-only concatenation expressions or anything else.
16+
17+
```
18+
export const myCollector = makeUsageCollector<Usage>({
19+
type: 'string_literal_only',
20+
...
21+
});
22+
```
23+
24+
### Usage
25+
26+
```bash
27+
node scripts/telemetry_extract.js
28+
```
29+
30+
This command has no additional flags or arguments. The `.telemetryrc.json` files specify the path to the directory where searching should start, output json files, and files to exclude.
31+
32+
33+
### Output
34+
35+
36+
The generated JSON files contain an ES mapping for each schema. This mapping is used to verify changes in the collectors and as the basis to map those fields into the external telemetry cluster.
37+
38+
**Example**:
39+
40+
```json
41+
{
42+
"properties": {
43+
"cloud": {
44+
"properties": {
45+
"isCloudEnabled": {
46+
"type": "boolean"
47+
}
48+
}
49+
}
50+
}
51+
}
52+
```
53+
54+
## Schema validation tool
55+
56+
### Description
57+
58+
The tool performs a number of checks on all telemetry collectors and verifies the following:
59+
60+
1. Verifies the collector structure, fields, and returned values are using the appropriate types.
61+
2. Verifies that the collector `fetch` function Type matches the specified `schema` in the collector.
62+
3. Verifies that the collector `schema` matches the stored json schema .
63+
64+
### Notes
65+
66+
We don't catch every possible misuse of the collectors, but only the most common and critical ones.
67+
68+
What will not be caught by the validator:
69+
70+
* Mistyped SavedObject/CallCluster return value. Since the hits returned from ES can be typed to anything without any checks. It is advised to add functional tests that grabs the schema json file and checks that the returned usage matches the types exactly.
71+
72+
* Fields in the schema that are never collected. If you are trying to report a field from ES but that value is never stored in ES, the check will not be able to detect if that field is ever collected in the first palce. It is advised to add unit/functional tests to check that all the fields are being reported as expected.
73+
74+
The tool looks for `.telemetryrc.json` files in the root of the project and in the `x-pack` dir for its runtime configurations.
75+
76+
Currently auto-fixer (`--fix`) can automatically fix the json files with the following errors:
77+
78+
* incompatible schema - this error means that the collector schema was changed but the stored json schema file was not updated.
79+
80+
* unused schemas - this error means that a collector was removed or its `type` renamed, the json schema file contains a schema that does not have a corrisponding collector.
81+
82+
### Usage
83+
84+
```bash
85+
node scripts/telemetry_check --fix
86+
```
87+
88+
* `--path` specifies a collector path instead of checking all collectors specified in the `.telemetryrc.json` files. Accepts a `.ts` file. The file must be discoverable by at least one rc file.
89+
* `--fix` tells the tool to try to fix as many violations as possible. All errors that tool won't be able to fix will be reported.
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed to Elasticsearch B.V. under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch B.V. licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
module.exports = {
21+
presets: ['@kbn/babel-preset/node_preset'],
22+
ignore: ['**/*.test.ts', '**/__fixture__/**'],
23+
};
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"name": "@kbn/telemetry-tools",
3+
"version": "1.0.0",
4+
"license": "Apache-2.0",
5+
"main": "./target/index.js",
6+
"private": true,
7+
"scripts": {
8+
"build": "babel src --out-dir target --delete-dir-on-start --extensions .ts --source-maps=inline",
9+
"kbn:bootstrap": "yarn build",
10+
"kbn:watch": "yarn build --watch"
11+
},
12+
"devDependencies": {
13+
"lodash": "npm:@elastic/[email protected]",
14+
"@kbn/dev-utils": "1.0.0",
15+
"@kbn/utility-types": "1.0.0",
16+
"@types/normalize-path": "^3.0.0",
17+
"normalize-path": "^3.0.0",
18+
"@types/lodash": "^3.10.1",
19+
"moment": "^2.24.0",
20+
"typescript": "3.9.5"
21+
}
22+
}
Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/*
2+
* Licensed to Elasticsearch B.V. under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch B.V. licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
import Listr from 'listr';
21+
import chalk from 'chalk';
22+
import { createFailError, run } from '@kbn/dev-utils';
23+
24+
import {
25+
createTaskContext,
26+
ErrorReporter,
27+
parseConfigsTask,
28+
extractCollectorsTask,
29+
checkMatchingSchemasTask,
30+
generateSchemasTask,
31+
checkCompatibleTypesTask,
32+
writeToFileTask,
33+
TaskContext,
34+
} from '../tools/tasks';
35+
36+
export function runTelemetryCheck() {
37+
run(
38+
async ({ flags: { fix = false, path }, log }) => {
39+
if (typeof fix !== 'boolean') {
40+
throw createFailError(`${chalk.white.bgRed(' TELEMETRY ERROR ')} --fix can't have a value`);
41+
}
42+
43+
if (typeof path === 'boolean') {
44+
throw createFailError(`${chalk.white.bgRed(' TELEMETRY ERROR ')} --path require a value`);
45+
}
46+
47+
if (fix && typeof path !== 'undefined') {
48+
throw createFailError(
49+
`${chalk.white.bgRed(' TELEMETRY ERROR ')} --fix is incompatible with --path flag.`
50+
);
51+
}
52+
53+
const list = new Listr([
54+
{
55+
title: 'Checking .telemetryrc.json files',
56+
task: () => new Listr(parseConfigsTask(), { exitOnError: true }),
57+
},
58+
{
59+
title: 'Extracting Collectors',
60+
task: (context) => new Listr(extractCollectorsTask(context, path), { exitOnError: true }),
61+
},
62+
{
63+
title: 'Checking Compatible collector.schema with collector.fetch type',
64+
task: (context) => new Listr(checkCompatibleTypesTask(context), { exitOnError: true }),
65+
},
66+
{
67+
title: 'Checking Matching collector.schema against stored json files',
68+
task: (context) => new Listr(checkMatchingSchemasTask(context), { exitOnError: true }),
69+
},
70+
{
71+
enabled: (_) => fix,
72+
skip: ({ roots }: TaskContext) => {
73+
return roots.every(({ esMappingDiffs }) => !esMappingDiffs || !esMappingDiffs.length);
74+
},
75+
title: 'Generating new telemetry mappings',
76+
task: (context) => new Listr(generateSchemasTask(context), { exitOnError: true }),
77+
},
78+
{
79+
enabled: (_) => fix,
80+
skip: ({ roots }: TaskContext) => {
81+
return roots.every(({ esMappingDiffs }) => !esMappingDiffs || !esMappingDiffs.length);
82+
},
83+
title: 'Updating telemetry mapping files',
84+
task: (context) => new Listr(writeToFileTask(context), { exitOnError: true }),
85+
},
86+
]);
87+
88+
try {
89+
const context = createTaskContext();
90+
await list.run(context);
91+
} catch (error) {
92+
process.exitCode = 1;
93+
if (error instanceof ErrorReporter) {
94+
error.errors.forEach((e: string | Error) => log.error(e));
95+
} else {
96+
log.error('Unhandled exception!');
97+
log.error(error);
98+
}
99+
}
100+
process.exit();
101+
},
102+
{
103+
flags: {
104+
allowUnexpected: true,
105+
guessTypesForUnexpectedFlags: true,
106+
},
107+
}
108+
);
109+
}

0 commit comments

Comments
 (0)