[Console] Generate autocomplete definitions from ES specification

packages/kbn-generate-console-definitions/README.md

@@ -1,10 +1,16 @@
 # Generate console definitions
-This package is a script to generate definitions used in Console to display autocomplete suggestions. The script is 
-a new implementation of `kbn-spec-to-console` package: The old script uses [JSON specs](https://github.com/elastic/elasticsearch/tree/main/rest-api-spec) from the Elasticsearch repo as the source, whereas this script uses the Elasticsearch specification [repo](https://github.com/elastic/elasticsearch-specification) as the source.
+This package is a script to generate definitions used in Console to display autocomplete suggestions. 
+The definitions files are generated from the Elasticsearch specification [repo](https://github.com/elastic/elasticsearch-specification).
+This script is 
+a new implementation of an old `kbn-spec-to-console` package: The old script used [JSON specs](https://github.com/elastic/elasticsearch/tree/main/rest-api-spec) in the Elasticsearch repo as the source.
 
 ## Instructions
 1. Checkout the Elasticsearch specification [repo](https://github.com/elastic/elasticsearch-specification).
 2. Run the command `node scripts/generate_console_definitions.js --source <ES_SPECIFICATION_REPO> --emptyDest`
-  This command will use the folder `<ES_SPECIFICATION_REPO>` as the source and the constant [`AUTOCOMPLETE_DEFINITIONS_FOLDER`](https://github.com/elastic/kibana/blob/main/src/plugins/console/common/constants/autocomplete_definitions.ts) as the destination. Based on the value of the constant, the autocomplete definitions will be generated in the folder `<KIBANA_REPO>/src/plugins/server/lib/spec_definitions/json/generated`. Using the flag `--emptyDest` will remove any existing files in the destination folder. 
+  This command will use the folder `<ES_SPECIFICATION_REPO>` as the source and the constant [`AUTOCOMPLETE_DEFINITIONS_FOLDER`](https://github.com/elastic/kibana/blob/main/src/plugins/console/common/constants/autocomplete_definitions.ts) as the destination. Based on the value of the constant, the autocomplete definitions will be generated in the folder `<KIBANA_REPO>/src/plugins/server/lib/spec_definitions/json/generated`. The flag `--emptyDest` indicates that all existing files in the destination folder will be removed. 
 3. It's possible to generate the definitions into a different folder. For that pass an option to the command `--dest <DEFINITIONS_FOLDER>` and also update the constant [`AUTOCOMPLETE_DEFINITIONS_FOLDER`](https://github.com/elastic/kibana/blob/main/src/plugins/console/common/constants/autocomplete_definitions.ts) so that the Console server will load the definitions from this folder. 
 
+## Functionality
+This script generates definitions for all endpoints defined in the ES specification at once. 
+The script generates fully functional autocomplete definition files with properties as described in the [Console README.md file](https://github.com/elastic/kibana/blob/main/src/plugins/console/README.md) except `data_autocomplete_rules`. Currently, this property needs to be written manually to add autocomplete suggestions for request body parameters.  
+

packages/kbn-generate-console-definitions/src/generate_availability.test.ts

@@ -21,149 +21,135 @@ describe('generateAvailability', () => {
     urls: [],
   };
 
-  it('converts empty availability to true', () => {
-    const endpoint = mockEndpoint;
+  const mockAvailability: SpecificationTypes.Availability = {
+    since: '7.7.0',
+    stability: SpecificationTypes.Stability.stable,
+  };
 
-    const availability = generateAvailability(endpoint);
-    expect(availability).toEqual({
-      stack: true,
-      serverless: true,
-    });
+  it('throws an error if `availability` if missing in the endpoint object', () => {
+    const endpointWithoutAvailability = {
+      ...mockEndpoint,
+      availability: undefined,
+    };
+
+    // @ts-expect-error according to types, availability is never missing
+    expect(() => generateAvailability(endpointWithoutAvailability)).toThrow(
+      'missing availability for test-endpoint'
+    );
   });
 
-  describe('converts correctly stack visibility', function () {
-    it('public visibility to true', () => {
+  describe('converts to false if the availability object is missing for either stack or serverless', () => {
+    it('if availability for stack is missing, the endpoint is not available there', () => {
       const endpoint = {
         ...mockEndpoint,
         availability: {
-          stack: {
-            visibility: SpecificationTypes.Visibility.public,
-          },
+          serverless: mockAvailability,
         },
       };
 
       const availability = generateAvailability(endpoint);
       expect(availability).toEqual({
-        stack: true,
+        stack: false,
         serverless: true,
       });
     });
 
-    it('private visibility to false', () => {
+    it('if availability for serverless is missing, the endpoint is not available there', () => {
       const endpoint = {
         ...mockEndpoint,
         availability: {
-          stack: {
-            visibility: SpecificationTypes.Visibility.private,
-          },
+          stack: mockAvailability,
         },
       };
 
       const availability = generateAvailability(endpoint);
       expect(availability).toEqual({
-        stack: false,
-        serverless: true,
+        stack: true,
+        serverless: false,
       });
     });
+  });
 
-    it('feature_flag visibility to false', () => {
+  describe('converts to true if the availability object is present and visibility is not set (public by default)', () => {
+    it('if availability for stack is set and its visibility is not set, the endpoint is available there', () => {
       const endpoint = {
         ...mockEndpoint,
         availability: {
-          stack: {
-            visibility: SpecificationTypes.Visibility.feature_flag,
-          },
+          stack: mockAvailability,
         },
       };
-
       const availability = generateAvailability(endpoint);
       expect(availability).toEqual({
-        stack: false,
-        serverless: true,
+        stack: true,
+        serverless: false,
       });
     });
 
-    it('missing visibility to true', () => {
+    it('if availability for serverless is set and its visibility is not set, the endpoint is available there', () => {
       const endpoint = {
         ...mockEndpoint,
         availability: {
-          stack: {},
+          serverless: mockAvailability,
         },
       };
-
       const availability = generateAvailability(endpoint);
       expect(availability).toEqual({
-        stack: true,
+        stack: false,
         serverless: true,
       });
     });
   });
 
-  describe('converts correctly serverless visibility', function () {
-    it('public visibility to true', () => {
+  describe('checks visibility value if the availability object is present and visibility is set', () => {
+    it('if visibility is set to public', () => {
       const endpoint = {
         ...mockEndpoint,
         availability: {
+          stack: { ...mockAvailability, visibility: SpecificationTypes.Visibility.public },
           serverless: {
+            ...mockAvailability,
             visibility: SpecificationTypes.Visibility.public,
           },
         },
       };
-
       const availability = generateAvailability(endpoint);
       expect(availability).toEqual({
         stack: true,
         serverless: true,
       });
     });
 
-    it('private visibility to false', () => {
+    it('if visibility is set to private', () => {
       const endpoint = {
         ...mockEndpoint,
         availability: {
-          serverless: {
-            visibility: SpecificationTypes.Visibility.private,
-          },
+          stack: { ...mockAvailability, visibility: SpecificationTypes.Visibility.private },
+          serverless: { ...mockAvailability, visibility: SpecificationTypes.Visibility.private },
         },
       };
-
       const availability = generateAvailability(endpoint);
       expect(availability).toEqual({
-        stack: true,
+        stack: false,
         serverless: false,
       });
     });
 
-    it('feature_flag visibility to false', () => {
+    it('if visibility is set to feature_flag', () => {
       const endpoint = {
         ...mockEndpoint,
         availability: {
+          stack: { ...mockAvailability, visibility: SpecificationTypes.Visibility.feature_flag },
           serverless: {
+            ...mockAvailability,
             visibility: SpecificationTypes.Visibility.feature_flag,
           },
         },
       };
-
       const availability = generateAvailability(endpoint);
       expect(availability).toEqual({
-        stack: true,
+        stack: false,
         serverless: false,
       });
     });
-
-    it('missing visibility to true', () => {
-      const endpoint = {
-        ...mockEndpoint,
-        availability: {
-          serverless: {},
-        },
-      };
-
-      const availability = generateAvailability(endpoint);
-      expect(availability).toEqual({
-        stack: true,
-        serverless: true,
-      });
-    });
   });
 });

packages/kbn-generate-console-definitions/src/generate_availability.ts

@@ -9,6 +9,34 @@
 import type { EndpointDescription } from '@kbn/console-plugin/common/types';
 import type { SpecificationTypes } from './types';
 
+/**
+ * Types important for this logic:
+ * export class Endpoint {
+ *   ...
+ *   availability: Availabilities
+ * }
+ * export class Availabilities {
+ *   stack?: Availability
+ *   serverless?: Availability
+ * }
+ * export class Availability {
+ *   ...
+ *   visibility?: Visibility
+ * }
+ * export enum Visibility {
+ *   public = 'public',
+ *   feature_flag = 'feature_flag',
+ *   private = 'private'
+ * }
+ *
+ * The property `availability` is required in the endpoint object according to types.
+ * Its properties `stack` and `serverless` are independent of each other.
+ *   - If `stack` or `serverless` property is missing in `availability`, the endpoint is NOT available there.
+ *   - If `stack` or `serverless` property is present
+ *     - If `visibility` is missing, its `public` by default -> the endpoint is available.
+ *     - If `visibility` is set to any value other than `public`-> the endpoint is not available.
+ */
+
 const DEFAULT_ENDPOINT_AVAILABILITY = true;
 
 export const generateAvailability = (
@@ -18,11 +46,33 @@ export const generateAvailability = (
     stack: DEFAULT_ENDPOINT_AVAILABILITY,
     serverless: DEFAULT_ENDPOINT_AVAILABILITY,
   };
-  if (endpoint.availability.stack?.visibility) {
-    availability.stack = endpoint.availability.stack?.visibility === 'public';
+  // availability is a required property of the endpoint
+  if (!endpoint.availability) {
+    throw new Error('missing availability for ' + endpoint.name);
   }
-  if (endpoint.availability.serverless?.visibility) {
-    availability.serverless = endpoint.availability.serverless?.visibility === 'public';
+  // if no availability object for stack, the endpoint is not available there
+  if (!endpoint.availability.stack) {
+    availability.stack = false;
+  } else {
+    // if the availability object for stack is present, check visibility property (public by default)
+    availability.stack =
+      // if visibility is missing, the endpoint is public by default
+      !endpoint.availability.stack.visibility ||
+      // if the visibility is set, anything other than public means not available
+      endpoint.availability.stack.visibility === 'public';
+  }
+  // the same logic for serverless
+
+  // if no availability object for serverless, the endpoint is not available there
+  if (!endpoint.availability.serverless) {
+    availability.serverless = false;
+  } else {
+    // if the availability object for serverless is present, check visibility property (public by default)
+    availability.serverless =
+      // if visibility is missing, the endpoint is public by default
+      !endpoint.availability.serverless.visibility ||
+      // if the visibility is set, anything other than public means not available
+      endpoint.availability.serverless.visibility === 'public';
   }
   return availability;
 };

packages/kbn-generate-console-definitions/src/helpers/convert_url_properties.ts

@@ -47,7 +47,7 @@ const convertValueOf = (
     return convertLiteralValue(valueOf);
   }
   // for query params we can ignore 'dictionary_of' and 'user_defined_value'
-  return '';
+  throw new Error('unexpected valueOf type ' + kind);
 };
 
 const convertInstanceOf = (

src/plugins/console/README.md

@@ -52,15 +52,15 @@ Kibana users benefit greatly from autocomplete suggestions since not all Elastic
 Autocomplete definitions are all created in the form of javascript objects loaded from `json` and `js` files. 
 
 ### Creating definitions
-The [`generated`](https://github.com/elastic/kibana/blob/main/src/plugins/console/server/lib/spec_definitions/json/generated) folder contains definitions created automatically from Elasticsearch REST API specifications. See this [README](https://github.com/elastic/kibana/blob/main/packages/kbn-spec-to-console/README.md) file for more information on the `spec-to-console` script. 
+The [`generated`](https://github.com/elastic/kibana/blob/main/src/plugins/console/server/lib/spec_definitions/json/generated) folder contains definitions created automatically from Elasticsearch specifications. See this [README](https://github.com/elastic/kibana/blob/main/packages/kbn-generate-console-definitions/README.md) file for more information on the `generate-console-definitions` script. The AppEx/Management team (@elastic/platform-deployment-management) regularly runs the script to update the definitions and is planning to automate this process. 
 
 Manually created override files in the [`overrides`](https://github.com/elastic/kibana/blob/main/src/plugins/console/server/lib/spec_definitions/json/overrides) folder contain additions for request body parameters since those
 are not created by the script. Any other fixes such as documentation links, request methods and patterns and url parameters 
-should be addressed at the source. That means this should be fixed in Elasticsearch REST API specifications and then 
+should be addressed at the source. That means this should be fixed in Elasticsearch specifications and then 
 autocomplete definitions can be re-generated with the script. 
 
 If there are any endpoints missing completely from the `generated` folder, this should also be addressed at the source, i.e. 
-Elasticsearch REST API specifications. If for some reason, that is not possible, then additional definitions files 
+Elasticsearch specifications. If for some reason, that is not possible, then additional definitions files 
 can be placed in the folder [`manual`]((https://github.com/elastic/kibana/blob/main/src/plugins/console/server/lib/spec_definitions/json/manual)).
 
 ### Top level keys
@@ -98,6 +98,15 @@ Query url parameters and their values. See the [Query url parameters](#query-url
 #### `priority`
 Value for selecting one autocomplete definition, if several configurations are loaded from the files. The highest number takes precedence.
 
+#### `availability`
+A property that describes if an endpoint is available in stack and serverless environments. Endpoints with a `false` boolean value are filtered out in the corresponding environment. An example of an endpoint that is not available in the serverless environment:
+```json
+"availability": {
+  "stack": true,
+  "serverless": false
+}
+```
+
 #### `data_autocomplete_rules`
 Request body parameters and their values. Only used in `overrides` files because REST API specs don't contain any information about body request parameters.
 Refer to Elasticsearch REST API documentation when configuring this object. See the [Request body parameters](#request-body-parameters) section below for more info. An example:

src/plugins/console/public/lib/kb/kb.js

@@ -124,6 +124,8 @@ export function setActiveApi(api) {
       dataType: 'json', // disable automatic guessing
       headers: {
         'kbn-xsrf': 'kibana',
+        // workaround for serverless
+        'x-elastic-internal-origin': 'Kibana',
       },
     }).then(
       function (data) {

src/plugins/console/server/lib/spec_definitions/json/generated/_internal.delete_desired_balance.json

@@ -6,6 +6,10 @@
     "patterns": [
       "_internal/desired_balance"
     ],
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/delete-desired-balance.html"
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/delete-desired-balance.html",
+    "availability": {
+      "stack": false,
+      "serverless": false
+    }
   }
 }

src/plugins/console/server/lib/spec_definitions/json/generated/_internal.delete_desired_nodes.json

@@ -6,6 +6,10 @@
     "patterns": [
       "_internal/desired_nodes"
     ],
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/delete-desired-nodes.html"
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/delete-desired-nodes.html",
+    "availability": {
+      "stack": false,
+      "serverless": false
+    }
   }
 }

src/plugins/console/server/lib/spec_definitions/json/generated/_internal.get_desired_balance.json

@@ -6,6 +6,10 @@
     "patterns": [
       "_internal/desired_balance"
     ],
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/get-desired-balance.html"
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/get-desired-balance.html",
+    "availability": {
+      "stack": false,
+      "serverless": false
+    }
   }
 }

src/plugins/console/server/lib/spec_definitions/json/generated/_internal.get_desired_nodes.json

@@ -6,6 +6,10 @@
     "patterns": [
       "_internal/desired_nodes/_latest"
     ],
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/get-desired-nodes.html"
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/get-desired-nodes.html",
+    "availability": {
+      "stack": false,
+      "serverless": false
+    }
   }
 }