Cucumber language server, VSCode extension and Monaco editor (#1689)

* More LSP work

* More LSP/VSCode work

* Write dry run messages

* Extract LSP server

* Fix test

* Prettier

* Add vscode extension

* More vscode

* Update index automatically

* warnings for undefined steps

* Fix CI

* Start implementing semantic tokens. Refactor to lsp lib.

* Implement semantic tokens

* Hightlight parameters

* Fix build

* highlight docstrings

* Refactor makeToken

* Highlight tables

* Semantic tokens for tagds

* highlight <placeholders>

* Refactor

* Ignore Scenario Outline steps

* Refactor

* Misc improvements

* Handle parse errors

* tweak colours

* Initialize index and expressions

* Extract new @cucumber/language-service library

* Misc updates

* Add Monaco plugin

* Build docs

* Improve diagnostics

* Documentation

* Add dummy test

.circleci/config.yml

@@ -191,6 +191,54 @@ jobs:
             cd suggest/javascript
             make
 
+  language-service-javascript:
+    executor: docker-circleci-node-14
+    steps:
+      - attach_workspace:
+          at: '~/cucumber'
+      - run:
+          name: language-service/javascript
+          command: |
+            sudo npm install --global npm@7
+            cd language-service/javascript
+            make
+
+  language-server-javascript:
+    executor: docker-circleci-node-14
+    steps:
+      - attach_workspace:
+          at: '~/cucumber'
+      - run:
+          name: language-server/javascript
+          command: |
+            sudo npm install --global npm@7
+            cd language-server/javascript
+            make
+
+  monaco-javascript:
+    executor: docker-circleci-node-14
+    steps:
+      - attach_workspace:
+          at: '~/cucumber'
+      - run:
+          name: monaco/javascript
+          command: |
+            sudo npm install --global npm@7
+            cd monaco/javascript
+            make
+
+  vscode-javascript:
+    executor: docker-circleci-node-14
+    steps:
+      - attach_workspace:
+          at: '~/cucumber'
+      - run:
+          name: vscode/javascript
+          command: |
+            sudo npm install --global npm@7
+            cd vscode/javascript
+            make
+
   message-streams-javascript:
     executor: docker-circleci-node-14
     steps:
@@ -917,6 +965,18 @@ workflows:
       - suggest-javascript:
           requires:
             - prepare-parallel
+      - language-service-javascript:
+          requires:
+            - prepare-parallel
+      - language-server-javascript:
+          requires:
+            - prepare-parallel
+      - monaco-javascript:
+          requires:
+            - prepare-parallel
+      - vscode-javascript:
+          requires:
+            - prepare-parallel
       - message-streams-javascript:
           requires:
             - prepare-parallel

.templates/javascript/.gitignore

@@ -16,3 +16,4 @@ acceptance/
 storybook-static
 *-go
 *.iml
+.vscode-test

Makefile

@@ -7,6 +7,10 @@ PACKAGES ?= messages \
 	gherkin-utils \
 	cucumber-expressions \
 	suggest \
+	language-service \
+	language-server \
+	monaco \
+	vscode \
 	tag-expressions \
 	create-meta \
 	fake-cucumber \

compatibility-kit/javascript/.gitignore

@@ -16,3 +16,4 @@ acceptance/
 storybook-static
 *-go
 *.iml
+.vscode-test

create-meta/javascript/.gitignore

@@ -16,3 +16,4 @@ acceptance/
 storybook-static
 *-go
 *.iml
+.vscode-test

cucumber-expressions/javascript/.gitignore

@@ -16,3 +16,4 @@ acceptance/
 storybook-static
 *-go
 *.iml
+.vscode-test

fake-cucumber/javascript/.gitignore

@@ -16,3 +16,4 @@ acceptance/
 storybook-static
 *-go
 *.iml
+.vscode-test

gherkin-streams/javascript/.gitignore

@@ -16,3 +16,4 @@ acceptance/
 storybook-static
 *-go
 *.iml
+.vscode-test

gherkin-utils/javascript/.gitignore

@@ -16,3 +16,4 @@ acceptance/
 storybook-static
 *-go
 *.iml
+.vscode-test

gherkin-utils/javascript/src/GherkinDocumentHandlers.ts

@@ -0,0 +1,16 @@
+import * as messages from '@cucumber/messages'
+
+export type GherkinDocumentHandlers<Acc> = {
+  feature: (feature: messages.Feature, acc: Acc) => Acc
+  background: (backgrounf: messages.Background, acc: Acc) => Acc
+  rule: (rule: messages.Rule, acc: Acc) => Acc
+  scenario: (scenario: messages.Scenario, acc: Acc) => Acc
+  step: (step: messages.Step, acc: Acc) => Acc
+  examples: (examples: messages.Examples, acc: Acc) => Acc
+  tag: (tag: messages.Tag, acc: Acc) => Acc
+  comment: (comment: messages.Comment, acc: Acc) => Acc
+  dataTable: (dataTable: messages.DataTable, acc: Acc) => Acc
+  tableRow: (tableRow: messages.TableRow, acc: Acc) => Acc
+  tableCell: (tableCell: messages.TableCell, acc: Acc) => Acc
+  docString: (docString: messages.DocString, acc: Acc) => Acc
+}

gherkin-utils/javascript/src/GherkinDocumentWalker.ts

@@ -1,3 +1,4 @@
+// This file is DEPRECATED - use ./walkGherkinDocument instead
 import * as messages from '@cucumber/messages'
 
 export interface IFilters {

gherkin-utils/javascript/src/index.ts

@@ -1,3 +1,5 @@
+export * from './walkGherkinDocument'
+export * from './GherkinDocumentHandlers'
 import pretty from './pretty'
 import Query from './Query'
 import GherkinDocumentWalker, { rejectAllFilters } from './GherkinDocumentWalker'

gherkin-utils/javascript/src/walkGherkinDocument.ts

@@ -0,0 +1,137 @@
+import * as messages from '@cucumber/messages'
+import { GherkinDocumentHandlers } from './GherkinDocumentHandlers'
+
+/**
+ * Walks a Gherkin Document, visiting each node depth first (in the order they appear in the source)
+ *
+ * @param gherkinDocument
+ * @param initialValue the initial value of the traversal
+ * @param handlers handlers for each node type, which may return a new value
+ * @return result the final value
+ */
+export function walkGherkinDocument<Acc>(
+  gherkinDocument: messages.GherkinDocument,
+  initialValue: Acc,
+  handlers: Partial<GherkinDocumentHandlers<Acc>>
+): Acc {
+  let acc = initialValue
+  const h: GherkinDocumentHandlers<Acc> = { ...makeDefaultHandlers<Acc>(), ...handlers }
+  const feature = gherkinDocument.feature
+  if (!feature) return acc
+  acc = walkTags(feature.tags || [], acc)
+  acc = h.feature(feature, acc)
+
+  for (const child of feature.children) {
+    if (child.background) {
+      acc = walkStepContainer(child.background, acc)
+    } else if (child.scenario) {
+      acc = walkStepContainer(child.scenario, acc)
+    } else if (child.rule) {
+      acc = walkTags(child.rule.tags || [], acc)
+      acc = h.rule(child.rule, acc)
+      for (const ruleChild of child.rule.children) {
+        if (ruleChild.background) {
+          acc = walkStepContainer(ruleChild.background, acc)
+        } else if (ruleChild.scenario) {
+          acc = walkStepContainer(ruleChild.scenario, acc)
+        }
+      }
+    }
+  }
+  return acc
+
+  function walkTags(tags: readonly messages.Tag[], acc: Acc): Acc {
+    return tags.reduce((acc, tag) => h.tag(tag, acc), acc)
+  }
+
+  function walkSteps(steps: readonly messages.Step[], acc: Acc): Acc {
+    return steps.reduce((acc, step) => walkStep(step, acc), acc)
+  }
+
+  function walkStep(step: messages.Step, acc: Acc): Acc {
+    acc = h.step(step, acc)
+    if (step.docString) {
+      acc = h.docString(step.docString, acc)
+    }
+    if (step.dataTable) {
+      acc = h.dataTable(step.dataTable, acc)
+      acc = walkTableRows(step.dataTable.rows, acc)
+    }
+    return acc
+  }
+
+  function walkTableRows(tableRows: readonly messages.TableRow[], acc: Acc): Acc {
+    return tableRows.reduce((acc, tableRow) => walkTableRow(tableRow, acc), acc)
+  }
+
+  function walkTableRow(tableRow: messages.TableRow, acc: Acc): Acc {
+    acc = h.tableRow(tableRow, acc)
+    return tableRow.cells.reduce((acc, tableCell) => h.tableCell(tableCell, acc), acc)
+  }
+
+  function walkStepContainer(
+    stepContainer: messages.Scenario | messages.Background,
+    acc: Acc
+  ): Acc {
+    const scenario: messages.Scenario = 'tags' in stepContainer ? stepContainer : null
+    acc = walkTags(scenario?.tags || [], acc)
+    acc = scenario
+      ? h.scenario(scenario, acc)
+      : h.background(stepContainer as messages.Background, acc)
+    acc = walkSteps(stepContainer.steps, acc)
+
+    if (scenario) {
+      for (const examples of scenario.examples || []) {
+        acc = walkTags(examples.tags || [], acc)
+        acc = h.examples(examples, acc)
+        if (examples.tableHeader) {
+          acc = walkTableRow(examples.tableHeader, acc)
+          acc = walkTableRows(examples.tableBody || [], acc)
+        }
+      }
+    }
+    return acc
+  }
+}
+
+function makeDefaultHandlers<Acc>() {
+  const defaultHandlers: GherkinDocumentHandlers<Acc> = {
+    feature(feature, acc) {
+      return acc
+    },
+    background(background, acc) {
+      return acc
+    },
+    rule(rule, acc) {
+      return acc
+    },
+    scenario(scenario, acc) {
+      return acc
+    },
+    step(step, acc) {
+      return acc
+    },
+    examples(examples, acc) {
+      return acc
+    },
+    tag(tag, acc) {
+      return acc
+    },
+    comment(comment, acc) {
+      return acc
+    },
+    dataTable(dataTable, acc) {
+      return acc
+    },
+    tableRow(tableRow, acc) {
+      return acc
+    },
+    tableCell(tableCell, acc) {
+      return acc
+    },
+    docString(docString, acc) {
+      return acc
+    },
+  }
+  return defaultHandlers
+}

gherkin-utils/javascript/test/GherkinDocumentWalkerTest.ts

@@ -377,7 +377,7 @@ Feature: hello
   context('handling objects', () => {
     describe('handleStep', () => {
       it('is called for each steps', () => {
-        const source = parse(`Feature: Solar System
+        const gherkinDocument = parse(`Feature: Solar System
 
         Scenario: Earth
           Given it is a planet
@@ -390,15 +390,15 @@ Feature: hello
             handleStep: (step) => stepText.push(step.text),
           }
         )
-        astWalker.walkGherkinDocument(source)
+        astWalker.walkGherkinDocument(gherkinDocument)
 
         assert.deepEqual(stepText, ['it is a planet'])
       })
     })
 
     describe('handleScenario', () => {
       it('is called for each scenarios', () => {
-        const source = parse(`Feature: Solar System
+        const gherkinDocument = parse(`Feature: Solar System
 
         Scenario: Earth
           Given it is a planet
@@ -414,15 +414,15 @@ Feature: hello
             handleScenario: (scenario) => scenarioName.push(scenario.name),
           }
         )
-        astWalker.walkGherkinDocument(source)
+        astWalker.walkGherkinDocument(gherkinDocument)
 
         assert.deepEqual(scenarioName, ['Earth', 'Saturn'])
       })
     })
 
     describe('handleBackground', () => {
       it('is called for each backgrounds', () => {
-        const source = parse(`Feature: Solar System
+        const gherkinDocument = parse(`Feature: Solar System
 
         Background: Milky Way
           Scenario: Earth
@@ -436,15 +436,15 @@ Feature: hello
             handleBackground: (background) => backgroundName.push(background.name),
           }
         )
-        astWalker.walkGherkinDocument(source)
+        astWalker.walkGherkinDocument(gherkinDocument)
 
         assert.deepEqual(backgroundName, ['Milky Way'])
       })
     })
 
     describe('handleRule', () => {
       it('is called for each rules', () => {
-        const source = parse(`Feature: Solar System
+        const gherkinDocument = parse(`Feature: Solar System
 
         Rule: On a planet
           Scenario: There is life
@@ -462,15 +462,15 @@ Feature: hello
             handleRule: (rule) => ruleName.push(rule.name),
           }
         )
-        astWalker.walkGherkinDocument(source)
+        astWalker.walkGherkinDocument(gherkinDocument)
 
         assert.deepEqual(ruleName, ['On a planet', 'On an exoplanet'])
       })
     })
 
     describe('handleFeature', () => {
       it('is called for each features', () => {
-        const source = parse(`Feature: Solar System
+        const gherkinDocument = parse(`Feature: Solar System
 
         Rule: On a planet
           Scenario: There is life
@@ -488,7 +488,7 @@ Feature: hello
             handleFeature: (feature) => featureName.push(feature.name),
           }
         )
-        astWalker.walkGherkinDocument(source)
+        astWalker.walkGherkinDocument(gherkinDocument)
 
         assert.deepEqual(featureName, ['Solar System'])
       })
@@ -497,10 +497,10 @@ Feature: hello
 
   describe('regression tests', () => {
     it('does not fail with empty/commented documents', () => {
-      const source = parse('# Feature: Solar System')
+      const gherkinDocument = parse('# Feature: Solar System')
       const astWalker = new GherkinDocumentWalker()
 
-      astWalker.walkGherkinDocument(source)
+      astWalker.walkGherkinDocument(gherkinDocument)
     })
   })
 })