support setting offset Source Information for grammar parser APIs

.github/workflows/database-bigquery-integration-test.yml

@@ -28,6 +28,9 @@ on:
       - master
 jobs:
   build:
+    # NOTE: we cannot run this action in fork because secrets are not accessible from forks
+    # See https://community.sonarsource.com/t/github-action-ci-build-fail-with-set-the-sonar-token-env-variable/38997
+    if: github.repository == 'finos/legend-engine'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo

.github/workflows/database-databricks-integration-test.yml

@@ -31,6 +31,9 @@ on:
       - master
 jobs:
   build:
+    # NOTE: we cannot run this action in fork because secrets are not accessible from forks
+    # See https://community.sonarsource.com/t/github-action-ci-build-fail-with-set-the-sonar-token-env-variable/38997
+    if: github.repository == 'finos/legend-engine'
     runs-on: ubuntu-latest
     steps:
       - name: Configure databricks CLI

.github/workflows/database-mssqlserver-integration-test.yml

@@ -28,6 +28,9 @@ on:
       - master
 jobs:
   build:
+    # NOTE: we cannot run this action in fork because secrets are not accessible from forks
+    # See https://community.sonarsource.com/t/github-action-ci-build-fail-with-set-the-sonar-token-env-variable/38997
+    if: github.repository == 'finos/legend-engine'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo

.github/workflows/database-redshift-integration-test.yml

@@ -29,6 +29,9 @@ on:
 jobs:
   build:
     name: Build
+    # NOTE: we cannot run this action in fork because secrets are not accessible from forks
+    # See https://community.sonarsource.com/t/github-action-ci-build-fail-with-set-the-sonar-token-env-variable/38997
+    if: github.repository == 'finos/legend-engine'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo

.github/workflows/database-redshift-sql-generation-integration-test.yml

@@ -29,6 +29,9 @@ on:
 jobs:
   build:
     name: Build
+    # NOTE: we cannot run this action in fork because secrets are not accessible from forks
+    # See https://community.sonarsource.com/t/github-action-ci-build-fail-with-set-the-sonar-token-env-variable/38997
+    if: github.repository == 'finos/legend-engine'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo

.github/workflows/database-snowflake-integration-test.yml

@@ -29,6 +29,9 @@ on:
       - refactor-integration-tests
 jobs:
   build:
+    # NOTE: we cannot run this action in fork because secrets are not accessible from forks
+    # See https://community.sonarsource.com/t/github-action-ci-build-fail-with-set-the-sonar-token-env-variable/38997
+    if: github.repository == 'finos/legend-engine'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo

.github/workflows/database-snowflake-sql-generation-integration-test.yml

@@ -28,6 +28,9 @@ on:
       - refactor-integration-tests
 jobs:
   build:
+    # NOTE: we cannot run this action in fork because secrets are not accessible from forks
+    # See https://community.sonarsource.com/t/github-action-ci-build-fail-with-set-the-sonar-token-env-variable/38997
+    if: github.repository == 'finos/legend-engine'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo

legend-engine-external-shared-format-runtime/src/test/java/org/finos/legend/engine/external/shared/runtime/test/TestExternalFormatQueries.java

@@ -93,7 +93,7 @@ protected String runTest(PureModelContextData generated, String grammar, String
             PureModel model = Compiler.compile(modelData, DeploymentMode.TEST, null);
 
             PureGrammarParser parser = PureGrammarParser.newInstance();
-            Lambda lambdaProtocol = parser.parseLambda(query, "query", true);
+            Lambda lambdaProtocol = parser.parseLambda(query);
             LambdaFunction<?> lambda = HelperValueSpecificationBuilder.buildLambda(lambdaProtocol.body, Lists.fixedSize.<Variable>empty(), model.getContext());
 
             ExecutionContext context = new Root_meta_pure_runtime_ExecutionContext_Impl(" ")._enableConstraints(true);

legend-engine-language-pure-grammar-api/src/main/java/org/finos/legend/engine/language/pure/grammar/api/grammarToJson/GrammarToJson.java

@@ -14,7 +14,12 @@
 import org.pac4j.core.profile.ProfileManager;
 import org.pac4j.jax.rs.annotations.Pac4JProfileManager;
 
-import javax.ws.rs.*;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import java.util.Map;
@@ -29,21 +34,30 @@ public class GrammarToJson extends GrammarAPI
     @ApiOperation(value = "Generates Pure protocol JSON from Pure language text")
     @Consumes({MediaType.TEXT_PLAIN, APPLICATION_ZLIB})
     @Produces(MediaType.APPLICATION_JSON)
-    public Response model(String input, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> pm, @DefaultValue("true") @QueryParam("returnSourceInfo") boolean returnSourceInfo)
+    public Response model(String text,
+                          @DefaultValue("") @ApiParam("The source ID to be used by the parser") @QueryParam("sourceId") String sourceId,
+                          @DefaultValue("0") @ApiParam("The line number the parser will offset by") @QueryParam("lineOffset") int lineOffset,
+                          @DefaultValue("true") @QueryParam("returnSourceInformation") boolean returnSourceInformation,
+                          @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> pm)
     {
         PureGrammarParserExtensions.logExtensionList();
-        return grammarToJson(input, (a, b) -> PureGrammarParser.newInstance().parseModel(a, b), pm, returnSourceInfo, "Grammar to Json : Model");
+        return grammarToJson(text, (a) -> PureGrammarParser.newInstance().parseModel(a, sourceId, lineOffset, 0, returnSourceInformation), pm, "Grammar to Json : Model");
     }
 
     @POST
     @Path("lambda")
     @ApiOperation(value = "Generates Pure protocol JSON from Pure language text")
     @Consumes({MediaType.TEXT_PLAIN, APPLICATION_ZLIB})
     @Produces(MediaType.APPLICATION_JSON)
-    public Response lambda(String input, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> pm, @DefaultValue ("true") @QueryParam("returnSourceInfo") boolean returnSourceInfo)
+    public Response lambda(String text,
+                           @DefaultValue("") @ApiParam("The source ID to be used by the parser") @QueryParam("sourceId") String sourceId,
+                           @DefaultValue("0") @ApiParam("The line number the parser will offset by") @QueryParam("lineOffset") int lineOffset,
+                           @DefaultValue("0") @ApiParam("The column number the parser will offset by") @QueryParam("columnOffset") int columnOffset,
+                           @DefaultValue("true") @QueryParam("returnSourceInformation") boolean returnSourceInformation,
+                           @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> pm)
     {
         PureGrammarParserExtensions.logExtensionList();
-        return grammarToJson(input, (a, b) -> PureGrammarParser.newInstance().parseLambda(a, "", b), pm, returnSourceInfo, "Grammar to Json : Lambda");
+        return grammarToJson(text, (a) -> PureGrammarParser.newInstance().parseLambda(a, sourceId, lineOffset, columnOffset, returnSourceInformation), pm, "Grammar to Json : Lambda");
     }
 
     // Required so that Jackson properly includes _type for the top level element
@@ -58,21 +72,26 @@ public TypedMap()
     @ApiOperation(value = "Generates Pure protocol JSON from Pure language text")
     @Consumes({MediaType.APPLICATION_JSON, APPLICATION_ZLIB})
     @Produces(MediaType.APPLICATION_JSON)
-    public Response lambdaBatch(Map<String, String> input, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> pm, @DefaultValue ("true") @QueryParam("returnSourceInfo") boolean returnSourceInfo)
+    public Response lambdaBatch(Map<String, ParserInput> input, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> pm)
     {
         PureGrammarParserExtensions.logExtensionList();
-        return grammarToJsonBatch(input, (a, b)-> PureGrammarParser.newInstance().parseLambda(a, "", b), new TypedMap(), pm, returnSourceInfo, "Grammar to Json : Lambda Batch");
+        return grammarToJsonBatch(input, (a, b, c, d, e)-> PureGrammarParser.newInstance().parseLambda(a, b, c, d, e), new TypedMap(), pm, "Grammar to Json : Lambda Batch");
     }
 
     @POST
     @Path("graphFetch")
     @ApiOperation(value = "Generates Pure protocol JSON from Pure language text")
     @Consumes({MediaType.TEXT_PLAIN, APPLICATION_ZLIB})
     @Produces(MediaType.APPLICATION_JSON)
-    public Response graphFetch(String input, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> pm, @DefaultValue ("true") @QueryParam("returnSourceInfo") boolean returnSourceInfo)
+    public Response graphFetch(String text,
+                               @DefaultValue("") @ApiParam("The source ID to be used by the parser") @QueryParam("sourceId") String sourceId,
+                               @DefaultValue("0") @ApiParam("The line number the parser will offset by") @QueryParam("lineOffset") int lineOffset,
+                               @DefaultValue("0") @ApiParam("The column number the parser will offset by") @QueryParam("columnOffset") int columnOffset,
+                               @DefaultValue("true") @QueryParam("returnSourceInformation") boolean returnSourceInformation,
+                               @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> pm)
     {
         PureGrammarParserExtensions.logExtensionList();
-        return grammarToJson(input, (a, b) -> PureGrammarParser.newInstance().parseGraphFetch(a, "", b), pm, returnSourceInfo, "Grammar to Json : GraphFetch");
+        return grammarToJson(text, (a) -> PureGrammarParser.newInstance().parseGraphFetch(a, sourceId, lineOffset, columnOffset, returnSourceInformation), pm, "Grammar to Json : GraphFetch");
     }
 
     // Required so that Jackson properly includes _type for the top level element
@@ -87,10 +106,10 @@ public TypedMapGraph()
     @ApiOperation(value = "Generates Pure protocol JSON from Pure language text")
     @Consumes({MediaType.APPLICATION_JSON, APPLICATION_ZLIB})
     @Produces(MediaType.APPLICATION_JSON)
-    public Response graphFetchBatch(Map<String, String> input, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> pm, @DefaultValue ("true") @QueryParam("returnSourceInfo") boolean returnSourceInfo)
+    public Response graphFetchBatch(Map<String, ParserInput> input, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> pm)
     {
         PureGrammarParserExtensions.logExtensionList();
-        return grammarToJsonBatch(input, (a, b)-> PureGrammarParser.newInstance().parseGraphFetch(a, "", b), new TypedMapGraph(), pm, returnSourceInfo, "Grammar to Json : GraphFetch Batch");
+        return grammarToJsonBatch(input, (a, b, c, d, e)-> PureGrammarParser.newInstance().parseGraphFetch(a, b, c, d, e), new TypedMapGraph(), pm, "Grammar to Json : GraphFetch Batch");
     }
 
 
@@ -99,10 +118,15 @@ public Response graphFetchBatch(Map<String, String> input, @ApiParam(hidden = tr
     @ApiOperation(value = "Generates Pure protocol JSON from Pure language text")
     @Consumes({MediaType.TEXT_PLAIN, APPLICATION_ZLIB})
     @Produces(MediaType.APPLICATION_JSON)
-    public Response valueSpecification(String input, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> pm, @DefaultValue ("true") @QueryParam("returnSourceInfo") boolean returnSourceInfo)
+    public Response valueSpecification(String text,
+                                       @DefaultValue("") @ApiParam("The source ID to be used by the parser") @QueryParam("sourceId") String sourceId,
+                                       @DefaultValue("0") @ApiParam("The line number the parser will offset by") @QueryParam("lineOffset") int lineOffset,
+                                       @DefaultValue("0") @ApiParam("The column number the parser will offset by") @QueryParam("columnOffset") int columnOffset,
+                                       @DefaultValue("true") @QueryParam("returnSourceInformation") boolean returnSourceInformation,
+                                       @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> pm)
     {
         PureGrammarParserExtensions.logExtensionList();
-        return grammarToJson(input, (a, b) -> PureGrammarParser.newInstance().parseValueSpecification(a, "", b), pm, returnSourceInfo, "Grammar to Json : Value Specification");
+        return grammarToJson(text, (a) -> PureGrammarParser.newInstance().parseValueSpecification(a, sourceId, lineOffset, columnOffset, returnSourceInformation), pm, "Grammar to Json : Value Specification");
     }
 
     // Required so that Jackson properly includes _type for the top level element
@@ -117,9 +141,9 @@ public TypedMapVS()
     @ApiOperation(value = "Generates Pure protocol JSON from Pure language text")
     @Consumes({MediaType.APPLICATION_JSON, APPLICATION_ZLIB})
     @Produces(MediaType.APPLICATION_JSON)
-    public Response valueSpecificationBatch(Map<String, String> input, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> pm, @DefaultValue ("true") @QueryParam("returnSourceInfo") boolean returnSourceInfo)
+    public Response valueSpecificationBatch(Map<String, ParserInput> input, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> pm)
     {
         PureGrammarParserExtensions.logExtensionList();
-        return grammarToJsonBatch(input, (a, b)-> PureGrammarParser.newInstance().parseValueSpecification(a, "", b), new TypedMapVS(), pm, returnSourceInfo, "Grammar to Json : Value Specification Batch");
+        return grammarToJsonBatch(input, (a, b, c, d, e)-> PureGrammarParser.newInstance().parseValueSpecification(a, b, c, d, e), new TypedMapVS(), pm, "Grammar to Json : Value Specification Batch");
     }
 }

legend-engine-language-pure-grammar-api/src/main/java/org/finos/legend/engine/language/pure/grammar/api/grammarToJson/TransformGrammarToJson.java

@@ -72,7 +72,7 @@ public Response transformGrammarToJson(GrammarToJsonInput grammarInput, @ApiPara
             {
                 try
                 {
-                    Lambda lambda = parser.parseLambda(value, key, returnSourceInfo);
+                    Lambda lambda = parser.parseLambda(value, key, 0, 0, returnSourceInfo);
                     lambdas.put(key, lambda);
                 }
                 catch (Exception e)
@@ -98,7 +98,7 @@ public Response transformGrammarToJson(GrammarToJsonInput grammarInput, @ApiPara
             {
                 try
                 {
-                    symmetricResult.modelDataContext = parser.parseModel(grammarInput.code, returnSourceInfo);
+                    symmetricResult.modelDataContext = parser.parseModel(grammarInput.code, "", 0, 0, returnSourceInfo);
                 }
                 catch (Exception e)
                 {