Prototype for EMF model instance language module

.gitattributes

@@ -1,2 +1,5 @@
 # Set the default behavior, in case people don't have core.autocrlf set.
 * text=auto
+
+*.emfatic text
+*.treeview text

.gitignore

@@ -33,3 +33,6 @@ hs_err_pid*
 # default result file name, should never be commited
 result/
 result.zip
+
+# macOS
+*.DS_Store

README.md

@@ -31,7 +31,8 @@ In the following, a list of all supported languages with their supported languag
 | [Scala](https://www.scala-lang.org)                    |  2.13.8 | scala             |                               beta                               | Scalameta |
 | [Scheme](http://www.scheme-reports.org)                |       ? | scheme            |                             unknown                              |  JavaCC   |
 | [Swift](https://www.swift.org)                         |     5.4 | swift             |                               beta                               |  ANTLR 4  |
-| [EMF Metamodel](https://www.eclipse.org/modeling/emf/) |  2.25.0 | emf               |                              alpha                               |    EMF    |
+| [EMF Metamodel](https://www.eclipse.org/modeling/emf/) |  2.25.0 | emf               |                               beta                               |    EMF    |
+| [EMF Model](https://www.eclipse.org/modeling/emf/)     |  2.25.0 | emf-model         |                              alpha                               |    EMF    |
 | Text (naive)                                           |       - | text              |                              legacy                              |  CoreNLP  |
 
 ## Download and Installation

cli/pom.xml

@@ -97,6 +97,11 @@
             <artifactId>emf-metamodel</artifactId>
             <version>${revision}</version>
         </dependency>
+        <dependency>
+            <groupId>de.jplag</groupId>
+            <artifactId>emf-model</artifactId>
+            <version>${revision}</version>
+        </dependency>
         <!-- CLI -->
         <dependency>
             <groupId>org.kohsuke.metainf-services</groupId>

cli/src/test/java/de/jplag/cli/LanguageTest.java

@@ -28,7 +28,7 @@ void testInvalidLanguage() throws Exception {
     @Test
     void testLoading() {
         var languages = LanguageLoader.getAllAvailableLanguages();
-        assertEquals(15, languages.size(), "Loaded Languages: " + languages.keySet());
+        assertEquals(16, languages.size(), "Loaded Languages: " + languages.keySet());
     }
 
     @Test

core/src/main/java/de/jplag/SubmissionSetBuilder.java

@@ -81,6 +81,13 @@ public SubmissionSet buildSubmissionSet() throws ExitException {
 
         // Merge everything in a submission set.
         List<Submission> submissions = new ArrayList<>(foundSubmissions.values());
+
+        // Some languages expect a certain order, which is ensured here:
+        if (options.language().expectsSubmissionOrder()) {
+            List<File> rootFiles = foundSubmissions.values().stream().map(it -> it.getRoot()).toList();
+            rootFiles = options.language().customizeSubmissionOrder(rootFiles);
+            submissions = new ArrayList<>(rootFiles.stream().map(foundSubmissions::get).toList());
+        }
         return new SubmissionSet(submissions, baseCodeSubmission.orElse(null), options);
     }
 

coverage-report/pom.xml

@@ -101,6 +101,16 @@
             <artifactId>emf-metamodel</artifactId>
             <version>${revision}</version>
         </dependency>
+        <dependency>
+            <groupId>de.jplag</groupId>
+            <artifactId>emf-metamodel-dynamic</artifactId>
+            <version>${revision}</version>
+        </dependency>
+        <dependency>
+            <groupId>de.jplag</groupId>
+            <artifactId>emf-model</artifactId>
+            <version>${revision}</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

language-api/src/main/java/de/jplag/Language.java

@@ -66,4 +66,21 @@ default boolean useViewFiles() {
     default String viewFileSuffix() {
         return "";
     }
+
+    /**
+     * Specifies if the submission order is relevant for this language.
+     * @return defaults to false.
+     */
+    default boolean expectsSubmissionOrder() {
+        return false;
+    }
+
+    /**
+     * Re-orders the provided submission according the requirements of the language.
+     * @param submissions is the list of submissions.
+     * @return the re-ordered list.
+     */
+    default List<File> customizeSubmissionOrder(List<File> submissions) {
+        return submissions;
+    }
 }

language-api/src/main/java/de/jplag/Token.java

@@ -46,6 +46,16 @@ public Token(TokenType type, File file, int line, int column, int length) {
         this.length = length;
     }
 
+    /**
+     * Creates a token with column and length information.
+     * @param type is the token type.
+     * @param file is the name of the source code file.
+     * @param trace is the tracing information of the token, meaning line, column, and length.
+     */
+    public Token(TokenType type, File file, TokenTrace trace) {
+        this(type, file, trace.line(), trace.column(), trace.length());
+    }
+
     /**
      * Creates a token with column, length and semantic information.
      * @param type is the token type.

language-api/src/main/java/de/jplag/TokenTrace.java

@@ -0,0 +1,17 @@
+package de.jplag;
+
+/**
+ * Tracing information to locate the corresponding code section of a token.
+ * @param line is the line index in the source code where the token resides. Index is 1-based.
+ * @param column is the column index, meaning where the token starts in the line. Index is 1-based.
+ * @param length is the length of the token in the source code.
+ */
+public record TokenTrace(int line, int column, int length) {
+
+    /**
+     * Creates a empty trace with line, column, and length set to {@link Token#NO_VALUE NO_VALUE}.
+     */
+    public TokenTrace() {
+        this(Token.NO_VALUE, Token.NO_VALUE, Token.NO_VALUE);
+    }
+}

languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/DynamicMetamodelToken.java

@@ -5,6 +5,7 @@
 
 import org.eclipse.emf.ecore.EObject;
 
+import de.jplag.TokenTrace;
 import de.jplag.TokenType;
 import de.jplag.emf.MetamodelToken;
 
@@ -15,7 +16,7 @@
 public class DynamicMetamodelToken extends MetamodelToken {
 
     public DynamicMetamodelToken(TokenType type, File file, EObject eObject) {
-        super(type, file, NO_VALUE, NO_VALUE, NO_VALUE, Optional.of(eObject));
+        super(type, file, new TokenTrace(), Optional.of(eObject));
     }
 
     public DynamicMetamodelToken(TokenType type, File file) {

languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/DynamicMetamodelTokenType.java

@@ -10,6 +10,7 @@ public DynamicMetamodelTokenType(EObject eObject) {
         this(eObject.eClass());
     }
 
+    @Override
     public String getDescription() {
         return eClass.getName();
     }

languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/Language.java

@@ -1,25 +1,33 @@
 package de.jplag.emf.dynamic;
 
-import org.kohsuke.MetaInfServices;
-
 import de.jplag.emf.dynamic.parser.DynamicEcoreParser;
+import de.jplag.emf.parser.EcoreParser;
 
 /**
  * Language for EMF metamodels from the Eclipse Modeling Framework (EMF). This language is based on a dynamically
  * created token set instead of a hand-picked one.
  * @author Timur Saglam
  */
-@MetaInfServices(de.jplag.Language.class)
-public class Language extends de.jplag.emf.Language {
+public class Language extends de.jplag.emf.Language { // currently not included in the CLI
     private static final String NAME = "EMF metamodels (dynamically created token set)";
     private static final String IDENTIFIER = "emf-dynamic";
 
     private static final int DEFAULT_MIN_TOKEN_MATCH = 10;
 
+    /**
+     * Creates an EMF language instance with a dynamic token parser.
+     */
     public Language() {
         super(new DynamicEcoreParser());
     }
 
+    /**
+     * Creates an EMF language instance with a custom token parser.
+     */
+    public Language(EcoreParser parser) {
+        super(parser);
+    }
+
     @Override
     public String getName() {
         return NAME;

languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/parser/DynamicEcoreParser.java

@@ -1,10 +1,12 @@
 package de.jplag.emf.dynamic.parser;
 
 import org.eclipse.emf.ecore.EObject;
+import org.eclipse.emf.ecore.resource.Resource;
 
+import de.jplag.TokenType;
 import de.jplag.emf.MetamodelToken;
 import de.jplag.emf.dynamic.DynamicMetamodelToken;
-import de.jplag.emf.dynamic.DynamicMetamodelTokenType;
+import de.jplag.emf.normalization.ModelSorter;
 import de.jplag.emf.parser.EcoreParser;
 import de.jplag.emf.util.AbstractMetamodelVisitor;
 
@@ -14,12 +16,24 @@
  */
 public class DynamicEcoreParser extends EcoreParser {
 
+    private final DynamicElementTokenizer tokenizer;
+
+    public DynamicEcoreParser() {
+        tokenizer = new DynamicElementTokenizer();
+    }
+
     @Override
     protected AbstractMetamodelVisitor createMetamodelVisitor() {
-        return new DynamicMetamodelTokenGenerator(this);
+        return new DynamicMetamodelTokenGenerator(this, tokenizer);
     }
 
-    public void addToken(DynamicMetamodelTokenType type, EObject source) {
+    @Override
+    protected void normalizeOrder(Resource modelResource) {
+        ModelSorter.sort(modelResource, tokenizer);
+    }
+
+    @Override
+    public void addToken(TokenType type, EObject source) {
         MetamodelToken token = new DynamicMetamodelToken(type, currentFile, source);
         MetamodelToken metadataEnrichedToken = treeView.convertToMetadataEnrichedToken(token);
         tokens.add(metadataEnrichedToken);

languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/parser/DynamicElementTokenizer.java

@@ -0,0 +1,38 @@
+package de.jplag.emf.dynamic.parser;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.eclipse.emf.ecore.EClass;
+import org.eclipse.emf.ecore.EObject;
+
+import de.jplag.TokenType;
+import de.jplag.emf.dynamic.DynamicMetamodelTokenType;
+import de.jplag.emf.parser.ModelingElementTokenizer;
+
+/**
+ * Tokenizes any {@link EObject} via its {@link EClass}. Tracks all known tokens.
+ */
+public class DynamicElementTokenizer implements ModelingElementTokenizer {
+
+    private final Set<TokenType> knownTokenTypes;
+
+    /**
+     * Creates the tokenizer, initially with an empty token set.
+     */
+    public DynamicElementTokenizer() {
+        knownTokenTypes = new HashSet<>();
+    }
+
+    @Override
+    public TokenType element2Token(EObject modelElement) {
+        DynamicMetamodelTokenType token = new DynamicMetamodelTokenType(modelElement);
+        knownTokenTypes.add(token);
+        return token;
+    }
+
+    @Override
+    public Set<TokenType> allTokenTypes() {
+        return Set.copyOf(knownTokenTypes);
+    }
+}

languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/parser/DynamicMetamodelTokenGenerator.java

@@ -2,7 +2,6 @@
 
 import org.eclipse.emf.ecore.EObject;
 
-import de.jplag.emf.dynamic.DynamicMetamodelTokenType;
 import de.jplag.emf.util.AbstractMetamodelVisitor;
 
 /**
@@ -11,19 +10,19 @@
  */
 public class DynamicMetamodelTokenGenerator extends AbstractMetamodelVisitor {
     private final DynamicEcoreParser parser;
+    private final DynamicElementTokenizer tokenizer;
 
     /**
      * Creates the visitor.
      * @param parser is the parser which receives the generated tokens.
      */
-    public DynamicMetamodelTokenGenerator(DynamicEcoreParser parser) {
-        super(false);
+    public DynamicMetamodelTokenGenerator(DynamicEcoreParser parser, DynamicElementTokenizer tokenizer) {
         this.parser = parser;
+        this.tokenizer = tokenizer;
     }
 
     @Override
     protected void visitEObject(EObject eObject) {
-        var tokenType = new DynamicMetamodelTokenType(eObject);
-        parser.addToken(tokenType, eObject);
+        parser.addToken(tokenizer.element2Token(eObject), eObject);
     }
 }

languages/emf-metamodel-dynamic/src/test/java/de/jplag/emf/dynamic/MinimalDynamicMetamodelTest.java

@@ -13,6 +13,7 @@
 
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -42,6 +43,7 @@ public void setUp() {
     }
 
     @Test
+    @DisplayName("Test tokens generated from example metamodels")
     void testBookstoreMetamodels() throws ParsingException {
         List<File> testFiles = Arrays.stream(TEST_SUBJECTS).map(path -> new File(BASE_PATH.toFile(), path)).toList();
         List<Token> result = language.parse(new HashSet<>(testFiles));

languages/emf-metamodel/pom.xml

@@ -12,7 +12,7 @@
         <dependency>
             <groupId>org.eclipse.emfatic</groupId>
             <artifactId>org.eclipse.emfatic.core</artifactId>
-            <version>1.0.0</version>
+            <version>${emfatic.version}</version>
         </dependency>
         <dependency>
             <groupId>org.eclipse.emf</groupId>

languages/emf-metamodel/src/main/java/de/jplag/emf/MetamodelToken.java

@@ -6,6 +6,7 @@
 import org.eclipse.emf.ecore.EObject;
 
 import de.jplag.Token;
+import de.jplag.TokenTrace;
 import de.jplag.TokenType;
 
 /**
@@ -22,8 +23,8 @@ public class MetamodelToken extends Token {
      * @param file is the source model file.
      * @param eObject is the corresponding eObject in the model from which this token was extracted.
      */
-    public MetamodelToken(MetamodelTokenType type, File file, EObject eObject) {
-        this(type, file, NO_VALUE, NO_VALUE, NO_VALUE, Optional.of(eObject));
+    public MetamodelToken(TokenType type, File file, EObject eObject) {
+        this(type, file, new TokenTrace(), Optional.of(eObject));
     }
 
     /**
@@ -32,20 +33,18 @@ public MetamodelToken(MetamodelTokenType type, File file, EObject eObject) {
      * @param file is the source model file.
      */
     public MetamodelToken(TokenType type, File file) {
-        this(type, file, NO_VALUE, NO_VALUE, NO_VALUE, Optional.empty());
+        this(type, file, new TokenTrace(), Optional.empty());
     }
 
     /**
      * Creates a token with column and length information.
      * @param type is the token type.
      * @param file is the source code file.
-     * @param line is the line index in the source code where the token resides. Cannot be smaller than 1.
-     * @param column is the column index, meaning where the token starts in the line.
-     * @param length is the length of the token in the source code.
+     * @param trace is the tracing information of the token, meaning line, column, and length.
      * @param eObject is the corresponding eObject in the model from which this token was extracted
      */
-    public MetamodelToken(TokenType type, File file, int line, int column, int length, Optional<EObject> eObject) {
-        super(type, file, line, column, length);
+    public MetamodelToken(TokenType type, File file, TokenTrace trace, Optional<EObject> eObject) {
+        super(type, file, trace);
         this.eObject = eObject;
     }