normalize fortran identifiers like gfortran

gridaphobe · gridaphobe · commit 1bfa324d54cb · 2018-04-20T16:20:54.000-04:00
diff --git a/src/org/opensolaris/opengrok/analysis/FileAnalyzer.java b/src/org/opensolaris/opengrok/analysis/FileAnalyzer.java
@@ -360,4 +360,8 @@ protected TokenStream normalize(String fieldName, TokenStream in) {
                 return new LowerCaseFilter(in);
         }
     }
+
+    public Definitions normalizeDefinitions(Definitions defs) {
+        return defs;
+    }
 }
diff --git a/src/org/opensolaris/opengrok/analysis/JFlexSymbolMatcher.java b/src/org/opensolaris/opengrok/analysis/JFlexSymbolMatcher.java
@@ -463,7 +463,24 @@ protected void onScopeChanged(ScopeAction action, String str, int start) {
      */
     protected boolean onFilteredSymbolMatched(String str, int start,
         Set<String> keywords) {
-        return onFilteredSymbolMatched(str, start, keywords, true);
+        return onFilteredSymbolMatched(str, str, start, keywords, true);
+    }
+
+    /**
+     * Calls
+     * {@link #onFilteredSymbolMatched(java.lang.String, int, java.util.Set, boolean)}
+     * with {@code str}, {@code start}, {@code keywords}, and {@code true}.
+     * @param literal the literal representation of the symbol
+     * @param str the text string
+     * @param start the text start position
+     * @param keywords an optional set to search for {@code str} as a member to
+     * indicate a keyword
+     * @return true if the {@code str} was not in {@code keywords} or if
+     * {@code keywords} was null
+     */
+    protected boolean onFilteredSymbolMatched(String literal, String str, int start,
+        Set<String> keywords) {
+        return onFilteredSymbolMatched(literal, str, start, keywords, true);
     }
 
     /**
@@ -483,15 +500,36 @@ protected boolean onFilteredSymbolMatched(String str, int start,
      */
     protected boolean onFilteredSymbolMatched(String str, int start,
         Set<String> keywords, boolean caseSensitive) {
+        return onFilteredSymbolMatched(str, str, start, keywords, caseSensitive);
+    }
+
+    /**
+     * Raises {@link #onKeywordMatched(java.lang.String, int)} if
+     * {@code keywords} is not null and {@code str} is found as a member (in a
+     * case-sensitive or case-less search per {@code caseSensitive}); otherwise
+     * raises {@link #onSymbolMatched(java.lang.String, int)}.
+     * @param literal the literal representation of the symbol
+     * @param str the text string
+     * @param start the text start position
+     * @param keywords an optional set to search for {@code str} as a member to
+     * indicate a keyword
+     * @param caseSensitive a value indicating if {@code keywords} should be
+     * searched for {@code str} as-is ({@code true}) or if the lower-case
+     * equivalent of {@code str} should be used ({@code false}).
+     * @return true if the {@code str} was not in {@code keywords} or if
+     * {@code keywords} was null
+     */
+    protected boolean onFilteredSymbolMatched(String literal, String str, int start,
+        Set<String> keywords, boolean caseSensitive) {
 
         if (keywords != null) {
-            String check = caseSensitive ? str : str.toLowerCase(Locale.ROOT);
+            String check = caseSensitive ? literal : literal.toLowerCase(Locale.ROOT);
             if (keywords.contains(check)) {
-                onKeywordMatched(str, start);
+                onKeywordMatched(literal, start);
                 return false;
             }
         }
-        onSymbolMatched(str, start);
+        onSymbolMatched(literal, str, start);
         return true;
     }
 
diff --git a/src/org/opensolaris/opengrok/analysis/JFlexXrefUtils.java b/src/org/opensolaris/opengrok/analysis/JFlexXrefUtils.java
@@ -277,7 +277,7 @@ public static boolean writeSymbol(Writer out, Definitions defs,
 
         String[] strs = new String[1];
         strs[0] = "";
-        String check = caseSensitive ? symbol : symbol.toLowerCase(Locale.ROOT);
+        String check = caseSensitive ? literal : literal.toLowerCase(Locale.ROOT);
         if (isKeyword || (keywords != null && keywords.contains( check ))) {
             // This is a keyword, so we don't create a link.
             out.append("<b>");
diff --git a/src/org/opensolaris/opengrok/analysis/fortran/FortranAnalyzer.java b/src/org/opensolaris/opengrok/analysis/fortran/FortranAnalyzer.java
@@ -24,6 +24,8 @@
 package org.opensolaris.opengrok.analysis.fortran;
 
 import java.io.Reader;
+import java.util.function.Function;
+import org.opensolaris.opengrok.analysis.Definitions;
 import org.opensolaris.opengrok.analysis.FileAnalyzer;
 import org.opensolaris.opengrok.analysis.JFlexTokenizer;
 import org.opensolaris.opengrok.analysis.JFlexXref;
@@ -36,6 +38,10 @@
  */
 public class FortranAnalyzer extends AbstractSourceCodeAnalyzer {
 
+    public static final Function<String, String> NORMALIZE = (id) -> {
+        return id.toLowerCase() + "_";
+    };
+
     FortranAnalyzer(FortranAnalyzerFactory factory) {
         super(factory, new JFlexTokenizer(new FortranSymbolTokenizer(
             FileAnalyzer.dummyReader)));
@@ -50,4 +56,15 @@ public class FortranAnalyzer extends AbstractSourceCodeAnalyzer {
     protected JFlexXref newXref(Reader reader) {
         return new JFlexXref(new FortranXref(reader), getFactory().getEnv());
     }
+
+    @Override
+    public Definitions normalizeDefinitions(Definitions oldDefs) {
+        Definitions defs = new Definitions();
+        for (Definitions.Tag tag : oldDefs.getTags()) {
+            defs.addTag(tag.line, NORMALIZE.apply(tag.symbol).intern(),
+                tag.type, tag.text, tag.namespace, tag.signature,
+                tag.lineStart, tag.lineEnd);
+        }
+        return defs;
+    }
 }
diff --git a/src/org/opensolaris/opengrok/analysis/fortran/FortranSymbolTokenizer.lex b/src/org/opensolaris/opengrok/analysis/fortran/FortranSymbolTokenizer.lex
@@ -51,7 +51,7 @@ import org.opensolaris.opengrok.analysis.JFlexSymbolMatcher;
  ^[^ \t\f\r\n]+ { yybegin(SCOMMENT); }
 {Identifier} {String id = yytext();
     if (!Consts.kwd.contains(id.toLowerCase(Locale.ROOT))) {
-                        onSymbolMatched(id, yychar);
+                        onSymbolMatched(id, FortranAnalyzer.NORMALIZE.apply(id), yychar);
                         return yystate(); }
               }
 
diff --git a/src/org/opensolaris/opengrok/analysis/fortran/FortranXref.lex b/src/org/opensolaris/opengrok/analysis/fortran/FortranXref.lex
@@ -89,7 +89,7 @@ File = [a-zA-Z]{FNameChar}* ".inc"
     String id = yytext();
     // For historical reasons, FortranXref doesn't link identifiers of length=1
     if (id.length() > 1) {
-        onFilteredSymbolMatched(id, yychar, Consts.kwd, false);
+        onFilteredSymbolMatched(id, FortranAnalyzer.NORMALIZE.apply(id), yychar, Consts.kwd, false);
     } else {
         onNonSymbolMatched(id, yychar);
     }
diff --git a/src/org/opensolaris/opengrok/analysis/plain/PlainAnalyzer.java b/src/org/opensolaris/opengrok/analysis/plain/PlainAnalyzer.java
@@ -109,6 +109,7 @@ public void analyze(Document doc, StreamSource src, Writer xrefOut)
         if (fullpath != null && ctags != null) {
             defs = ctags.doCtags(fullpath);
             if (defs != null && defs.numberOfSymbols() > 0) {
+                defs = normalizeDefinitions(defs);
                 tryAddingDefs(doc, defs, src, fullpath);
                 byte[] tags = defs.serialize();
                 doc.add(new StoredField(QueryBuilder.TAGS, tags));                
diff --git a/test/org/opensolaris/opengrok/analysis/JFlexTokenizerTest.java b/test/org/opensolaris/opengrok/analysis/JFlexTokenizerTest.java
@@ -76,7 +76,8 @@ public void testOffsetAttribute() throws Exception {
         // create a text fragment that it understands
         testOffsetAttribute(FortranSymbolTokenizer.class,
                 "1 token1 = token2 + token3",
-                new String[]{"token1", "token2", "token3"});
+                new String[]{"token1", "token2", "token3"},
+                true);
     }
 
     /**
@@ -98,6 +99,11 @@ private void testOffsetAttribute(Class<? extends JFlexSymbolMatcher> klass)
     private void testOffsetAttribute(Class<? extends JFlexSymbolMatcher> klass,
             String inputText, String[] expectedTokens)
             throws Exception {
+        testOffsetAttribute(klass, inputText, expectedTokens, false);
+    }
+    private void testOffsetAttribute(Class<? extends JFlexSymbolMatcher> klass,
+            String inputText, String[] expectedTokens, Boolean matchPrefix)
+            throws Exception {
         JFlexSymbolMatcher matcher = klass.getConstructor(Reader.class).
             newInstance(new StringReader(inputText));
         JFlexTokenizer tokenizer = new JFlexTokenizer(matcher);
@@ -109,7 +115,11 @@ private void testOffsetAttribute(Class<? extends JFlexSymbolMatcher> klass,
         while (tokenizer.incrementToken()) {
             assertTrue("too many tokens", count < expectedTokens.length);
             String expected = expectedTokens[count];
-            assertEquals("term", expected, term.toString());
+            if (matchPrefix) {
+                assertEquals("term", term.toString().indexOf(expected), 0);
+            } else {
+                assertEquals("term", expected, term.toString());
+            }
             assertEquals("start",
                     inputText.indexOf(expected), offset.startOffset());
             assertEquals("end",
diff --git a/test/org/opensolaris/opengrok/analysis/fortran/FortranSymbolTokenizerTest.java b/test/org/opensolaris/opengrok/analysis/fortran/FortranSymbolTokenizerTest.java
@@ -27,8 +27,11 @@
 import java.io.BufferedReader;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.util.AbstractMap.SimpleEntry;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import static org.junit.Assert.assertNotNull;
 import org.junit.Test;
 import static org.opensolaris.opengrok.util.CustomAssertions.assertSymbolStream;
@@ -52,17 +55,23 @@ public void testFortranSymbolStream() throws Exception {
         assertNotNull("despite samplesymbols.txt as resource,", wdsres);
 
         List<String> expectedSymbols = new ArrayList<>();
+        Map<Integer, SimpleEntry<String, String>> overrides = new HashMap<>();
+        int i = 1;
         try (BufferedReader wdsr = new BufferedReader(new InputStreamReader(
             wdsres, "UTF-8"))) {
             String line;
             while ((line = wdsr.readLine()) != null) {
                 int hasho = line.indexOf('#');
                 if (hasho != -1) line = line.substring(0, hasho);
-                expectedSymbols.add(line.trim());
+                String literal = line.trim();
+                String symbol = FortranAnalyzer.NORMALIZE.apply(literal);
+                expectedSymbols.add(symbol);
+                overrides.put(i, new SimpleEntry(literal, symbol));
+                i++;
             }
         }
 
         assertSymbolStream(FortranSymbolTokenizer.class, fres,
-            expectedSymbols);
+            overrides, expectedSymbols);
     }
 }
diff --git a/test/org/opensolaris/opengrok/analysis/fortran/FortranXrefTest.java b/test/org/opensolaris/opengrok/analysis/fortran/FortranXrefTest.java
@@ -108,6 +108,9 @@ private int writeFortranXref(PrintStream oss, InputStream iss,
         analyzer.setFoldingEnabled(true);
         WriteXrefArgs wargs = new WriteXrefArgs(
             new InputStreamReader(iss, "UTF-8"), sw);
+        if (defs != null) {
+            defs = analyzer.normalizeDefinitions(defs);
+        }
         wargs.setDefs(defs);
         Xrefer xref = analyzer.writeXref(wargs);
         oss.print(sw.toString());
diff --git a/test/org/opensolaris/opengrok/analysis/fortran/sample_xref.html b/test/org/opensolaris/opengrok/analysis/fortran/sample_xref.html

Original file line number	Diff line number	Diff line change
`@@ -360,4 +360,8 @@ protected TokenStream normalize(String fieldName, TokenStream in) {`
`360`	`360`	`return new LowerCaseFilter(in);`
`361`	`361`	`}`
`362`	`362`	`}`
	`363`	`+`
	`364`	`+ public Definitions normalizeDefinitions(Definitions defs) {`
	`365`	`+ return defs;`
	`366`	`+ }`
`363`	`367`	`}`
Original file line number	Diff line number	Diff line change
`@@ -51,7 +51,7 @@ import org.opensolaris.opengrok.analysis.JFlexSymbolMatcher;`
`51`	`51`	`^[^ \t\f\r\n]+ { yybegin(SCOMMENT); }`
`52`	`52`	`{Identifier} {String id = yytext();`
`53`	`53`	`if (!Consts.kwd.contains(id.toLowerCase(Locale.ROOT))) {`
`54`		`- onSymbolMatched(id, yychar);`
	`54`	`+ onSymbolMatched(id, FortranAnalyzer.NORMALIZE.apply(id), yychar);`
`55`	`55`	`return yystate(); }`
`56`	`56`	`}`
`57`	`57`
Original file line number	Diff line number	Diff line change
`@@ -89,7 +89,7 @@ File = [a-zA-Z]{FNameChar}* ".inc"`
`89`	`89`	`String id = yytext();`
`90`	`90`	`// For historical reasons, FortranXref doesn't link identifiers of length=1`
`91`	`91`	`if (id.length() > 1) {`
`92`		`- onFilteredSymbolMatched(id, yychar, Consts.kwd, false);`
	`92`	`+ onFilteredSymbolMatched(id, FortranAnalyzer.NORMALIZE.apply(id), yychar, Consts.kwd, false);`
`93`	`93`	`} else {`
`94`	`94`	`onNonSymbolMatched(id, yychar);`
`95`	`95`	`}`