... I S S U E # 9 9 9 : Add classes to support Objective-C analysis

Also, make H extension matching imprecise but
with definite C fallback.

Author: idodeclare

build.xml

@@ -303,6 +303,8 @@ Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
         <run-jflex dir="${gen.analysis.dir}/javascript" name="JavaScriptXref"/>
         <run-jflex dir="${gen.analysis.dir}/lua" name="LuaSymbolTokenizer"/>
         <run-jflex dir="${gen.analysis.dir}/lua" name="LuaXref"/>
+        <run-jflex dir="${gen.analysis.dir}/objectivec" name="ObjectiveCLexer"/>
+        <run-jflex dir="${gen.analysis.dir}/objectivec" name="ObjectiveCNamer"/>
         <run-jflex dir="${gen.analysis.dir}/powershell" name="PowershellSymbolTokenizer"/>
         <run-jflex dir="${gen.analysis.dir}/powershell" name="PowershellXref"/>
         <run-jflex dir="${gen.analysis.dir}/python" name="PythonSymbolTokenizer"/>

src/org/opensolaris/opengrok/analysis/AnalyzerGuru.java

@@ -97,6 +97,7 @@
 import org.opensolaris.opengrok.analysis.clojure.ClojureAnalyzerFactory;
 import org.opensolaris.opengrok.analysis.json.JsonAnalyzerFactory;
 import org.opensolaris.opengrok.analysis.kotlin.KotlinAnalyzerFactory;
+import org.opensolaris.opengrok.analysis.objectivec.ObjectiveCAnalyzerFactory;
 import org.opensolaris.opengrok.analysis.sh.ShAnalyzerFactory;
 import org.opensolaris.opengrok.analysis.powershell.PowershellAnalyzerFactory;
 import org.opensolaris.opengrok.analysis.ruby.RubyAnalyzerFactory;
@@ -278,7 +279,8 @@ public AnalyzerGuru(RuntimeEnvironment env) {
                 new JarAnalyzerFactory(env),
                 new ZipAnalyzerFactory(env),
                 new TarAnalyzerFactory(env),
-                new CAnalyzerFactory(env),
+                new ObjectiveCAnalyzerFactory(env), // Just before C.
+                new CAnalyzerFactory(env), // Just after Objective-C.
                 new CSharpAnalyzerFactory(env),
                 new VBAnalyzerFactory(env),
                 new CxxAnalyzerFactory(env),
@@ -344,15 +346,15 @@ public AnalyzerGuru(RuntimeEnvironment env) {
      * {@link FileAnalyzerFactory} subclasses are revised to target more or
      * different files.
      * @return a value whose lower 32-bits are a static value
-     * 20180212_07
+     * 20180214_00
      * for the current implementation and whose higher-32 bits are non-zero if
      * {@link #addExtension(java.lang.String, org.opensolaris.opengrok.analysis.FileAnalyzerFactory)}
      * or
      * {@link #addPrefix(java.lang.String, org.opensolaris.opengrok.analysis.FileAnalyzerFactory)}
      * has been called.
      */
     public long getVersionNo() {
-        final int ver32 = 20180212_07; // Edit comment above too!
+        final int ver32 = 20180214_00; // Edit comment above too!
         long ver = ver32;
         if (customizationHashCode != 0) {
             ver |= (long)customizationHashCode << 32;

src/org/opensolaris/opengrok/analysis/Ctags.java

@@ -34,6 +34,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.Function;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
@@ -52,6 +53,7 @@ public class Ctags implements Resettable {
 
     private final RuntimeEnvironment env;
     private volatile boolean closing;
+    private Function<String, String> matchReducer;
     private Process ctags;
     private Thread errThread;
     private OutputStreamWriter ctagsIn;
@@ -102,12 +104,29 @@ public void setCTagsExtraOptionsFile(String CTagsExtraOptionsFile) {
         this.CTagsExtraOptionsFile = CTagsExtraOptionsFile;
     }
 
+    /**
+     * Sets a function that is used as a last resort to reduce the text that is
+     * matched in searches against source text.
+     * <p>
+     * E.g., for Objective-C with multi-part method names that are interleaved
+     * with parameter types and names, a reducer could lop off parts of the
+     * method name for last-resort searching.
+     * <p>
+     * The reducer object can return {@code null} to indicate that no
+     * last-resort search should be done.
+     * @param obj a defined instance or {@code null}
+     */
+    public void setMatchReducer(Function<String, String> obj) {
+        matchReducer = obj;
+    }
+
     /**
      * Resets the instance for use for another file but without closing any
      * running ctags instance.
      */
     @Override
     public void reset() {
+        setMatchReducer(null);
         setTabSize(0);
     }
 
@@ -393,6 +412,7 @@ public Definitions doCtags(String file) throws IOException,
         }
 
         CtagsReader rdr = new CtagsReader();
+        rdr.setMatchReducer(matchReducer);
         rdr.setSplitterSupplier(() -> { return trySplitSource(file); });
         rdr.setTabSize(tabSize);
         Definitions ret;
@@ -469,6 +489,7 @@ public void destroy() {
         };
 
         CtagsReader rdr = new CtagsReader();
+        rdr.setMatchReducer(matchReducer);
         rdr.setTabSize(tabSize);
         readTags(rdr);
         Definitions ret = rdr.getDefinitions();

src/org/opensolaris/opengrok/analysis/CtagsReader.java

@@ -25,6 +25,7 @@
 package org.opensolaris.opengrok.analysis;
 
 import java.util.EnumMap;
+import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -91,6 +92,8 @@ public class CtagsReader {
     private boolean triedSplitterSupplier;
     private SourceSplitter splitter;
 
+    private Function<String, String> matchReducer;
+
     private int tabSize;
 
     /**
@@ -197,6 +200,22 @@ public void setSplitterSupplier(Supplier<SourceSplitter> obj) {
         splitterSupplier = obj;
     }
 
+    /**
+     * Sets a function that is used as a last resort to reduce the text that is
+     * matched in searches against source text.
+     * <p>
+     * E.g., for Objective-C with multi-part method names that are interleaved
+     * with parameter types and names, a reducer could lop off parts of the
+     * method name for last-resort searching.
+     * <p>
+     * The reducer object can return {@code null} to indicate that no
+     * last-resort search should be done.
+     * @param obj a defined instance or {@code null}
+     */
+    public void setMatchReducer(Function<String, String> obj) {
+        matchReducer = obj;
+    }
+
     /**
      * Reads a line into the instance's definitions.
      * @param tagLine a defined line or null to no-op
@@ -436,6 +455,13 @@ private CpatIndex bestIndexOfTag(int lineno, String whole, String str) {
             e = ExpandTabsReader.translate(whole, woff + str.length(), t);
             return new CpatIndex(lineno, s, e);
         }
+
+        if (matchReducer != null) {
+            String newStr = matchReducer.apply(str);
+            if (newStr != null && !newStr.equals(str)) {
+                return bestIndexOfTag(lineno, origWhole, newStr);
+            }
+        }
         /**
          * When ctags has truncated a pattern, or when it spans multiple lines,
          * then `str' might not be found in `whole'. In that case, return an
@@ -543,6 +569,12 @@ private CpatIndex bestIndexOfArg(int lineno, String whole, String arg) {
             }
         }
 
+        if (matchReducer != null) {
+            String newArg = matchReducer.apply(arg);
+            if (newArg != null && !newArg.equals(arg)) {
+                return bestIndexOfArg(lineno, whole, newArg);
+            }
+        }
         /**
          * When no match is found, return an imprecise index for the last
          * character as the best we can do.

src/org/opensolaris/opengrok/analysis/FileAnalyzer.java

@@ -29,6 +29,7 @@
 import java.io.StringReader;
 import java.io.Writer;
 import java.util.Locale;
+import java.util.function.Function;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -155,6 +156,16 @@ protected int getSpecializedVersionNo() {
         return 0; // FileAnalyzer is not specialized.
     }
 
+    /**
+     * Subclasses can override to return a function that is used as a last
+     * resort to reduce the text that is matched in searches for ctags positions
+     * w.r.t. source text.
+     * @return {@code null}
+     */
+    public Function<String, String> getMatchReducer() {
+        return null;
+    }
+
     public void setCtags(Ctags ctags) {
         this.ctags = ctags;
     }

src/org/opensolaris/opengrok/analysis/JFlexSymbolMatcher.java

@@ -126,6 +126,23 @@ protected void onSymbolMatched(String str, int start) {
         }
     }
 
+    /**
+     * Raises
+     * {@link SymbolMatchedListener#symbolMatched(org.opensolaris.opengrok.analysis.SymbolMatchedEvent)}
+     * for a subscribed listener.
+     * @param literal the literal representation of the symbol
+     * @param str the symbol string
+     * @param start the symbol literal start position
+     */
+    protected void onSymbolMatched(String literal, String str, int start) {
+        SymbolMatchedListener l = symbolListener;
+        if (l != null) {
+            SymbolMatchedEvent evt = new SymbolMatchedEvent(this, literal, str,
+                start, start + literal.length());
+            l.symbolMatched(evt);
+        }
+    }
+
     /**
      * Raises
      * {@link SymbolMatchedListener#sourceCodeSeen(org.opensolaris.opengrok.analysis.SourceCodeSeenEvent)}

src/org/opensolaris/opengrok/analysis/JFlexXref.java

@@ -210,7 +210,8 @@ public void setFoldingEnabled(boolean foldingEnabled) {
     public void symbolMatched(SymbolMatchedEvent evt) {
         try {
             JFlexXrefUtils.writeSymbol(out, defs, urlPrefix, project,
-                evt.getStr(), null, matcher.getLineNumber(), false, false);
+                evt.getLiteral(), evt.getStr(), null, matcher.getLineNumber(),
+                false, false);
         } catch (IOException ex) {
             throw new RuntimeException(ex);
         }

src/org/opensolaris/opengrok/analysis/JFlexXrefUtils.java

@@ -222,7 +222,11 @@ public static void writeEMailAddress(Writer out, String address,
     }
 
     /**
-     * Writes a symbol and generate links as appropriate.
+     * Calls
+     * {@link #writeSymbol(java.io.Writer, org.opensolaris.opengrok.analysis.Definitions, java.lang.String, org.opensolaris.opengrok.configuration.Project, java.lang.String, java.lang.String, java.util.Set, int, boolean, boolean)}
+     * with {@code out}, {@code defs}, {@code urlPrefix}, {@code project},
+     * {@code symbol} (twice), {@code keywords}, {@code line},
+     * {@code caseSensitive}, and {@code isKeyword}.
      *
      * @param out a defined, target instance
      * @param defs a possibly defined instance or null
@@ -241,16 +245,43 @@ public static void writeEMailAddress(Writer out, String address,
      */
     public static boolean writeSymbol(Writer out, Definitions defs,
         String urlPrefix, Project project, String symbol, Set<String> keywords,
-        int line, boolean caseSensitive, boolean isKeyword)
-            throws IOException {
+        int line, boolean caseSensitive, boolean isKeyword) throws IOException {
+
+        return writeSymbol(out, defs, urlPrefix, project, symbol, symbol,
+            keywords, line, caseSensitive, isKeyword);
+    }
+
+    /**
+     * Writes a symbol and generate links as appropriate.
+     *
+     * @param out a defined, target instance
+     * @param defs a possibly defined instance or null
+     * @param urlPrefix a defined instance
+     * @param project a possibly defined instance or null
+     * @param literal the literal representation of the symbol
+     * @param symbol the symbol to write
+     * @param keywords a set of keywords recognized by this analyzer (no links
+     * will be generated if the symbol is a keyword)
+     * @param line the line number on which the symbol appears
+     * @param caseSensitive Whether the keyword list is case sensitive
+     * @param isKeyword Whether the symbol is certainly a keyword without
+     * bothering to look up in a defined {@code keywords}
+     * @return true if the {@code symbol} was not in {@code keywords} or if
+     * {@code keywords} was null and if-and-only-if {@code isKeyword} is false
+     * @throws IOException if an error occurs while writing to the stream
+     */
+    public static boolean writeSymbol(Writer out, Definitions defs,
+        String urlPrefix, Project project, String literal, String symbol,
+        Set<String> keywords, int line, boolean caseSensitive,
+        boolean isKeyword) throws IOException {
+
         String[] strs = new String[1];
         strs[0] = "";
-
         String check = caseSensitive ? symbol : symbol.toLowerCase(Locale.ROOT);
         if (isKeyword || (keywords != null && keywords.contains( check ))) {
             // This is a keyword, so we don't create a link.
             out.append("<b>");
-            Util.htmlize(symbol, out);
+            Util.htmlize(literal, out);
             out.append("</b>");
             return false;
         }
@@ -296,10 +327,10 @@ public static boolean writeSymbol(Writer out, Definitions defs,
             out.append(" intelliWindow-symbol\"");
             out.append(" data-definition-place=\"def\"");
             out.append(">");
-            Util.htmlize(symbol, out);
+            Util.htmlize(literal, out);
             out.append("</a>");
         } else if (defs != null && defs.occurrences(symbol) == 1) {
-            writeSameFileLinkSymbol(out, symbol);
+            writeSameFileLinkSymbol(out, literal, symbol);
         } else {
             // This is a symbol that is not defined in this file, or a symbol
             // that is defined more than once in this file. In either case, we
@@ -314,22 +345,37 @@ public static boolean writeSymbol(Writer out, Definitions defs,
             out.append(" class=\"intelliWindow-symbol\"");
             out.append(" data-definition-place=\"undefined-in-file\"");
             out.append(">");
-            Util.htmlize(symbol, out);
+            Util.htmlize(literal, out);
             out.append("</a>");
         }
         return true;
     }
 
     /**
-     * Write a symbol with a link to its definition which is expected at
-     * exactly one location in the same file.
+     * Calls
+     * {@link #writeSameFileLinkSymbol(java.io.Writer, java.lang.String, java.lang.String)}
+     * with {@code out} and {@code symbol} (twice).
      * @param out a defined, target instance
      * @param symbol the symbol to write
      * @throws IOException if {@link Writer#append(java.lang.CharSequence)}
      * fails
      */
     public static void writeSameFileLinkSymbol(Writer out, String symbol)
             throws IOException {
+        writeSameFileLinkSymbol(out, symbol, symbol);
+    }
+
+    /**
+     * Write a symbol with a link to its definition which is expected at
+     * exactly one location in the same file.
+     * @param out a defined, target instance
+     * @param literal the literal representation of the symbol
+     * @param symbol the symbol to write
+     * @throws IOException if {@link Writer#append(java.lang.CharSequence)}
+     * fails
+     */
+    public static void writeSameFileLinkSymbol(Writer out, String literal,
+            String symbol) throws IOException {
         // This is a reference to a symbol defined exactly once in this file.
         String style_class = "d";
 
@@ -341,7 +387,7 @@ public static void writeSameFileLinkSymbol(Writer out, String symbol)
         out.append("\"");
         out.append(" data-definition-place=\"defined-in-file\"");
         out.append(">");
-        Util.htmlize(symbol, out);
+        Util.htmlize(literal, out);
         out.append("</a>");
     }