Merge pull request #9 from nexB/ignore-anon-tags

Ignore anonymous tags in symbols_ctags

README.rst

@@ -17,10 +17,13 @@ This utility is designed to work on Linux and POSIX OS with these utilities:
 - xgettext that comes with GNU gettext.
 - universal ctags, version 5.9 or higher, built with JSON support.
 
- On Debian systems run this::
+On Debian systems run this::
 
     sudo apt-get install universal-ctags gettext
 
+On MacOS systems run this::
+
+    brew install universal-ctags gettext
 
 To get started:
 ~~~~~~~~~~~~~~~~

azure-pipelines.yml

@@ -27,30 +27,36 @@ jobs:
                  sudo apt-get install universal-ctags gettext
                  venv/bin/pytest -n 2 -vvs
 
-# - template: etc/ci/azure-posix.yml
-#   parameters:
-#       job_name: macos11_cpython
-#       image_name: macOS-11
-#       python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
-#       test_suites:
-#           all: venv/bin/pytest -n 2 -vvs
-#
-# - template: etc/ci/azure-posix.yml
-#   parameters:
-#       job_name: macos12_cpython
-#       image_name: macOS-12
-#       python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
-#       test_suites:
-#           all: venv/bin/pytest -n 2 -vvs
-#
-# - template: etc/ci/azure-posix.yml
-#   parameters:
-#       job_name: macos13_cpython
-#       image_name: macOS-13
-#       python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
-#       test_suites:
-#           all: venv/bin/pytest -n 2 -vvs
-#
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: macos11_cpython
+          image_name: macOS-11
+          python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
+          test_suites:
+              all: |
+                 brew install universal-ctags gettext
+                 venv/bin/pytest -n 2 -vvs
+
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: macos12_cpython
+          image_name: macOS-12
+          python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
+          test_suites:
+              all: |
+                 brew install universal-ctags gettext
+                 venv/bin/pytest -n 2 -vvs
+
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: macos13_cpython
+          image_name: macOS-13
+          python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
+          test_suites:
+              all: |
+                 brew install universal-ctags gettext
+                 venv/bin/pytest -n 2 -vvs
+
 # - template: etc/ci/azure-win.yml
 #   parameters:
 #       job_name: win2019_cpython

setup.cfg

@@ -51,8 +51,8 @@ where = src
 
 [options.entry_points]
 scancode_scan =
-    source_symbol = source_inpector.symbols_ctags:CtagsSymbolScannerPlugin
-    source_string = source_inpector.strings_xgettext:XgettextStringScannerPlugin
+    source_symbol = source_inspector.symbols_ctags:CtagsSymbolScannerPlugin
+    source_string = source_inspector.strings_xgettext:XgettextStringScannerPlugin
 
 
 [options.extras_require]

src/source_inpector/strings_xgettext.py → src/source_inspector/strings_xgettext.py

@@ -8,14 +8,12 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-import json
 import logging
 
 import attr
 from commoncode import command
 from commoncode.cliutils import SCAN_GROUP
 from commoncode.cliutils import PluggableCommandLineOption
-from normality import stringify
 from plugincode.scan import ScanPlugin
 from plugincode.scan import scan_impl
 

src/source_inpector/symbols_ctags.py → src/source_inspector/symbols_ctags.py

@@ -32,7 +32,7 @@ class CtagsSymbolScannerPlugin(ScanPlugin):
     """
 
     resource_attributes = dict(
-        symbols=attr.ib(default=attr.Factory(list), repr=False),
+        source_symbols=attr.ib(default=attr.Factory(list), repr=False),
     )
 
     options = [
@@ -57,7 +57,7 @@ def get_symbols(location, **kwargs):
     """
     Return a mapping of symbols for a source file at ``location``.
     """
-    return dict(symbols=list(collect_symbols(location=location)))
+    return dict(source_symbols=list(collect_symbols(location=location)))
 
 
 def collect_symbols(location):
@@ -93,11 +93,38 @@ def collect_symbols(location):
             continue
         tag = json.loads(line)
 
+        # Ignore the anonymous tags.
+        if (name := tag.get("name")) and is_anon_string(name):
+            continue
+
+        if (scope := tag.get("scope")) and "__anon" in scope:
+            tag["scope"] = "anonymous"
+
         # only keep some fields
         # see ctags --output-format=json --list-fields for a full list
         yield {k: v for k, v in tag.items() if k in supported_fields}
 
 
+def is_anon_string(symbol):
+    """
+    Check if a symbol is an anonymous string.
+    An anonymous symbol starts with `__anon` followed by a hexadecimal string.
+
+    Examples:
+    >>> is_anon_string("__anon722f7bb60308")
+    True
+    >>> is_anon_string("__anon_user_id")
+    False
+    """
+    if symbol.startswith("__anon"):
+        hex_part = symbol[6:]
+        for c in hex_part:
+            if c not in "0123456789abcdef":
+                return False
+        return True
+    return False
+
+
 _IS_CTAGS_INSTALLED = None
 
 

tests/data/symbols_ctags/if_ath.c-expected.json

@@ -3,7 +3,7 @@
     {
       "path": "if_ath.c",
       "type": "file",
-      "symbols": [
+      "source_symbols": [
         {
           "name": "AR5K_AR5212_STA_ID1",
           "pattern": "/^#define AR5K_AR5212_STA_ID1 /",
@@ -566,32 +566,6 @@
           "file": true,
           "kind": "macro"
         },
-        {
-          "name": "anonymous",
-          "pattern": "/^enum {$/",
-          "file": true,
-          "kind": "enum"
-        },
-        {
-          "name": "anonymous",
-          "pattern": "/^enum {$/",
-          "file": true,
-          "kind": "enum"
-        },
-        {
-          "name": "anonymous",
-          "pattern": "/^\tstatic const struct {$/",
-          "file": true,
-          "kind": "struct",
-          "scope": "ath_setcurmode",
-          "scopeKind": "function"
-        },
-        {
-          "name": "anonymous",
-          "pattern": "/^enum {$/",
-          "file": true,
-          "kind": "enum"
-        },
         {
           "name": "ar_device",
           "pattern": "/^ar_device(int devid)$/",

tests/data/symbols_ctags/test3.cpp-expected.json

@@ -3,7 +3,7 @@
     {
       "path": "test3.cpp",
       "type": "file",
-      "symbols": [
+      "source_symbols": [
         {
           "name": "EIP_IN_SIGNAL_FRAME",
           "pattern": "/^#define EIP_IN_SIGNAL_FRAME /",

tests/test_symbols_ctags.py

@@ -8,41 +8,18 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-import json
 import os
 
 from commoncode.testcase import FileBasedTesting
 from scancode.cli_test_utils import check_json_scan
 from scancode.cli_test_utils import run_scan_click
 
-from source_inpector.symbols_ctags import is_ctags_installed
+from source_inspector.symbols_ctags import is_ctags_installed
 
 # Used for tests to regenerate fixtures with regen=True
 REGEN_TEST_FIXTURES = os.getenv("SCANCODE_REGEN_TEST_FIXTURES", False)
 
 
-def clean_ctags(json_scan_file):
-    """
-    Clean a JSON ``json_scan_file`` file and save back in place.
-    """
-    with open(json_scan_file) as inp:
-        scan = json.load(inp)
-        for file in scan["files"]:
-            for sym in file["symbols"]:
-                # these change on each machine/version
-                scope = sym.get("scope")
-                if scope and "__anon" in scope:
-                    sym["scope"] = "anonymous"
-
-                # these change on each machine/version
-                name = sym.get("name")
-                if name and "__anon" in name:
-                    sym["name"] = "anonymous"
-
-    with open(json_scan_file, "w") as out:
-        json.dump(scan, out, indent=2)
-
-
 class TestCtagsSymbolScannerPlugin(FileBasedTesting):
 
     test_data_dir = os.path.join(os.path.dirname(__file__), "data/symbols_ctags")
@@ -55,7 +32,6 @@ def test_symbols_scanner_basic_cli_cpp(self):
         result_file = self.get_temp_file("json")
         args = ["--source-symbol", test_file, "--json-pp", result_file]
         run_scan_click(args)
-        clean_ctags(json_scan_file=result_file)
 
         expected_loc = self.get_test_loc("test3.cpp-expected.json")
         check_json_scan(expected_loc, result_file, regen=REGEN_TEST_FIXTURES)
@@ -65,7 +41,6 @@ def test_symbols_scanner_long_cli(self):
         result_file = self.get_temp_file("json")
         args = ["--source-symbol", test_file, "--json-pp", result_file]
         run_scan_click(args)
-        clean_ctags(json_scan_file=result_file)
 
         expected_loc = self.get_test_loc("if_ath.c-expected.json")
         check_json_scan(expected_loc, result_file, regen=REGEN_TEST_FIXTURES)

tests/test_symbols_xgettext.py

@@ -8,15 +8,14 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-import json
 import os
 
 from commoncode.testcase import FileBasedTesting
 from scancode.cli_test_utils import check_json_scan
 from scancode.cli_test_utils import run_scan_click
 
-from source_inpector.strings_xgettext import is_xgettext_installed
-from source_inpector.strings_xgettext import parse_po_text
+from source_inspector.strings_xgettext import is_xgettext_installed
+from source_inspector.strings_xgettext import parse_po_text
 
 # Used for tests to regenerate fixtures with regen=True
 REGEN_TEST_FIXTURES = os.getenv("SCANCODE_REGEN_TEST_FIXTURES", False)