Cython seq (#4500)

* Modify CyDifflib the same way we modified difflib

Signed-off-by: Jono Yang <[email protected]>

* Update assignment code

Signed-off-by: Jono Yang <[email protected]>

* Use unordered_map instead of vector

Signed-off-by: Jono Yang <[email protected]>

* Build seq.pyx on install and use as default

    * Ignore .so files in src/

Signed-off-by: Jono Yang <[email protected]>

* Install cython in venv

Signed-off-by: Jono Yang <[email protected]>

* State argument types in function declarations

Signed-off-by: Jono Yang <[email protected]>

* Do not perform list comprehension  when returning matching blocks

Signed-off-by: Jono Yang <[email protected]>

* Install cython in create_virtualenv function

Signed-off-by: Jono Yang <[email protected]>

* Update import in test_seq.py

Signed-off-by: Jono Yang <[email protected]>

* Add ABOUT and LICENSE file for seq.pyx

Signed-off-by: Jono Yang <[email protected]>

* Modify extend_match for use in Python

Signed-off-by: Jono Yang <[email protected]>

* Remove seq.pyx and import it as cyseq

Signed-off-by: Jono Yang <[email protected]>

* Do not install cython into the venv

Signed-off-by: Jono Yang <[email protected]>

* Remove unneeded gitignore rule

Signed-off-by: Jono Yang <[email protected]>

---------

Signed-off-by: Jono Yang <[email protected]>

Author: JonoYang

configure

@@ -118,7 +118,7 @@ CFG_ROOT_DIR="$( cd "$( dirname "${CFG_BIN}" )" && pwd )"
 
 CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
 
-# force relaunching under X86-64 architecture on macOS M1/ARM 
+# force relaunching under X86-64 architecture on macOS M1/ARM
 if [[ $OSTYPE == 'darwin'* && $(uname -m) == 'arm64' && $(sysctl -in sysctl.proc_translated) == 0 ]]; then
     arch -x86_64 /bin/bash -c "$CFG_ROOT_DIR/configure $*"
     exit $?
@@ -253,7 +253,6 @@ install_packages() {
     # be reinstalled a second time and reused from the virtualenv and this
     # speeds up the installation.
     # We always have the PEP517 build dependencies installed already.
-
     "$CFG_BIN_DIR/pip" install \
         --upgrade \
         --no-build-isolation \

pyproject.toml

@@ -69,7 +69,7 @@ include = [
   "."
 
 ]
-# ignore test data and testfiles: they should never be linted nor formatted 
+# ignore test data and testfiles: they should never be linted nor formatted
 exclude = [
 # main style
   "**/tests/data/**/*",

setup.cfg

@@ -117,6 +117,7 @@ install_requires =
     typecode >= 30.0.1
     typecode[full] >= 30.0.1
     extractcode[full] >= 31.0.0
+    cyseq; platform_system == 'Linux'
 
 
 [options.packages.find]

src/licensedcode/index.py

@@ -28,13 +28,19 @@
 from licensedcode import match_spdx_lid
 from licensedcode import match_unknown
 from licensedcode.dmp import match_blocks as match_blocks_dmp
-from licensedcode.seq import match_blocks as match_blocks_seq
 from licensedcode import query
 from licensedcode import tokenize
 from licensedcode.spans import Span
 from typing import NamedTuple
 from typing import Callable
 
+try:
+    # Use Cython seq.py implementation
+    from cyseq import match_blocks as match_blocks_seq
+except ImportError:
+    # Use Python seq.py if it is not available
+    from licensedcode.seq import match_blocks as match_blocks_seq
+
 """
 Main license index construction, query processing and matching entry points for
 license detection.

src/licensedcode/match_seq.py

@@ -63,7 +63,12 @@ def match_sequence(
         return []
 
     if not match_blocks:
-        from licensedcode.seq import match_blocks
+        try:
+            # Use Cython seq.py implementation
+            from cyseq import match_blocks
+        except ImportError:
+            # Use Python seq.py if it is not available
+            from licensedcode.seq import match_blocks
 
     rid = rule.rid
     itokens = idx.tids_by_rid[rid]