fix: cast to varchar in non-validating dialect (#420)

Author: reata

sqllineage/core/parser/sqlparse/models.py

@@ -152,7 +152,7 @@ def _extract_source_columns(token: Token) -> List[ColumnQualifierTuple]:
         elif isinstance(token, Identifier):
             real_name = token.get_real_name()
             # ignore function dtypes that don't need to check for extract column
-            FUNC_DTYPE = ["decimal", "numeric"]
+            FUNC_DTYPE = ["decimal", "numeric", "varchar"]
             has_function = any(
                 isinstance(t, Function) and t.get_real_name() not in FUNC_DTYPE
                 for t in token.tokens

tests/sql/column/test_column_select_cast.py

@@ -49,7 +49,9 @@ def test_cast_with_comparison():
     )
 
 
-@pytest.mark.parametrize("dtype", ["string", "timestamp", "date", "decimal(18, 0)"])
+@pytest.mark.parametrize(
+    "dtype", ["string", "timestamp", "date", "decimal(18, 0)", "varchar(255)"]
+)
 def test_cast_to_data_type(dtype: str):
     sql = f"""INSERT INTO tab1
 SELECT cast(col1 as {dtype}) AS col1
@@ -60,7 +62,9 @@ def test_cast_to_data_type(dtype: str):
     )
 
 
-@pytest.mark.parametrize("dtype", ["string", "timestamp", "date", "decimal(18, 0)"])
+@pytest.mark.parametrize(
+    "dtype", ["string", "timestamp", "date", "decimal(18, 0)", "varchar(255)"]
+)
 def test_nested_cast_to_data_type(dtype: str):
     sql = f"""INSERT INTO tab1
 SELECT cast(cast(col1 AS {dtype}) AS {dtype}) AS col1
@@ -78,7 +82,9 @@ def test_nested_cast_to_data_type(dtype: str):
     )
 
 
-@pytest.mark.parametrize("dtype", ["string", "timestamp", "date", "decimal(18, 0)"])
+@pytest.mark.parametrize(
+    "dtype", ["string", "timestamp", "date", "decimal(18, 0)", "varchar(255)"]
+)
 def test_cast_to_data_type_with_case_when(dtype: str):
     sql = f"""INSERT INTO tab1
 SELECT cast(case when col1 > 0 then col2 else col3 end as {dtype}) AS col1

tests/sql/table/sqlfluff_only/__init__.py renamed to tests/sql/sqlfluff_only/__init__.py

@@ -1,7 +1,7 @@
 import pytest
 
 from sqllineage.utils.entities import ColumnQualifierTuple
-from ....helpers import assert_column_lineage_equal, assert_table_lineage_equal
+from ...helpers import assert_column_lineage_equal, assert_table_lineage_equal
 
 
 @pytest.mark.parametrize("dialect", ["tsql"])

tests/sql/table/sqlfluff_only/test_dialect_specific.py renamed to tests/sql/sqlfluff_only/test_dialect_specific.py

@@ -3,7 +3,7 @@
 """
 
 from sqllineage.utils.entities import ColumnQualifierTuple
-from ....helpers import assert_column_lineage_equal, assert_table_lineage_equal
+from ...helpers import assert_column_lineage_equal, assert_table_lineage_equal
 
 
 def test_view_with_subquery_custom_columns():

tests/sql/table/sqlfluff_only/test_dml_specify_column.py renamed to tests/sql/sqlfluff_only/test_dml_specify_column.py

@@ -4,7 +4,7 @@
 
 
 from sqllineage.utils.entities import ColumnQualifierTuple
-from ....helpers import assert_column_lineage_equal, assert_table_lineage_equal
+from ...helpers import assert_column_lineage_equal, assert_table_lineage_equal
 
 
 def test_non_reserved_keyword_as_source():
@@ -71,3 +71,28 @@ def test_current_timestamp():
         ],
         test_sqlparse=False,
     )
+
+
+def test_coalesce_with_whitespace():
+    """
+    coalesce is a keyword since ANSI SQL-2023
+    usually it's parsed as a function. however, when whitespace followed which is valid syntax,
+    sqlparse cannot produce the correct AST
+    """
+    sql = """INSERT INTO tab1
+SELECT coalesce (col1, col2) as col3
+FROM tab2"""
+    assert_column_lineage_equal(
+        sql,
+        [
+            (
+                ColumnQualifierTuple("col1", "tab2"),
+                ColumnQualifierTuple("col3", "tab1"),
+            ),
+            (
+                ColumnQualifierTuple("col2", "tab2"),
+                ColumnQualifierTuple("col3", "tab1"),
+            ),
+        ],
+        test_sqlparse=False,
+    )

tests/sql/table/sqlfluff_only/test_keyword_as_identifier.py renamed to tests/sql/sqlfluff_only/test_keyword_as_identifier.py

@@ -1,5 +1,5 @@
 from sqllineage.utils.entities import ColumnQualifierTuple
-from ....helpers import assert_column_lineage_equal
+from ...helpers import assert_column_lineage_equal
 
 
 def test_subquery_expression_without_source_table():

tests/sql/table/sqlfluff_only/test_parenthesized_expression.py renamed to tests/sql/sqlfluff_only/test_parenthesized_expression.py

@@ -5,7 +5,7 @@
 """
 
 from sqllineage.utils.entities import ColumnQualifierTuple
-from ....helpers import assert_column_lineage_equal, assert_table_lineage_equal
+from ...helpers import assert_column_lineage_equal, assert_table_lineage_equal
 
 
 def test_insert_into_qualified_table_with_parenthesized_query():