[ENG-3848][ENG-3861]Shiki Code block Experimental (#4030)

* Shiki Code block Experimental

* refactor

* update code

* remove console.log

* add transformers to namespace

* some validations

* fix components paths

* fix ruff

* add a high-level component

* fix mapping

* fix mapping

* python 3.9+

* see if this fixes the tests

* fix pyi and annotations

* minimal update of theme and language map

* add hack for reflex-web/flexdown

* unit test file commit

* [ENG-3895] [ENG-3896] Update styling for shiki code block

* strip transformer triggers

* minor refactor

* linter

* fix pyright

* pyi fix

* add unit tests

* sneaky pyright ignore

* the transformer trigger regex should remove the language comment character

* minor refactor

* fix silly mistake

* component mapping in markdown should use the first child for codeblock

* use ternary operator in numbers.py, move code block args to class for docs discoverability

* precommit

* pyright fix

* remove id on copy button animation

* pyright fix for real

* pyi fix

* pyi fix fr

* check if svg exists

* copy event chain

* do a concatenation instead of first child

* added comment

---------

Co-authored-by: Carlos <cutillascarlos@gmail.com>

reflex/.templates/web/components/shiki/code.js

@@ -0,0 +1,29 @@
+import { useEffect, useState } from "react"
+import { codeToHtml} from "shiki"
+
+export function Code ({code, theme, language, transformers, ...divProps}) {
+    const [codeResult, setCodeResult] = useState("")
+    useEffect(() => {
+        async function fetchCode() {
+          let final_code;
+
+          if (Array.isArray(code)) {
+            final_code = code[0];
+          } else {
+            final_code = code;
+          }
+          const result = await codeToHtml(final_code, {
+            lang: language,
+            theme,
+            transformers
+          });
+          setCodeResult(result);
+        }
+        fetchCode();
+      }, [code, language, theme, transformers]
+
+    )
+    return (
+        <div dangerouslySetInnerHTML={{__html: codeResult}} {...divProps}  ></div>
+    )
+}

reflex/components/markdown/markdown.py

@@ -20,6 +20,8 @@
 from reflex.utils import types
 from reflex.utils.imports import ImportDict, ImportVar
 from reflex.vars.base import LiteralVar, Var
+from reflex.vars.function import ARRAY_ISARRAY
+from reflex.vars.number import ternary_operation
 
 # Special vars used in the component map.
 _CHILDREN = Var(_js_expr="children", _var_type=str)
@@ -199,7 +201,16 @@ def get_component(self, tag: str, **props) -> Component:
             raise ValueError(f"No markdown component found for tag: {tag}.")
 
         special_props = [_PROPS_IN_TAG]
-        children = [_CHILDREN]
+        children = [
+            _CHILDREN
+            if tag != "codeblock"
+            # For codeblock, the mapping for some cases returns an array of elements. Let's join them into a string.
+            else ternary_operation(
+                ARRAY_ISARRAY.call(_CHILDREN),  # type: ignore
+                _CHILDREN.to(list).join("\n"),
+                _CHILDREN,
+            ).to(str)
+        ]
 
         # For certain tags, the props from the markdown renderer are not actually valid for the component.
         if tag in NO_PROPS_TAGS:

reflex/experimental/__init__.py

@@ -2,6 +2,7 @@
 
 from types import SimpleNamespace
 
+from reflex.components.datadisplay.shiki_code_block import code_block as code_block
 from reflex.components.props import PropsBase
 from reflex.components.radix.themes.components.progress import progress as progress
 from reflex.components.sonner.toast import toast as toast
@@ -67,4 +68,5 @@ def register_component_warning(component_name: str):
     layout=layout,
     PropsBase=PropsBase,
     run_in_thread=run_in_thread,
+    code_block=code_block,
 )

reflex/vars/function.py

@@ -180,6 +180,7 @@ def create(
 
 
 JSON_STRINGIFY = FunctionStringVar.create("JSON.stringify")
+ARRAY_ISARRAY = FunctionStringVar.create("Array.isArray")
 PROTOTYPE_TO_STRING = FunctionStringVar.create(
     "((__to_string) => __to_string.toString())"
 )

reflex/vars/sequence.py

@@ -529,6 +529,26 @@ def array_join_operation(array: ArrayVar, sep: StringVar[Any] | str = ""):
     return var_operation_return(js_expression=f"{array}.join({sep})", var_type=str)
 
 
+@var_operation
+def string_replace_operation(
+    string: StringVar, search_value: StringVar | str, new_value: StringVar | str
+):
+    """Replace a string with a value.
+
+    Args:
+        string: The string.
+        search_value: The string to search.
+        new_value: The value to be replaced with.
+
+    Returns:
+        The string replace operation.
+    """
+    return var_operation_return(
+        js_expression=f"{string}.replace({search_value}, {new_value})",
+        var_type=str,
+    )
+
+
 # Compile regex for finding reflex var tags.
 _decode_var_pattern_re = (
     rf"{constants.REFLEX_VAR_OPENING_TAG}(.*?){constants.REFLEX_VAR_CLOSING_TAG}"

tests/units/components/datadisplay/test_shiki_code.py

@@ -0,0 +1,172 @@
+import pytest
+
+from reflex.components.datadisplay.shiki_code_block import (
+    ShikiBaseTransformers,
+    ShikiCodeBlock,
+    ShikiHighLevelCodeBlock,
+    ShikiJsTransformer,
+)
+from reflex.components.el.elements.forms import Button
+from reflex.components.lucide.icon import Icon
+from reflex.components.radix.themes.layout.box import Box
+from reflex.style import Style
+from reflex.vars import Var
+
+
+@pytest.mark.parametrize(
+    "library, fns, expected_output, raises_exception",
+    [
+        ("some_library", ["function_one"], ["function_one"], False),
+        ("some_library", [123], None, True),
+        ("some_library", [], [], False),
+        (
+            "some_library",
+            ["function_one", "function_two"],
+            ["function_one", "function_two"],
+            False,
+        ),
+        ("", ["function_one"], ["function_one"], False),
+        ("some_library", ["function_one", 789], None, True),
+        ("", [], [], False),
+    ],
+)
+def test_create_transformer(library, fns, expected_output, raises_exception):
+    if raises_exception:
+        # Ensure ValueError is raised for invalid cases
+        with pytest.raises(ValueError):
+            ShikiCodeBlock.create_transformer(library, fns)
+    else:
+        transformer = ShikiCodeBlock.create_transformer(library, fns)
+        assert isinstance(transformer, ShikiBaseTransformers)
+        assert transformer.library == library
+
+        # Verify that the functions are correctly wrapped in FunctionStringVar
+        function_names = [str(fn) for fn in transformer.fns]
+        assert function_names == expected_output
+
+
+@pytest.mark.parametrize(
+    "code_block, children, props, expected_first_child, expected_styles",
+    [
+        ("print('Hello')", ["print('Hello')"], {}, "print('Hello')", {}),
+        (
+            "print('Hello')",
+            ["print('Hello')", "More content"],
+            {},
+            "print('Hello')",
+            {},
+        ),
+        (
+            "print('Hello')",
+            ["print('Hello')"],
+            {
+                "transformers": [
+                    ShikiBaseTransformers(
+                        library="lib", fns=[], style=Style({"color": "red"})
+                    )
+                ]
+            },
+            "print('Hello')",
+            {"color": "red"},
+        ),
+        (
+            "print('Hello')",
+            ["print('Hello')"],
+            {
+                "transformers": [
+                    ShikiBaseTransformers(
+                        library="lib", fns=[], style=Style({"color": "red"})
+                    )
+                ],
+                "style": {"background": "blue"},
+            },
+            "print('Hello')",
+            {"color": "red", "background": "blue"},
+        ),
+    ],
+)
+def test_create_shiki_code_block(
+    code_block, children, props, expected_first_child, expected_styles
+):
+    component = ShikiCodeBlock.create(code_block, *children, **props)
+
+    # Test that the created component is a Box
+    assert isinstance(component, Box)
+
+    # Test that the first child is the code
+    code_block_component = component.children[0]
+    assert code_block_component.code._var_value == expected_first_child  # type: ignore
+
+    applied_styles = component.style
+    for key, value in expected_styles.items():
+        assert Var.create(applied_styles[key])._var_value == value
+
+
+@pytest.mark.parametrize(
+    "children, props, expected_transformers, expected_button_type",
+    [
+        (["print('Hello')"], {"use_transformers": True}, [ShikiJsTransformer], None),
+        (["print('Hello')"], {"can_copy": True}, None, Button),
+        (
+            ["print('Hello')"],
+            {
+                "can_copy": True,
+                "copy_button": Button.create(Icon.create(tag="a_arrow_down")),
+            },
+            None,
+            Button,
+        ),
+    ],
+)
+def test_create_shiki_high_level_code_block(
+    children, props, expected_transformers, expected_button_type
+):
+    component = ShikiHighLevelCodeBlock.create(*children, **props)
+
+    # Test that the created component is a Box
+    assert isinstance(component, Box)
+
+    # Test that the first child is the code block component
+    code_block_component = component.children[0]
+    assert code_block_component.code._var_value == children[0]  # type: ignore
+
+    # Check if the transformer is set correctly if expected
+    if expected_transformers:
+        exp_trans_names = [t.__name__ for t in expected_transformers]
+        for transformer in code_block_component.transformers._var_value:  # type: ignore
+            assert type(transformer).__name__ in exp_trans_names
+
+    # Check if the second child is the copy button if can_copy is True
+    if props.get("can_copy", False):
+        if props.get("copy_button"):
+            assert isinstance(component.children[1], expected_button_type)
+            assert component.children[1] == props["copy_button"]
+        else:
+            assert isinstance(component.children[1], expected_button_type)
+    else:
+        assert len(component.children) == 1
+
+
+@pytest.mark.parametrize(
+    "children, props",
+    [
+        (["print('Hello')"], {"theme": "dark"}),
+        (["print('Hello')"], {"language": "javascript"}),
+    ],
+)
+def test_shiki_high_level_code_block_theme_language_mapping(children, props):
+    component = ShikiHighLevelCodeBlock.create(*children, **props)
+
+    # Test that the theme is mapped correctly
+    if "theme" in props:
+        assert component.children[
+            0
+        ].theme._var_value == ShikiHighLevelCodeBlock._map_themes(props["theme"])  # type: ignore
+
+    # Test that the language is mapped correctly
+    if "language" in props:
+        assert component.children[
+            0
+        ].language._var_value == ShikiHighLevelCodeBlock._map_languages(  # type: ignore
+            props["language"]
+        )