feat: guess code language

Signed-off-by: Innei <[email protected]>

electron.vite.config.ts

@@ -52,8 +52,7 @@ export default defineConfig({
           excludeReplayWorker: true,
         },
         moduleMetadata: {
-          appVersion:
-            pkg.version,
+          appVersion: pkg.version,
         },
         sourcemaps: {
           filesToDeleteAfterUpload: ["dist/renderer/assets/*.js.map"],

eslint.config.mjs

@@ -10,7 +10,7 @@ export default defineConfig(
       lineBreak: "after",
     },
     lessOpinionated: true,
-    ignores: ["src/renderer/src/hono.ts", "src/hono.ts"],
+    ignores: ["src/renderer/src/hono.ts", "src/hono.ts", "resources/**"],
     preferESM: false,
   },
   {

forge.config.ts

@@ -41,7 +41,7 @@ async function cleanSources(
   ])
 
   // Keep only node_modules to be included in the app
-  const modules = new Set(["font-list"])
+  const modules = new Set(["font-list", "@vscode/vscode-languagedetection"])
   await Promise.all([
     ...(await readdir(buildPath).then((items) =>
       items
@@ -70,7 +70,7 @@ const config: ForgeConfig = {
     ],
     afterCopy: [cleanSources],
     asar: true,
-    ignore: [/^\/node_modules\/(?!font-list)/],
+    ignore: [/^\/node_modules\/(?!font-list|@vscode\/vscode-languagedetection)/],
     prune: true,
     ...(process.env.APPLE_ID &&
       process.env.APPLE_PASSWORD &&

package.json

@@ -69,6 +69,7 @@
     "@tanstack/react-query-devtools": "5.51.15",
     "@tanstack/react-query-persist-client": "5.51.15",
     "@use-gesture/react": "10.3.1",
+    "@vscode/vscode-languagedetection": "1.0.22",
     "@yornaath/batshit": "0.10.1",
     "builder-util-runtime": "9.2.5-alpha.3",
     "class-variance-authority": "0.7.0",

src/env.ts

@@ -16,11 +16,19 @@ export const env = createEnv({
   },
 
   emptyStringAsUndefined: true,
-  runtimeEnv:
-    "process" in globalThis ? process.env : injectExternalEnv(import.meta.env),
+  runtimeEnv: getRuntimeEnv(),
+
   skipValidation: !isDev,
 })
 
+function getRuntimeEnv() {
+  try {
+    return injectExternalEnv(import.meta.env)
+  } catch {
+    return process.env
+  }
+}
+
 function injectExternalEnv<T>(originEnv: T): T {
   if (!("document" in globalThis)) {
     return originEnv

src/main/index.ts

@@ -1,5 +1,3 @@
-import "dotenv/config"
-
 import { electronApp, optimizer } from "@electron-toolkit/utils"
 import { APP_PROTOCOL, DEEPLINK_SCHEME } from "@shared/constants"
 import { extractElectronWindowOptions } from "@shared/electron"

src/main/tipc/reader.ts

@@ -1,6 +1,9 @@
+import { createRequire } from "node:module"
+
 import { readability } from "../lib/readability"
 import { t } from "./_instance"
 
+const require = createRequire(import.meta.url)
 export const readerRoute = {
   readability: t.procedure
     .input<{ url: string }>()
@@ -12,6 +15,14 @@ export const readerRoute = {
       }
       const result = await readability(url)
 
+      return result
+    }),
+  detectCodeStringLanguage: t.procedure
+    .input<{ codeString: string }>()
+    .action(async ({ input }) => {
+      const { ModelOperations } = require("@vscode/vscode-languagedetection")
+      const modulOperations = new ModelOperations()
+      const result = await modulOperations.runModel(input.codeString)
       return result
     }),
 }

src/renderer/src/components/ui/code-highlighter/shiki/Shiki.tsx

@@ -1,7 +1,14 @@
 import { useUISettingSelector } from "@renderer/atoms/settings/ui"
+import { tipcClient } from "@renderer/lib/client"
 import { cn } from "@renderer/lib/utils"
 import type { FC } from "react"
-import { useLayoutEffect, useMemo, useRef, useState } from "react"
+import {
+  useInsertionEffect,
+  useLayoutEffect,
+  useMemo,
+  useRef,
+  useState,
+} from "react"
 import type {
   BundledLanguage,
   BundledTheme,
@@ -16,10 +23,7 @@ import { shikiTransformers } from "./shared"
 import styles from "./shiki.module.css"
 
 const shiki = await createHighlighterCore({
-  themes: [
-
-    import("shiki/themes/github-dark.mjs"),
-  ],
+  themes: [import("shiki/themes/github-dark.mjs")],
   langs: [],
   loadWasm: getWasm,
 })
@@ -42,15 +46,49 @@ let langModule: Record<
 > | null = null
 let themeModule: Record<BundledTheme, DynamicImportThemeRegistration> | null =
   null
-
+let bundledLanguagesKeysSet: Set<string> | null = null
 export const ShikiHighLighter: FC<ShikiProps> = (props) => {
   const { code, language, className, theme: overrideTheme } = props
+  const [currentLanguage, setCurrentLanguage] = useState(
+    language || "plaintext",
+  )
+
+  useInsertionEffect(() => {
+    if (language || !ELECTRON) return
+
+    if (!bundledLanguagesKeysSet) {
+      import("shiki/langs")
+        .then(({ bundledLanguages }) => {
+          langModule = bundledLanguages
+          bundledLanguagesKeysSet = new Set(Object.keys(bundledLanguages))
+        })
+        .then(guessLanguage)
+    } else {
+      guessLanguage()
+    }
+
+    function guessLanguage() {
+      tipcClient
+        ?.detectCodeStringLanguage({ codeString: code })
+        .then((result) => {
+          for (const item of result) {
+            if (bundledLanguagesKeysSet?.has(item.languageId)) {
+              setCurrentLanguage(item.languageId)
+              break
+            }
+          }
+        })
+    }
+  }, [])
+
   const loadThemesRef = useRef([] as string[])
   const loadLanguagesRef = useRef([] as string[])
 
   const [loaded, setLoaded] = useState(false)
 
-  const codeTheme = useUISettingSelector((s) => overrideTheme || s.codeHighlightTheme)
+  const codeTheme = useUISettingSelector(
+    (s) => overrideTheme || s.codeHighlightTheme,
+  )
   useLayoutEffect(() => {
     let isMounted = true
     setLoaded(false)
@@ -69,7 +107,7 @@ export const ShikiHighLighter: FC<ShikiProps> = (props) => {
     }
 
     async function register() {
-      if (!language || !codeTheme) return
+      if (!currentLanguage || !codeTheme) return
 
       const [{ bundledLanguages }, { bundledThemes }] =
         langModule && themeModule ?
@@ -85,20 +123,20 @@ export const ShikiHighLighter: FC<ShikiProps> = (props) => {
       themeModule = bundledThemes
 
       if (
-        language &&
-        loadLanguagesRef.current.includes(language) &&
+        currentLanguage &&
+        loadLanguagesRef.current.includes(currentLanguage) &&
         codeTheme &&
-        (loadThemesRef.current.includes(codeTheme))
+        loadThemesRef.current.includes(codeTheme)
       ) {
         return
       }
       return Promise.all([
         (async () => {
-          if (language) {
-            const importFn = (bundledLanguages as any)[language]
+          if (currentLanguage) {
+            const importFn = (bundledLanguages as any)[currentLanguage]
             if (!importFn) return
-            await loadShikiLanguage(language || "", importFn)
-            loadLanguagesRef.current.push(language)
+            await loadShikiLanguage(currentLanguage || "", importFn)
+            loadLanguagesRef.current.push(currentLanguage)
           }
         })(),
         (async () => {
@@ -119,7 +157,7 @@ export const ShikiHighLighter: FC<ShikiProps> = (props) => {
     return () => {
       isMounted = false
     }
-  }, [codeTheme, language])
+  }, [codeTheme, currentLanguage])
 
   if (!loaded) {
     return (
@@ -128,13 +166,16 @@ export const ShikiHighLighter: FC<ShikiProps> = (props) => {
       </pre>
     )
   }
-  return <ShikiCode {...props} codeTheme={codeTheme} />
+  return (
+    <ShikiCode {...props} language={currentLanguage} codeTheme={codeTheme} />
+  )
 }
 
-const ShikiCode: FC<ShikiProps & {
-  codeTheme: string
-
-}> = ({ code, language, codeTheme, className, transparent }) => {
+const ShikiCode: FC<
+  ShikiProps & {
+    codeTheme: string
+  }
+> = ({ code, language, codeTheme, className, transparent }) => {
   const rendered = useMemo(() => {
     try {
       return shiki.codeToHtml(code, {
@@ -159,9 +200,19 @@ const ShikiCode: FC<ShikiProps & {
     )
   }
   return (
-    <div className={cn("group relative my-4", styles["shiki-wrapper"], transparent ? styles["transparent"] : null, className)}>
+    <div
+      className={cn(
+        "group relative my-4",
+        styles["shiki-wrapper"],
+        transparent ? styles["transparent"] : null,
+        className,
+      )}
+    >
       <div dangerouslySetInnerHTML={{ __html: rendered }} />
-      <CopyButton value={code} className="absolute right-1 top-1 opacity-0 duration-200 group-hover:opacity-100" />
+      <CopyButton
+        value={code}
+        className="absolute right-1 top-1 opacity-0 duration-200 group-hover:opacity-100"
+      />
     </div>
   )
 }

src/renderer/src/lib/parse-html.ts

@@ -92,14 +92,13 @@ export const parseHtml = async (
         pre: ({ node, ...props }) => {
           if (!props.children) return null
 
-          let language = "plaintext"
+          let language = ""
           let codeString = null as string | null
           if (props.className?.includes("language-")) {
             language = props.className.replace("language-", "")
           }
 
           if (typeof props.children !== "object") {
-            language = "plaintext"
             codeString = props.children.toString()
           } else {
             if (
@@ -115,9 +114,8 @@ export const parseHtml = async (
             const code =
               "props" in props.children && props.children.props.children
             if (!code) return null
-            const $text = document.createElement("div")
-            $text.innerHTML = renderToString(code)
-            codeString = $text.textContent
+
+            codeString = extractCodeFromHtml(renderToString(code))
           }
 
           if (!codeString) return null
@@ -127,17 +125,18 @@ export const parseHtml = async (
             language: language.toLowerCase(),
           })
         },
-        table: ({ node, ...props }) => createElement(
-          "div",
-          {
-            className: "w-full overflow-x-auto",
-          },
+        table: ({ node, ...props }) =>
+          createElement(
+            "div",
+            {
+              className: "w-full overflow-x-auto",
+            },
 
-          createElement("table", {
-            ...props,
-            className: tw`w-full my-0`,
-          }),
-        ),
+            createElement("table", {
+              ...props,
+              className: tw`w-full my-0`,
+            }),
+          ),
       },
     }),
   }
@@ -161,3 +160,22 @@ const Img: Components["img"] = ({ node, ...props }) => {
 
   return createElement(MarkdownBlockImage, nextProps)
 }
+
+function extractCodeFromHtml(htmlString: string) {
+  const tempDiv = document.createElement("div")
+  tempDiv.innerHTML = htmlString
+
+  const divElements = tempDiv.querySelectorAll("div")
+
+  let code = ""
+
+  divElements.forEach((div) => {
+    code += `${div.textContent}\n`
+  })
+
+  if (divElements.length === 0) {
+    return tempDiv.textContent
+  }
+
+  return code
+}