Merge pull request #35 from hockyy/feature/32

[#32] Add support for kuromoji and set up without external data

.gitattributes

@@ -0,0 +1 @@
+renderer/public/** filter=lfs diff=lfs merge=lfs -text

main/background.ts

@@ -12,6 +12,7 @@ import {
 } from 'jmdict-simplified-node';
 import fs from "fs";
 import path from "path";
+import {getTokenizer} from "kuromojin";
 
 
 const isProd: boolean = process.env.NODE_ENV === 'production';
@@ -165,6 +166,13 @@ if (isProd) {
 
 
   })
+
+  ipcMain.handle('loadDefaultMode', async (event) => {
+    mecabCommand = 'kuromoji';
+    return await checkJMDict({
+      jmdict: path.join(__dirname, 'dict/jmdict.json')
+    });
+  })
   ipcMain.handle('removeDictCache', (event) => {
     removeJMDictCache()
     return true;
@@ -235,13 +243,10 @@ if (isProd) {
     return okSetup;
   }
 
-  ipcMain.handle('getMecabCommand', async (event, mecab, text) => {
-    return mecabCommand
+  ipcMain.handle('getTokenizerMode', async () => {
+    return mecabCommand;
   })
   ipcMain.handle('validateConfig', async (event, config) => {
-
-    // const dicdirRes = checkDicdir(config);
-    // if (dicdirRes.ok !== 1) return dicdirRes
     let jmdictRes;
     if (!config.cached) {
       jmdictRes = await checkJMDict(config);
@@ -259,9 +264,18 @@ if (isProd) {
   })
   let packageJsonPath = path.join(app.getAppPath(), 'package.json');
   let packageJson = JSON.parse(fs.readFileSync(packageJsonPath).toString());
-  ipcMain.handle('getAppVersion', async (event, ...args) => {
+  ipcMain.handle('getAppVersion', async () => {
     return packageJson.version;
   });
+  let tokenizer = null;
+  getTokenizer({dicPath: path.join(__dirname, 'dict/')}).then(loadedTokenizer => {
+    tokenizer = loadedTokenizer;
+  }).catch(e => {
+    console.log(e)
+  })
+  ipcMain.handle('tokenizeUsingKuromoji', async (event, sentence) => {
+    return tokenizer.tokenizeForSentence(sentence);
+  });
   protocol.registerFileProtocol(scheme, requestHandler); /* eng-disable PROTOCOL_HANDLER_JS_CHECK */
   if (isProd) {
     await mainWindow.loadURL('app://./home.html');

package.json

@@ -2,7 +2,7 @@
   "private": true,
   "name": "miteiru",
   "description": "Miteiru",
-  "version": "1.2.1",
+  "version": "1.3.0",
   "author": "Hocky Yudhiono <[email protected]>",
   "main": "app/background.js",
   "build": {
@@ -22,6 +22,8 @@
     }
   },
   "scripts": {
+    "copy": "rsync -r renderer/public/ app/",
+    "predev": "npm run copy",
     "dev": "nextron",
     "build": "nextron build",
     "build:linux": "nextron build --linux",
@@ -40,12 +42,14 @@
     "html-react-parser": "^3.0.16",
     "iconv-lite": "^0.6.3",
     "jmdict-simplified-node": "^1.1.2",
+    "kuromojin": "^3.0.0",
     "patch-package": "^7.0.0",
+    "react-awesome-button": "^7.0.5",
     "react-colorful": "^5.6.1",
     "react-dropzone": "^14.2.3",
     "react-smooth-collapse": "^2.1.2",
     "react-video-seek-slider": "^6.0.4",
-    "shunou": "^0.0.36",
+    "shunou": "^1.0.2",
     "styled-components": "^6.0.3",
     "video.js": "^8.3.0"
   },

patches/jmdict-simplified-node+1.1.2.patch

@@ -1,8 +1,12 @@
 diff --git a/node_modules/jmdict-simplified-node/index.js b/node_modules/jmdict-simplified-node/index.js
-index b50ce2d..233738c 100644
+index b50ce2d..d81da04 100644
 --- a/node_modules/jmdict-simplified-node/index.js
 +++ b/node_modules/jmdict-simplified-node/index.js
-@@ -34,43 +34,48 @@ function setup(DBNAME, filename = '', verbose = false) {
+@@ -31,46 +31,52 @@ function setup(DBNAME, filename = '', verbose = false) {
+             // pass
+         }
+         if (!filename) {
++            db.close();
              throw new Error('database not found but cannot create it if no `filename` given');
          }
          const raw = JSON.parse(yield fs_1.promises.readFile(filename, 'utf8'));

renderer/components/ContainerHome.tsx

@@ -1,6 +1,6 @@
 export const ContainerHome = (props) => {
   return (
-      <div className={' border border-black w-fit rounded-lg bg-blue-100 p-3'}>
+      <div className={props.className + ' border border-black w-fit rounded-lg bg-blue-100 p-3'}>
         {props.children}
       </div>
   )

renderer/components/DataStructures.ts

@@ -33,8 +33,8 @@ export class Line {
     this.content = strContent;
   }
 
-  fillContentFurigana(mecab) {
-    this.content = getFurigana(this.content as string, mecab);
+  async fillContentFurigana(tokenizeMiteiru: (string) => Promise<any[]>) {
+    this.content = await tokenizeMiteiru(this.content as string);
   }
 
   async fillContentWithLearningKotoba() {
@@ -133,11 +133,11 @@ export class SubtitleContainer {
     return subtitleContainer
   }
 
-  async adjustJapanese(mecab) {
+  async adjustJapanese(tokenizeMiteiru: (string) => Promise<any[]>) {
     for (let i = 0; i < this.lines.length; i++) {
       if (globalSubtitleId !== this.id) return;
       const line = this.lines[i];
-      await line.fillContentFurigana(mecab)
+      await line.fillContentFurigana(tokenizeMiteiru)
       await line.fillContentWithLearningKotoba();
       this.progress = `${((i + 1) * 100 / this.lines.length).toFixed(2)}%`;
     }

renderer/components/KeyboardHelp.tsx

@@ -46,7 +46,7 @@ export const Key = ({value, extraClass = ""}) => {
 
 export const KeyboardHelp = () => {
   return <div
-      className={"grid grid-cols-2 gap-5 content-start w-auto px-5 py-3 m-2 bg-blue-50 rounded-lg border-2 border-black"}>
+      className={"grid grid-cols-2 gap-5 content-start w-full md:w-4/5 px-5 py-3 m-2 bg-blue-50 rounded-lg border-2 border-black"}>
     {shortcutInformation.map((val, idx) => {
       return <div
           className={"font-mono font-bold text-black flex flex-row items-center gap-3 font-bold"}

renderer/components/MeaningBox.tsx

@@ -1,17 +1,17 @@
 import React, {useEffect, useState} from "react";
 import {ipcRenderer} from "electron";
-import {getFurigana} from "shunou";
 import {Sentence} from "./Sentence";
 import {CJKStyling, defaultMeaningBoxStyling} from "../utils/CJKStyling";
+import {joinString} from "../utils/utils";
 
 const initialContentState = {sense: [], kanji: []};
 
 const MeaningBox = ({
                       meaning,
                       setMeaning,
-                      mecab,
+                      tokenizeMiteiru,
                       subtitleStyling = defaultMeaningBoxStyling
-                    }: { meaning: string, setMeaning: any, mecab: string, subtitleStyling?: CJKStyling }) => {
+                    }: { meaning: string, setMeaning: any, tokenizeMiteiru: (value: string) => Promise<any[]>, subtitleStyling?: CJKStyling }) => {
   const [meaningContent, setMeaningContent] = useState(initialContentState)
   const [otherMeanings, setOtherMeanings] = useState([]);
   const [meaningIndex, setMeaningIndex] = useState(0);
@@ -34,14 +34,25 @@ const MeaningBox = ({
       setTags(val)
     })
   }, [meaning]);
-  const joinString = (arr, separator = '; ') => {
-    let total = "";
-    arr.forEach(val => {
-      if (total !== '') total += separator
-      total += val.toString();
-    })
-    return total;
-  }
+
+
+  const [furiganizedData, setFuriganizedData] = useState([]);
+
+  useEffect(() => {
+    const fetchData = async () => {
+      const data = await Promise.all(meaningContent.kanji.map(async (val) => {
+        const furiganized = await tokenizeMiteiru(val.text);
+        return {
+          key: val.key,
+          furiganized
+        };
+      }));
+
+      setFuriganizedData(data);
+    };
+    if (meaningContent.kanji.length) fetchData();
+  }, [meaningContent.kanji]); // Add your dependencies here
+
 
   if (meaningContent.kanji.length > 0) {
     return (<div onClick={() => {
@@ -69,23 +80,23 @@ const MeaningBox = ({
           <div className={"flex flex-wrap gap-2"} style={{
             fontFamily: "Arial",
             fontSize: "40px",
-          }}>{meaningContent.kanji.map((val, meanKey) => {
-            const furiganized = getFurigana(val.text, mecab);
-            return (
-                <div key={meanKey}
+          }}>
+            {furiganizedData.map(({key, furiganized}) => (
+                <div key={key}
                      className={"bg-white rounded-xl p-2 border-2 border-blue-700 w-fit unselectable"}>
-                  {[...furiganized.map((val, idx) => {
-                    return (<Sentence key={idx}
-                                      origin={val.origin}
-                                      setMeaning={setMeaning}
-                                      separation={val.separation}
-                                      extraClass={"meaning-kanji text-md"}
-                                      subtitleStyling={subtitleStyling}/>)
-                  })]}
-                </div>);
-          })}</div>
+                  {[...furiganized.map((val, idx) => (
+                      <Sentence key={idx}
+                                origin={val.origin}
+                                setMeaning={setMeaning}
+                                separation={val.separation}
+                                extraClass={"unselectable meaning-kanji text-md"}
+                                subtitleStyling={subtitleStyling}/>
+                  ))]}
+                </div>
+            ))}
+          </div>
           {meaningIndex + 1 < otherMeanings.length &&
-              < button className={"bg-blue-800 p-3 rounded-md m-4"} onClick={(e) => {
+              <button className={"bg-blue-800 p-3 rounded-md m-4"} onClick={(e) => {
                 e.stopPropagation()
                 setMeaningIndex((old) => {
                   setMeaningContent(otherMeanings[old + 1])