From 26ebcb2fbd0d74ce560b97d8f026104a26d048e0 Mon Sep 17 00:00:00 2001 From: H <43509927+guoyuhao2330@users.noreply.github.com> Date: Thu, 8 Aug 2024 12:39:19 +0800 Subject: [PATCH] Add GitHub, deepl, baidu-fanyi (#1857) ### What problem does this PR solve? #1739 ### Type of change - [x] New Feature (non-breaking change which adds functionality) --------- Co-authored-by: Kevin Hu --- agent/component/__init__.py | 4 +- agent/component/baidufanyi.py | 114 ++++++++++++++++++++++++++++++++++ agent/component/deepl.py | 77 +++++++++++++++++++++++ agent/component/github.py | 61 ++++++++++++++++++ requirements.txt | 1 + requirements_arm.txt | 1 + requirements_dev.txt | 1 + 7 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 agent/component/baidufanyi.py create mode 100644 agent/component/deepl.py create mode 100644 agent/component/github.py diff --git a/agent/component/__init__.py b/agent/component/__init__.py index 199015e47f..43fa3eeb80 100644 --- a/agent/component/__init__.py +++ b/agent/component/__init__.py @@ -17,7 +17,9 @@ from .google import Google, GoogleParam from .bing import Bing, BingParam from .googlescholar import GoogleScholar, GoogleScholarParam - +from .deepl import DeepL, DeepLParam +from .github import GitHub, GitHubParam +from .baidufanyi import BaiduFanyi, BaiduFanyiParam def component_class(class_name): m = importlib.import_module("agent.component") diff --git a/agent/component/baidufanyi.py b/agent/component/baidufanyi.py new file mode 100644 index 0000000000..f77fa80d6e --- /dev/null +++ b/agent/component/baidufanyi.py @@ -0,0 +1,114 @@ +# +# Copyright 2024 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import random +from abc import ABC +import requests +import re +from agent.component.base import ComponentBase, ComponentParamBase +from hashlib import md5 + + +class BaiduFanyiParam(ComponentParamBase): + """ + Define the BaiduFanyi component parameters. + """ + + def __init__(self): + super().__init__() + self.prompt = "" + self.appid = "xxx" + self.secret_key = "xxx" + self.trans_type = 'translate' + self.parameters = [] + self.source_lang = 'auto' + self.target_lang = 'auto' + self.domain = 'finance' + + def check(self): + self.check_positive_integer(self.top_n, "Top N") + self.check_empty(self.appid, "BaiduFanyi APPID") + self.check_empty(self.secret_key, "BaiduFanyi Secret Key") + self.check_valid_value(self.trans_type, "Translate type", ['translate', 'fieldtranslate']) + self.check_valid_value(self.trans_type, "Translate domain", + ['it', 'finance', 'machinery', 'senimed', 'novel', 'academic', 'aerospace', 'wiki', + 'news', 'law', 'contract']) + self.check_valid_value(self.source_lang, "Source language", + ['auto', 'zh', 'en', 'yue', 'wyw', 'jp', 'kor', 'fra', 'spa', 'th', 'ara', 'ru', 'pt', + 'de', 'it', 'el', 'nl', 'pl', 'bul', 'est', 'dan', 'fin', 'cs', 'rom', 'slo', 'swe', + 'hu', 'cht', 'vie']) + self.check_valid_value(self.target_lang, "Target language", + ['auto', 'zh', 'en', 'yue', 'wyw', 'jp', 'kor', 'fra', 'spa', 'th', 'ara', 'ru', 'pt', + 'de', 'it', 'el', 'nl', 'pl', 'bul', 'est', 'dan', 'fin', 'cs', 'rom', 'slo', 'swe', + 'hu', 'cht', 'vie']) + self.check_valid_value(self.domain, "Translate field", + ['it', 'finance', 'machinery', 'senimed', 'novel', 'academic', 'aerospace', 'wiki', + 'news', 'law', 'contract']) + + +class BaiduFanyi(ComponentBase, ABC): + component_name = "BaiduFanyi" + + def _run(self, history, **kwargs): + prompt = self._param.prompt + + ans = self.get_input() + ans = " - ".join(ans["content"]) if "content" in ans else "" + if not ans: + return BaiduFanyi.be_output("") + + for para in self._param.parameters: + cpn = self._canvas.get_component(para["component_id"])["obj"] + _, out = cpn.output(allow_partial=False) + if "content" not in out.columns: + kwargs[para["key"]] = "Nothing" + else: + kwargs[para["key"]] = " - " + "\n - ".join(out["content"]) + + kwargs["input"] = ans + for n, v in kwargs.items(): + prompt = re.sub(r"\{%s\}" % n, str(v), prompt) + + try: + source_lang = self._param.source_lang + target_lang = self._param.target_lang + appid = self._param.appid + salt = random.randint(32768, 65536) + secret_key = self._param.secret_key + + if self._param.trans_type == 'translate': + sign = md5((appid + prompt + salt + secret_key).encode('utf-8')).hexdigest() + url = 'http://api.fanyi.baidu.com/api/trans/vip/translate?' + 'q=' + prompt + '&from=' + source_lang + '&to=' + target_lang + '&appid=' + appid + '&salt=' + salt + '&sign=' + sign + headers = {"Content-Type": "application/x-www-form-urlencoded"} + response = requests.post(url=url, headers=headers).json() + + if response.get('error_code'): + BaiduFanyi.be_output("**Error**:" + response['error_msg']) + + return BaiduFanyi.be_output(response['trans_result'][0]['dst']) + elif self._param.trans_type == 'fieldtranslate': + domain = self._param.domain + sign = md5((appid + prompt + salt + domain + secret_key).encode('utf-8')).hexdigest() + url = 'http://api.fanyi.baidu.com/api/trans/vip/fieldtranslate?' + 'q=' + prompt + '&from=' + source_lang + '&to=' + target_lang + '&appid=' + appid + '&salt=' + salt + '&domain=' + domain + '&sign=' + sign + headers = {"Content-Type": "application/x-www-form-urlencoded"} + response = requests.post(url=url, headers=headers).json() + + if response.get('error_code'): + BaiduFanyi.be_output("**Error**:" + response['error_msg']) + + return BaiduFanyi.be_output(response['trans_result'][0]['dst']) + + except Exception as e: + BaiduFanyi.be_output("**Error**:" + str(e)) diff --git a/agent/component/deepl.py b/agent/component/deepl.py new file mode 100644 index 0000000000..be518db3ef --- /dev/null +++ b/agent/component/deepl.py @@ -0,0 +1,77 @@ +# +# Copyright 2024 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from abc import ABC +import re +from agent.component.base import ComponentBase, ComponentParamBase +import deepl + + +class DeepLParam(ComponentParamBase): + """ + Define the DeepL component parameters. + """ + + def __init__(self): + super().__init__() + self.prompt = "" + self.auth_key = "xxx" + self.parameters = [] + self.source_lang = 'ZH' + self.target_lang = 'EN-GB' + + def check(self): + self.check_positive_integer(self.top_n, "Top N") + self.check_valid_value(self.source_lang, "Source language", + ['AR', 'BG', 'CS', 'DA', 'DE', 'EL', 'EN', 'ES', 'ET', 'FI', 'FR', 'HU', 'ID', 'IT', + 'JA', 'KO', 'LT', 'LV', 'NB', 'NL', 'PL', 'PT', 'RO', 'RU', 'SK', 'SL', 'SV', 'TR', + 'UK', 'ZH']) + self.check_valid_value(self.target_lang, "Target language", + ['AR', 'BG', 'CS', 'DA', 'DE', 'EL', 'EN-GB', 'EN-US', 'ES', 'ET', 'FI', 'FR', 'HU', + 'ID', 'IT', 'JA', 'KO', 'LT', 'LV', 'NB', 'NL', 'PL', 'PT-BR', 'PT-PT', 'RO', 'RU', + 'SK', 'SL', 'SV', 'TR', 'UK', 'ZH']) + + +class DeepL(ComponentBase, ABC): + component_name = "GitHub" + + def _run(self, history, **kwargs): + prompt = self._param.prompt + + ans = self.get_input() + ans = " - ".join(ans["content"]) if "content" in ans else "" + if not ans: + return DeepL.be_output("") + + for para in self._param.parameters: + cpn = self._canvas.get_component(para["component_id"])["obj"] + _, out = cpn.output(allow_partial=False) + if "content" not in out.columns: + kwargs[para["key"]] = "Nothing" + else: + kwargs[para["key"]] = " - " + "\n - ".join(out["content"]) + + kwargs["input"] = ans + for n, v in kwargs.items(): + prompt = re.sub(r"\{%s\}" % n, str(v), prompt) + + try: + translator = deepl.Translator(self._param.auth_key) + result = translator.translate_text(prompt, source_lang=self._param.source_lang, + target_lang=self._param.target_lang) + + return DeepL.be_output(result.text) + except Exception as e: + DeepL.be_output("**Error**:" + str(e)) diff --git a/agent/component/github.py b/agent/component/github.py new file mode 100644 index 0000000000..98180431e9 --- /dev/null +++ b/agent/component/github.py @@ -0,0 +1,61 @@ +# +# Copyright 2024 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from abc import ABC +import pandas as pd +import requests +from agent.settings import DEBUG +from agent.component.base import ComponentBase, ComponentParamBase + + +class GitHubParam(ComponentParamBase): + """ + Define the GitHub component parameters. + """ + + def __init__(self): + super().__init__() + self.top_n = 10 + + def check(self): + self.check_positive_integer(self.top_n, "Top N") + + +class GitHub(ComponentBase, ABC): + component_name = "GitHub" + + def _run(self, history, **kwargs): + ans = self.get_input() + ans = " - ".join(ans["content"]) if "content" in ans else "" + if not ans: + return GitHub.be_output("") + + try: + url = 'https://api.github.com/search/repositories?q=' + ans + '&sort=stars&order=desc&per_page=' + str( + self._param.top_n) + headers = {"Content-Type": "application/vnd.github+json", "X-GitHub-Api-Version": '2022-11-28'} + response = requests.get(url=url, headers=headers).json() + + github_res = [{"content": '' + i["name"] + '' + str( + i["description"]) + '\n stars:' + str(i['watchers'])} for i in response['items']] + except Exception as e: + return GitHub.be_output("**ERROR**: " + str(e)) + + if not github_res: + return GitHub.be_output("") + + df = pd.DataFrame(github_res) + if DEBUG: print(df, ":::::::::::::::::::::::::::::::::") + return df diff --git a/requirements.txt b/requirements.txt index b87c7e9275..30f92db2d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,7 @@ cn2an==0.5.22 cohere==5.6.2 dashscope==1.14.1 datrie==0.8.2 +deepl==1.18.0 demjson3==3.0.6 discord.py==2.3.2 duckduckgo_search==6.1.9 diff --git a/requirements_arm.txt b/requirements_arm.txt index 68e6f33f2c..58e62d51d6 100644 --- a/requirements_arm.txt +++ b/requirements_arm.txt @@ -159,3 +159,4 @@ google_search_results==2.4.2 editdistance==0.8.1 markdown_to_json==2.1.1 scholarly==1.7.11 +deepl==1.18.0 diff --git a/requirements_dev.txt b/requirements_dev.txt index bb9f164506..636989f357 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -144,3 +144,4 @@ google_search_results==2.4.2 editdistance==0.8.1 markdown_to_json==2.1.1 scholarly==1.7.11 +deepl==1.18.0