From 5e6fc58db3700214634fb6b43e373c165e5ec32f Mon Sep 17 00:00:00 2001 From: -LAN- Date: Mon, 22 Jul 2024 15:29:39 +0800 Subject: [PATCH] Feat/environment variables in workflow (#6515) Co-authored-by: JzoNg --- .gitignore | 1 + api/app.py | 9 +- api/configs/feature/__init__.py | 1 - api/constants/__init__.py | 2 + api/contexts/__init__.py | 3 + api/controllers/console/app/app.py | 9 +- api/controllers/console/app/workflow.py | 19 +- .../app/app_config/base_app_config_manager.py | 29 +-- .../features/file_upload/manager.py | 5 +- .../features/text_to_speech/manager.py | 4 +- .../app/apps/advanced_chat/app_generator.py | 12 +- api/core/app/apps/advanced_chat/app_runner.py | 8 +- .../generate_response_converter.py | 15 +- .../advanced_chat/generate_task_pipeline.py | 8 +- .../workflow_event_trigger_callback.py | 4 +- .../base_app_generate_response_converter.py | 27 +-- api/core/app/apps/workflow/app_generator.py | 17 +- api/core/app/apps/workflow/app_runner.py | 3 +- .../workflow_event_trigger_callback.py | 4 +- .../app/apps/workflow_logging_callback.py | 4 +- api/core/app/entities/app_invoke_entities.py | 5 +- api/core/app/segments/__init__.py | 27 +++ api/core/app/segments/factory.py | 64 ++++++ api/core/app/segments/parser.py | 17 ++ api/core/app/segments/segment_group.py | 19 ++ api/core/app/segments/segments.py | 39 ++++ api/core/app/segments/types.py | 17 ++ api/core/app/segments/variables.py | 83 +++++++ .../agent_tool_callback_handler.py | 8 +- api/core/file/message_file_parser.py | 5 +- .../helper/code_executor/code_executor.py | 6 +- api/core/helper/encrypter.py | 9 +- api/core/model_manager.py | 1 + .../model_providers/azure_openai/llm/llm.py | 2 +- .../rag/datasource/keyword/keyword_base.py | 1 + api/core/rag/datasource/vdb/vector_base.py | 1 + api/core/rag/datasource/vdb/vector_factory.py | 1 + api/core/tools/tool/tool.py | 8 +- api/core/tools/tool_engine.py | 11 +- .../callbacks/base_workflow_callback.py | 4 +- api/core/workflow/entities/node_entities.py | 5 +- api/core/workflow/entities/variable_pool.py | 180 +++++++++------ api/core/workflow/nodes/answer/answer_node.py | 38 +--- api/core/workflow/nodes/base_node.py | 14 +- api/core/workflow/nodes/code/code_node.py | 7 +- api/core/workflow/nodes/end/end_node.py | 15 +- .../nodes/http_request/http_executor.py | 21 +- .../nodes/http_request/http_request_node.py | 4 + .../workflow/nodes/if_else/if_else_node.py | 19 +- .../nodes/iteration/iteration_node.py | 15 +- .../knowledge_retrieval_node.py | 3 +- api/core/workflow/nodes/llm/llm_node.py | 18 +- .../parameter_extractor_node.py | 8 +- .../question_classifier_node.py | 6 +- api/core/workflow/nodes/start/start_node.py | 4 +- .../template_transform_node.py | 9 +- api/core/workflow/nodes/tool/entities.py | 1 + api/core/workflow/nodes/tool/tool_node.py | 80 +++---- .../variable_aggregator_node.py | 21 +- .../utils/variable_template_parser.py | 50 ++++- api/core/workflow/workflow_engine_manager.py | 92 ++++---- api/fields/workflow_fields.py | 31 +++ ...e_add_environment_variable_to_workflow_.py | 33 +++ api/models/account.py | 4 +- api/models/workflow.py | 86 ++++++- api/schedule/clean_unused_datasets_task.py | 2 +- api/services/account_service.py | 4 +- api/services/app_dsl_service.py | 22 +- api/services/model_load_balancing_service.py | 1 + api/services/recommended_app_service.py | 1 - api/services/workflow/workflow_converter.py | 3 +- api/services/workflow_service.py | 30 ++- .../workflow/nodes/test_code.py | 12 +- .../workflow/nodes/test_http.py | 6 +- .../workflow/nodes/test_llm.py | 8 +- .../nodes/test_parameter_extractor.py | 10 +- .../workflow/nodes/test_template_transform.py | 6 +- .../workflow/nodes/test_tool.py | 8 +- api/tests/unit_tests/app/test_segment.py | 53 +++++ api/tests/unit_tests/app/test_variables.py | 91 ++++++++ .../unit_tests/configs/test_dify_config.py | 5 +- .../core/workflow/nodes/test_answer.py | 6 +- .../core/workflow/nodes/test_if_else.py | 42 ++-- api/tests/unit_tests/models/test_workflow.py | 95 ++++++++ web/app/(commonLayout)/apps/AppCard.tsx | 39 +++- web/app/components/app-sidebar/app-info.tsx | 55 ++++- .../components/app/app-publisher/index.tsx | 4 +- .../line/communication/message-play.svg | 5 - .../icons/assets/vender/line/others/env.svg | 11 + .../line/communication/MessagePlay.json | 39 ---- .../src/vender/line/communication/index.ts | 1 - .../icons/src/vender/line/others/Env.json | 90 ++++++++ .../MessagePlay.tsx => others/Env.tsx} | 4 +- .../icons/src/vender/line/others/index.ts | 1 + .../workflow-variable-block/component.tsx | 47 ++-- web/app/components/workflow/constants.ts | 1 + .../workflow/dsl-export-confirm-modal.tsx | 85 +++++++ .../components/workflow/header/checklist.tsx | 13 +- .../components/workflow/header/env-button.tsx | 22 ++ web/app/components/workflow/header/index.tsx | 31 +-- .../workflow/header/run-and-history.tsx | 27 ++- .../workflow/header/view-history.tsx | 8 +- web/app/components/workflow/hooks/index.ts | 1 + .../workflow/hooks/use-nodes-sync-draft.ts | 2 + .../hooks/use-workflow-interactions.ts | 40 +++- .../workflow/hooks/use-workflow-run.ts | 6 + .../workflow/hooks/use-workflow-start-run.tsx | 6 + .../workflow/hooks/use-workflow-variables.ts | 69 ++++++ .../components/workflow/hooks/use-workflow.ts | 9 +- web/app/components/workflow/index.tsx | 18 ++ .../add-variable-popup-with-position.tsx | 16 +- .../readonly-input-with-select-var.tsx | 29 ++- .../nodes/_base/components/variable-tag.tsx | 44 ++-- .../nodes/_base/components/variable/utils.ts | 81 +++++-- .../variable/var-reference-picker.tsx | 19 +- .../variable/var-reference-vars.tsx | 16 +- .../_base/hooks/use-available-var-list.ts | 9 +- .../nodes/_base/hooks/use-one-step-run.ts | 4 +- .../workflow/nodes/code/use-config.ts | 2 +- .../components/workflow/nodes/end/node.tsx | 34 ++- .../nodes/http/components/api-input.tsx | 2 +- .../http/components/authorization/index.tsx | 49 +++- .../nodes/http/components/edit-body/index.tsx | 2 +- .../key-value/key-value-edit/input-item.tsx | 2 +- .../components/workflow/nodes/http/panel.tsx | 1 + .../workflow/nodes/http/use-config.ts | 2 +- .../if-else/components/condition-value.tsx | 8 +- .../workflow/nodes/llm/use-config.ts | 4 +- .../nodes/tool/components/input-var-list.tsx | 2 +- .../components/node-group-item.tsx | 6 +- .../components/node-variable-item.tsx | 28 ++- .../workflow/nodes/variable-assigner/hooks.ts | 27 ++- .../nodes/variable-assigner/panel.tsx | 4 +- .../components/workflow/panel-contextmenu.tsx | 4 +- .../workflow/panel/env-panel/index.tsx | 210 ++++++++++++++++++ .../panel/env-panel/variable-modal.tsx | 151 +++++++++++++ .../panel/env-panel/variable-trigger.tsx | 68 ++++++ web/app/components/workflow/panel/index.tsx | 11 +- .../workflow/panel/workflow-preview.tsx | 6 +- web/app/components/workflow/store.ts | 14 ++ web/app/components/workflow/types.ts | 8 + web/i18n/en-US/workflow.ts | 23 +- web/i18n/zh-Hans/workflow.ts | 23 +- web/service/apps.ts | 4 +- web/service/workflow.ts | 2 +- web/types/workflow.ts | 2 + 146 files changed, 2486 insertions(+), 746 deletions(-) create mode 100644 api/contexts/__init__.py create mode 100644 api/core/app/segments/__init__.py create mode 100644 api/core/app/segments/factory.py create mode 100644 api/core/app/segments/parser.py create mode 100644 api/core/app/segments/segment_group.py create mode 100644 api/core/app/segments/segments.py create mode 100644 api/core/app/segments/types.py create mode 100644 api/core/app/segments/variables.py create mode 100644 api/migrations/versions/8e5588e6412e_add_environment_variable_to_workflow_.py create mode 100644 api/tests/unit_tests/app/test_segment.py create mode 100644 api/tests/unit_tests/app/test_variables.py create mode 100644 api/tests/unit_tests/models/test_workflow.py delete mode 100644 web/app/components/base/icons/assets/vender/line/communication/message-play.svg create mode 100644 web/app/components/base/icons/assets/vender/line/others/env.svg delete mode 100644 web/app/components/base/icons/src/vender/line/communication/MessagePlay.json create mode 100644 web/app/components/base/icons/src/vender/line/others/Env.json rename web/app/components/base/icons/src/vender/line/{communication/MessagePlay.tsx => others/Env.tsx} (85%) create mode 100644 web/app/components/workflow/dsl-export-confirm-modal.tsx create mode 100644 web/app/components/workflow/header/env-button.tsx create mode 100644 web/app/components/workflow/hooks/use-workflow-variables.ts create mode 100644 web/app/components/workflow/panel/env-panel/index.tsx create mode 100644 web/app/components/workflow/panel/env-panel/variable-modal.tsx create mode 100644 web/app/components/workflow/panel/env-panel/variable-trigger.tsx diff --git a/.gitignore b/.gitignore index 2f44cf7934919b..97b7333ddeb2f4 100644 --- a/.gitignore +++ b/.gitignore @@ -174,5 +174,6 @@ sdks/python-client/dify_client.egg-info .vscode/* !.vscode/launch.json pyrightconfig.json +api/.vscode .idea/ diff --git a/api/app.py b/api/app.py index f5a6d40e1ac978..2c484ace850162 100644 --- a/api/app.py +++ b/api/app.py @@ -1,7 +1,5 @@ import os -from configs import dify_config - if os.environ.get("DEBUG", "false").lower() != 'true': from gevent import monkey @@ -23,7 +21,9 @@ from flask_cors import CORS from werkzeug.exceptions import Unauthorized +import contexts from commands import register_commands +from configs import dify_config # DO NOT REMOVE BELOW from events import event_handlers @@ -181,7 +181,10 @@ def load_user_from_request(request_from_flask_login): decoded = PassportService().verify(auth_token) user_id = decoded.get('user_id') - return AccountService.load_logged_in_account(account_id=user_id, token=auth_token) + account = AccountService.load_logged_in_account(account_id=user_id, token=auth_token) + if account: + contexts.tenant_id.set(account.current_tenant_id) + return account @login_manager.unauthorized_handler diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index c22a89b158f360..369b25d788a440 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -406,7 +406,6 @@ class DataSetConfig(BaseSettings): default=False, ) - class WorkspaceConfig(BaseSettings): """ Workspace configs diff --git a/api/constants/__init__.py b/api/constants/__init__.py index e69de29bb2d1d6..08a27869948c95 100644 --- a/api/constants/__init__.py +++ b/api/constants/__init__.py @@ -0,0 +1,2 @@ +# TODO: Update all string in code to use this constant +HIDDEN_VALUE = '[__HIDDEN__]' \ No newline at end of file diff --git a/api/contexts/__init__.py b/api/contexts/__init__.py new file mode 100644 index 00000000000000..306fac3a931298 --- /dev/null +++ b/api/contexts/__init__.py @@ -0,0 +1,3 @@ +from contextvars import ContextVar + +tenant_id: ContextVar[str] = ContextVar('tenant_id') \ No newline at end of file diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index 1c42a57d43e3de..2f304b970c6050 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -212,7 +212,7 @@ def post(self, app_model): parser.add_argument('icon_background', type=str, location='json') args = parser.parse_args() - data = AppDslService.export_dsl(app_model=app_model) + data = AppDslService.export_dsl(app_model=app_model, include_secret=True) app = AppDslService.import_and_create_new_app( tenant_id=current_user.current_tenant_id, data=data, @@ -234,8 +234,13 @@ def get(self, app_model): if not current_user.is_editor: raise Forbidden() + # Add include_secret params + parser = reqparse.RequestParser() + parser.add_argument('include_secret', type=inputs.boolean, default=False, location='args') + args = parser.parse_args() + return { - "data": AppDslService.export_dsl(app_model=app_model) + "data": AppDslService.export_dsl(app_model=app_model, include_secret=args['include_secret']) } diff --git a/api/controllers/console/app/workflow.py b/api/controllers/console/app/workflow.py index 9f745ca120acf7..686ef7b4bebaaa 100644 --- a/api/controllers/console/app/workflow.py +++ b/api/controllers/console/app/workflow.py @@ -13,6 +13,7 @@ from controllers.console.wraps import account_initialization_required from core.app.apps.base_app_queue_manager import AppQueueManager from core.app.entities.app_invoke_entities import InvokeFrom +from core.app.segments import factory from core.errors.error import AppInvokeQuotaExceededError from fields.workflow_fields import workflow_fields from fields.workflow_run_fields import workflow_run_node_execution_fields @@ -41,7 +42,7 @@ def get(self, app_model: App): # The role of the current user in the ta table must be admin, owner, or editor if not current_user.is_editor: raise Forbidden() - + # fetch draft workflow by app_model workflow_service = WorkflowService() workflow = workflow_service.get_draft_workflow(app_model=app_model) @@ -64,13 +65,15 @@ def post(self, app_model: App): if not current_user.is_editor: raise Forbidden() - content_type = request.headers.get('Content-Type') + content_type = request.headers.get('Content-Type', '') if 'application/json' in content_type: parser = reqparse.RequestParser() parser.add_argument('graph', type=dict, required=True, nullable=False, location='json') parser.add_argument('features', type=dict, required=True, nullable=False, location='json') parser.add_argument('hash', type=str, required=False, location='json') + # TODO: set this to required=True after frontend is updated + parser.add_argument('environment_variables', type=list, required=False, location='json') args = parser.parse_args() elif 'text/plain' in content_type: try: @@ -84,7 +87,8 @@ def post(self, app_model: App): args = { 'graph': data.get('graph'), 'features': data.get('features'), - 'hash': data.get('hash') + 'hash': data.get('hash'), + 'environment_variables': data.get('environment_variables') } except json.JSONDecodeError: return {'message': 'Invalid JSON data'}, 400 @@ -94,12 +98,15 @@ def post(self, app_model: App): workflow_service = WorkflowService() try: + environment_variables_list = args.get('environment_variables') or [] + environment_variables = [factory.build_variable_from_mapping(obj) for obj in environment_variables_list] workflow = workflow_service.sync_draft_workflow( app_model=app_model, - graph=args.get('graph'), - features=args.get('features'), + graph=args['graph'], + features=args['features'], unique_hash=args.get('hash'), - account=current_user + account=current_user, + environment_variables=environment_variables, ) except WorkflowHashNotEqualError: raise DraftWorkflowNotSync() diff --git a/api/core/app/app_config/base_app_config_manager.py b/api/core/app/app_config/base_app_config_manager.py index 353fe85b74374a..3dea305e984143 100644 --- a/api/core/app/app_config/base_app_config_manager.py +++ b/api/core/app/app_config/base_app_config_manager.py @@ -1,6 +1,7 @@ -from typing import Optional, Union +from collections.abc import Mapping +from typing import Any -from core.app.app_config.entities import AppAdditionalFeatures, EasyUIBasedAppModelConfigFrom +from core.app.app_config.entities import AppAdditionalFeatures from core.app.app_config.features.file_upload.manager import FileUploadConfigManager from core.app.app_config.features.more_like_this.manager import MoreLikeThisConfigManager from core.app.app_config.features.opening_statement.manager import OpeningStatementConfigManager @@ -10,37 +11,19 @@ SuggestedQuestionsAfterAnswerConfigManager, ) from core.app.app_config.features.text_to_speech.manager import TextToSpeechConfigManager -from models.model import AppMode, AppModelConfig +from models.model import AppMode class BaseAppConfigManager: - - @classmethod - def convert_to_config_dict(cls, config_from: EasyUIBasedAppModelConfigFrom, - app_model_config: Union[AppModelConfig, dict], - config_dict: Optional[dict] = None) -> dict: - """ - Convert app model config to config dict - :param config_from: app model config from - :param app_model_config: app model config - :param config_dict: app model config dict - :return: - """ - if config_from != EasyUIBasedAppModelConfigFrom.ARGS: - app_model_config_dict = app_model_config.to_dict() - config_dict = app_model_config_dict.copy() - - return config_dict - @classmethod - def convert_features(cls, config_dict: dict, app_mode: AppMode) -> AppAdditionalFeatures: + def convert_features(cls, config_dict: Mapping[str, Any], app_mode: AppMode) -> AppAdditionalFeatures: """ Convert app config to app model config :param config_dict: app config :param app_mode: app mode """ - config_dict = config_dict.copy() + config_dict = dict(config_dict.items()) additional_features = AppAdditionalFeatures() additional_features.show_retrieve_source = RetrievalResourceConfigManager.convert( diff --git a/api/core/app/app_config/features/file_upload/manager.py b/api/core/app/app_config/features/file_upload/manager.py index 2049b573cdf8e7..86799fb1abe133 100644 --- a/api/core/app/app_config/features/file_upload/manager.py +++ b/api/core/app/app_config/features/file_upload/manager.py @@ -1,11 +1,12 @@ -from typing import Optional +from collections.abc import Mapping +from typing import Any, Optional from core.app.app_config.entities import FileExtraConfig class FileUploadConfigManager: @classmethod - def convert(cls, config: dict, is_vision: bool = True) -> Optional[FileExtraConfig]: + def convert(cls, config: Mapping[str, Any], is_vision: bool = True) -> Optional[FileExtraConfig]: """ Convert model config to model config diff --git a/api/core/app/app_config/features/text_to_speech/manager.py b/api/core/app/app_config/features/text_to_speech/manager.py index b516fa46abd7d3..f11e268e7380db 100644 --- a/api/core/app/app_config/features/text_to_speech/manager.py +++ b/api/core/app/app_config/features/text_to_speech/manager.py @@ -3,13 +3,13 @@ class TextToSpeechConfigManager: @classmethod - def convert(cls, config: dict) -> bool: + def convert(cls, config: dict): """ Convert model config to model config :param config: model config args """ - text_to_speech = False + text_to_speech = None text_to_speech_dict = config.get('text_to_speech') if text_to_speech_dict: if text_to_speech_dict.get('enabled'): diff --git a/api/core/app/apps/advanced_chat/app_generator.py b/api/core/app/apps/advanced_chat/app_generator.py index 84723cb5c7bb57..0141dbec58de6e 100644 --- a/api/core/app/apps/advanced_chat/app_generator.py +++ b/api/core/app/apps/advanced_chat/app_generator.py @@ -1,3 +1,4 @@ +import contextvars import logging import os import threading @@ -8,6 +9,7 @@ from flask import Flask, current_app from pydantic import ValidationError +import contexts from core.app.app_config.features.file_upload.manager import FileUploadConfigManager from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager from core.app.apps.advanced_chat.app_runner import AdvancedChatAppRunner @@ -107,6 +109,7 @@ def generate( extras=extras, trace_manager=trace_manager ) + contexts.tenant_id.set(application_generate_entity.app_config.tenant_id) return self._generate( app_model=app_model, @@ -173,6 +176,7 @@ def single_iteration_generate(self, app_model: App, inputs=args['inputs'] ) ) + contexts.tenant_id.set(application_generate_entity.app_config.tenant_id) return self._generate( app_model=app_model, @@ -225,6 +229,8 @@ def _generate(self, app_model: App, 'queue_manager': queue_manager, 'conversation_id': conversation.id, 'message_id': message.id, + 'user': user, + 'context': contextvars.copy_context() }) worker_thread.start() @@ -249,7 +255,9 @@ def _generate_worker(self, flask_app: Flask, application_generate_entity: AdvancedChatAppGenerateEntity, queue_manager: AppQueueManager, conversation_id: str, - message_id: str) -> None: + message_id: str, + user: Account, + context: contextvars.Context) -> None: """ Generate worker in a new thread. :param flask_app: Flask app @@ -259,6 +267,8 @@ def _generate_worker(self, flask_app: Flask, :param message_id: message ID :return: """ + for var, val in context.items(): + var.set(val) with flask_app.app_context(): try: runner = AdvancedChatAppRunner() diff --git a/api/core/app/apps/advanced_chat/app_runner.py b/api/core/app/apps/advanced_chat/app_runner.py index 38566217007f46..18db0ab22d4ded 100644 --- a/api/core/app/apps/advanced_chat/app_runner.py +++ b/api/core/app/apps/advanced_chat/app_runner.py @@ -1,7 +1,8 @@ import logging import os import time -from typing import Optional, cast +from collections.abc import Mapping +from typing import Any, Optional, cast from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfig from core.app.apps.advanced_chat.workflow_event_trigger_callback import WorkflowEventTriggerCallback @@ -14,6 +15,7 @@ ) from core.app.entities.queue_entities import QueueAnnotationReplyEvent, QueueStopEvent, QueueTextChunkEvent from core.moderation.base import ModerationException +from core.workflow.callbacks.base_workflow_callback import WorkflowCallback from core.workflow.entities.node_entities import SystemVariable from core.workflow.nodes.base_node import UserFrom from core.workflow.workflow_engine_manager import WorkflowEngineManager @@ -87,7 +89,7 @@ def run(self, application_generate_entity: AdvancedChatAppGenerateEntity, db.session.close() - workflow_callbacks = [WorkflowEventTriggerCallback( + workflow_callbacks: list[WorkflowCallback] = [WorkflowEventTriggerCallback( queue_manager=queue_manager, workflow=workflow )] @@ -161,7 +163,7 @@ def handle_input_moderation( self, queue_manager: AppQueueManager, app_record: App, app_generate_entity: AdvancedChatAppGenerateEntity, - inputs: dict, + inputs: Mapping[str, Any], query: str, message_id: str ) -> bool: diff --git a/api/core/app/apps/advanced_chat/generate_response_converter.py b/api/core/app/apps/advanced_chat/generate_response_converter.py index 08069332ba3e54..ef579827b47c7e 100644 --- a/api/core/app/apps/advanced_chat/generate_response_converter.py +++ b/api/core/app/apps/advanced_chat/generate_response_converter.py @@ -1,9 +1,11 @@ import json from collections.abc import Generator -from typing import cast +from typing import Any, cast from core.app.apps.base_app_generate_response_converter import AppGenerateResponseConverter from core.app.entities.task_entities import ( + AppBlockingResponse, + AppStreamResponse, ChatbotAppBlockingResponse, ChatbotAppStreamResponse, ErrorStreamResponse, @@ -18,12 +20,13 @@ class AdvancedChatAppGenerateResponseConverter(AppGenerateResponseConverter): _blocking_response_type = ChatbotAppBlockingResponse @classmethod - def convert_blocking_full_response(cls, blocking_response: ChatbotAppBlockingResponse) -> dict: + def convert_blocking_full_response(cls, blocking_response: AppBlockingResponse) -> dict[str, Any]: """ Convert blocking full response. :param blocking_response: blocking response :return: """ + blocking_response = cast(ChatbotAppBlockingResponse, blocking_response) response = { 'event': 'message', 'task_id': blocking_response.task_id, @@ -39,7 +42,7 @@ def convert_blocking_full_response(cls, blocking_response: ChatbotAppBlockingRes return response @classmethod - def convert_blocking_simple_response(cls, blocking_response: ChatbotAppBlockingResponse) -> dict: + def convert_blocking_simple_response(cls, blocking_response: AppBlockingResponse) -> dict[str, Any]: """ Convert blocking simple response. :param blocking_response: blocking response @@ -53,8 +56,7 @@ def convert_blocking_simple_response(cls, blocking_response: ChatbotAppBlockingR return response @classmethod - def convert_stream_full_response(cls, stream_response: Generator[ChatbotAppStreamResponse, None, None]) \ - -> Generator[str, None, None]: + def convert_stream_full_response(cls, stream_response: Generator[AppStreamResponse, None, None]) -> Generator[str, Any, None]: """ Convert stream full response. :param stream_response: stream response @@ -83,8 +85,7 @@ def convert_stream_full_response(cls, stream_response: Generator[ChatbotAppStrea yield json.dumps(response_chunk) @classmethod - def convert_stream_simple_response(cls, stream_response: Generator[ChatbotAppStreamResponse, None, None]) \ - -> Generator[str, None, None]: + def convert_stream_simple_response(cls, stream_response: Generator[AppStreamResponse, None, None]) -> Generator[str, Any, None]: """ Convert stream simple response. :param stream_response: stream response diff --git a/api/core/app/apps/advanced_chat/generate_task_pipeline.py b/api/core/app/apps/advanced_chat/generate_task_pipeline.py index 4b089f033f3d93..be72d89c1e1bd1 100644 --- a/api/core/app/apps/advanced_chat/generate_task_pipeline.py +++ b/api/core/app/apps/advanced_chat/generate_task_pipeline.py @@ -118,7 +118,7 @@ def __init__( self._stream_generate_routes = self._get_stream_generate_routes() self._conversation_name_generate_thread = None - def process(self) -> Union[ChatbotAppBlockingResponse, Generator[ChatbotAppStreamResponse, None, None]]: + def process(self): """ Process generate task pipeline. :return: @@ -141,8 +141,7 @@ def process(self) -> Union[ChatbotAppBlockingResponse, Generator[ChatbotAppStrea else: return self._to_blocking_response(generator) - def _to_blocking_response(self, generator: Generator[StreamResponse, None, None]) \ - -> ChatbotAppBlockingResponse: + def _to_blocking_response(self, generator: Generator[StreamResponse, None, None]) -> ChatbotAppBlockingResponse: """ Process blocking response. :return: @@ -172,8 +171,7 @@ def _to_blocking_response(self, generator: Generator[StreamResponse, None, None] raise Exception('Queue listening stopped unexpectedly.') - def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) \ - -> Generator[ChatbotAppStreamResponse, None, None]: + def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) -> Generator[ChatbotAppStreamResponse, Any, None]: """ To stream response. :return: diff --git a/api/core/app/apps/advanced_chat/workflow_event_trigger_callback.py b/api/core/app/apps/advanced_chat/workflow_event_trigger_callback.py index 78fe077e6b06ab..8d43155a0886bf 100644 --- a/api/core/app/apps/advanced_chat/workflow_event_trigger_callback.py +++ b/api/core/app/apps/advanced_chat/workflow_event_trigger_callback.py @@ -14,13 +14,13 @@ QueueWorkflowStartedEvent, QueueWorkflowSucceededEvent, ) -from core.workflow.callbacks.base_workflow_callback import BaseWorkflowCallback +from core.workflow.callbacks.base_workflow_callback import WorkflowCallback from core.workflow.entities.base_node_data_entities import BaseNodeData from core.workflow.entities.node_entities import NodeType from models.workflow import Workflow -class WorkflowEventTriggerCallback(BaseWorkflowCallback): +class WorkflowEventTriggerCallback(WorkflowCallback): def __init__(self, queue_manager: AppQueueManager, workflow: Workflow): self._queue_manager = queue_manager diff --git a/api/core/app/apps/base_app_generate_response_converter.py b/api/core/app/apps/base_app_generate_response_converter.py index bacd1a5477c999..1165314a7f2bd8 100644 --- a/api/core/app/apps/base_app_generate_response_converter.py +++ b/api/core/app/apps/base_app_generate_response_converter.py @@ -1,7 +1,7 @@ import logging from abc import ABC, abstractmethod from collections.abc import Generator -from typing import Union +from typing import Any, Union from core.app.entities.app_invoke_entities import InvokeFrom from core.app.entities.task_entities import AppBlockingResponse, AppStreamResponse @@ -15,44 +15,41 @@ class AppGenerateResponseConverter(ABC): @classmethod def convert(cls, response: Union[ AppBlockingResponse, - Generator[AppStreamResponse, None, None] - ], invoke_from: InvokeFrom) -> Union[ - dict, - Generator[str, None, None] - ]: + Generator[AppStreamResponse, Any, None] + ], invoke_from: InvokeFrom): if invoke_from in [InvokeFrom.DEBUGGER, InvokeFrom.SERVICE_API]: - if isinstance(response, cls._blocking_response_type): + if isinstance(response, AppBlockingResponse): return cls.convert_blocking_full_response(response) else: - def _generate(): + def _generate_full_response() -> Generator[str, Any, None]: for chunk in cls.convert_stream_full_response(response): if chunk == 'ping': yield f'event: {chunk}\n\n' else: yield f'data: {chunk}\n\n' - return _generate() + return _generate_full_response() else: - if isinstance(response, cls._blocking_response_type): + if isinstance(response, AppBlockingResponse): return cls.convert_blocking_simple_response(response) else: - def _generate(): + def _generate_simple_response() -> Generator[str, Any, None]: for chunk in cls.convert_stream_simple_response(response): if chunk == 'ping': yield f'event: {chunk}\n\n' else: yield f'data: {chunk}\n\n' - return _generate() + return _generate_simple_response() @classmethod @abstractmethod - def convert_blocking_full_response(cls, blocking_response: AppBlockingResponse) -> dict: + def convert_blocking_full_response(cls, blocking_response: AppBlockingResponse) -> dict[str, Any]: raise NotImplementedError @classmethod @abstractmethod - def convert_blocking_simple_response(cls, blocking_response: AppBlockingResponse) -> dict: + def convert_blocking_simple_response(cls, blocking_response: AppBlockingResponse) -> dict[str, Any]: raise NotImplementedError @classmethod @@ -68,7 +65,7 @@ def convert_stream_simple_response(cls, stream_response: Generator[AppStreamResp raise NotImplementedError @classmethod - def _get_simple_metadata(cls, metadata: dict) -> dict: + def _get_simple_metadata(cls, metadata: dict[str, Any]): """ Get simple metadata. :param metadata: metadata diff --git a/api/core/app/apps/workflow/app_generator.py b/api/core/app/apps/workflow/app_generator.py index 0f547ca16427c1..b1986dbcee21f0 100644 --- a/api/core/app/apps/workflow/app_generator.py +++ b/api/core/app/apps/workflow/app_generator.py @@ -1,3 +1,4 @@ +import contextvars import logging import os import threading @@ -8,6 +9,7 @@ from flask import Flask, current_app from pydantic import ValidationError +import contexts from core.app.app_config.features.file_upload.manager import FileUploadConfigManager from core.app.apps.base_app_generator import BaseAppGenerator from core.app.apps.base_app_queue_manager import AppQueueManager, GenerateTaskStoppedException, PublishFrom @@ -38,7 +40,7 @@ def generate( invoke_from: InvokeFrom, stream: bool = True, call_depth: int = 0, - ) -> Union[dict, Generator[dict, None, None]]: + ): """ Generate App response. @@ -86,6 +88,7 @@ def generate( call_depth=call_depth, trace_manager=trace_manager ) + contexts.tenant_id.set(application_generate_entity.app_config.tenant_id) return self._generate( app_model=app_model, @@ -126,7 +129,8 @@ def _generate( worker_thread = threading.Thread(target=self._generate_worker, kwargs={ 'flask_app': current_app._get_current_object(), 'application_generate_entity': application_generate_entity, - 'queue_manager': queue_manager + 'queue_manager': queue_manager, + 'context': contextvars.copy_context() }) worker_thread.start() @@ -150,8 +154,7 @@ def single_iteration_generate(self, app_model: App, node_id: str, user: Account, args: dict, - stream: bool = True) \ - -> Union[dict, Generator[dict, None, None]]: + stream: bool = True): """ Generate App response. @@ -193,6 +196,7 @@ def single_iteration_generate(self, app_model: App, inputs=args['inputs'] ) ) + contexts.tenant_id.set(application_generate_entity.app_config.tenant_id) return self._generate( app_model=app_model, @@ -205,7 +209,8 @@ def single_iteration_generate(self, app_model: App, def _generate_worker(self, flask_app: Flask, application_generate_entity: WorkflowAppGenerateEntity, - queue_manager: AppQueueManager) -> None: + queue_manager: AppQueueManager, + context: contextvars.Context) -> None: """ Generate worker in a new thread. :param flask_app: Flask app @@ -213,6 +218,8 @@ def _generate_worker(self, flask_app: Flask, :param queue_manager: queue manager :return: """ + for var, val in context.items(): + var.set(val) with flask_app.app_context(): try: # workflow app diff --git a/api/core/app/apps/workflow/app_runner.py b/api/core/app/apps/workflow/app_runner.py index 050319e5520b52..24f4a83217a239 100644 --- a/api/core/app/apps/workflow/app_runner.py +++ b/api/core/app/apps/workflow/app_runner.py @@ -10,6 +10,7 @@ InvokeFrom, WorkflowAppGenerateEntity, ) +from core.workflow.callbacks.base_workflow_callback import WorkflowCallback from core.workflow.entities.node_entities import SystemVariable from core.workflow.nodes.base_node import UserFrom from core.workflow.workflow_engine_manager import WorkflowEngineManager @@ -57,7 +58,7 @@ def run(self, application_generate_entity: WorkflowAppGenerateEntity, db.session.close() - workflow_callbacks = [WorkflowEventTriggerCallback( + workflow_callbacks: list[WorkflowCallback] = [WorkflowEventTriggerCallback( queue_manager=queue_manager, workflow=workflow )] diff --git a/api/core/app/apps/workflow/workflow_event_trigger_callback.py b/api/core/app/apps/workflow/workflow_event_trigger_callback.py index e423a40bcb12f9..4472a7e9b5a85c 100644 --- a/api/core/app/apps/workflow/workflow_event_trigger_callback.py +++ b/api/core/app/apps/workflow/workflow_event_trigger_callback.py @@ -14,13 +14,13 @@ QueueWorkflowStartedEvent, QueueWorkflowSucceededEvent, ) -from core.workflow.callbacks.base_workflow_callback import BaseWorkflowCallback +from core.workflow.callbacks.base_workflow_callback import WorkflowCallback from core.workflow.entities.base_node_data_entities import BaseNodeData from core.workflow.entities.node_entities import NodeType from models.workflow import Workflow -class WorkflowEventTriggerCallback(BaseWorkflowCallback): +class WorkflowEventTriggerCallback(WorkflowCallback): def __init__(self, queue_manager: AppQueueManager, workflow: Workflow): self._queue_manager = queue_manager diff --git a/api/core/app/apps/workflow_logging_callback.py b/api/core/app/apps/workflow_logging_callback.py index f617c671e98c72..2e6431d6d05f4d 100644 --- a/api/core/app/apps/workflow_logging_callback.py +++ b/api/core/app/apps/workflow_logging_callback.py @@ -2,7 +2,7 @@ from core.app.entities.queue_entities import AppQueueEvent from core.model_runtime.utils.encoders import jsonable_encoder -from core.workflow.callbacks.base_workflow_callback import BaseWorkflowCallback +from core.workflow.callbacks.base_workflow_callback import WorkflowCallback from core.workflow.entities.base_node_data_entities import BaseNodeData from core.workflow.entities.node_entities import NodeType @@ -15,7 +15,7 @@ } -class WorkflowLoggingCallback(BaseWorkflowCallback): +class WorkflowLoggingCallback(WorkflowCallback): def __init__(self) -> None: self.current_node_id = None diff --git a/api/core/app/entities/app_invoke_entities.py b/api/core/app/entities/app_invoke_entities.py index 1d2ad4a3735063..9a861c29e2634c 100644 --- a/api/core/app/entities/app_invoke_entities.py +++ b/api/core/app/entities/app_invoke_entities.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from enum import Enum from typing import Any, Optional @@ -76,7 +77,7 @@ class AppGenerateEntity(BaseModel): # app config app_config: AppConfig - inputs: dict[str, Any] + inputs: Mapping[str, Any] files: list[FileVar] = [] user_id: str @@ -140,7 +141,7 @@ class AdvancedChatAppGenerateEntity(AppGenerateEntity): app_config: WorkflowUIBasedAppConfig conversation_id: Optional[str] = None - query: Optional[str] = None + query: str class SingleIterationRunEntity(BaseModel): """ diff --git a/api/core/app/segments/__init__.py b/api/core/app/segments/__init__.py new file mode 100644 index 00000000000000..e5cecd35fd5682 --- /dev/null +++ b/api/core/app/segments/__init__.py @@ -0,0 +1,27 @@ +from .segment_group import SegmentGroup +from .segments import Segment +from .types import SegmentType +from .variables import ( + ArrayVariable, + FileVariable, + FloatVariable, + IntegerVariable, + ObjectVariable, + SecretVariable, + StringVariable, + Variable, +) + +__all__ = [ + 'IntegerVariable', + 'FloatVariable', + 'ObjectVariable', + 'SecretVariable', + 'FileVariable', + 'StringVariable', + 'ArrayVariable', + 'Variable', + 'SegmentType', + 'SegmentGroup', + 'Segment' +] diff --git a/api/core/app/segments/factory.py b/api/core/app/segments/factory.py new file mode 100644 index 00000000000000..4f0b361d956e53 --- /dev/null +++ b/api/core/app/segments/factory.py @@ -0,0 +1,64 @@ +from collections.abc import Mapping +from typing import Any + +from core.file.file_obj import FileVar + +from .segments import Segment, StringSegment +from .types import SegmentType +from .variables import ( + ArrayVariable, + FileVariable, + FloatVariable, + IntegerVariable, + ObjectVariable, + SecretVariable, + StringVariable, + Variable, +) + + +def build_variable_from_mapping(m: Mapping[str, Any], /) -> Variable: + if (value_type := m.get('value_type')) is None: + raise ValueError('missing value type') + if not m.get('name'): + raise ValueError('missing name') + if (value := m.get('value')) is None: + raise ValueError('missing value') + match value_type: + case SegmentType.STRING: + return StringVariable.model_validate(m) + case SegmentType.NUMBER if isinstance(value, int): + return IntegerVariable.model_validate(m) + case SegmentType.NUMBER if isinstance(value, float): + return FloatVariable.model_validate(m) + case SegmentType.SECRET: + return SecretVariable.model_validate(m) + case SegmentType.NUMBER if not isinstance(value, float | int): + raise ValueError(f'invalid number value {value}') + raise ValueError(f'not supported value type {value_type}') + + +def build_anonymous_variable(value: Any, /) -> Variable: + if isinstance(value, str): + return StringVariable(name='anonymous', value=value) + if isinstance(value, int): + return IntegerVariable(name='anonymous', value=value) + if isinstance(value, float): + return FloatVariable(name='anonymous', value=value) + if isinstance(value, dict): + # TODO: Limit the depth of the object + obj = {k: build_anonymous_variable(v) for k, v in value.items()} + return ObjectVariable(name='anonymous', value=obj) + if isinstance(value, list): + # TODO: Limit the depth of the array + elements = [build_anonymous_variable(v) for v in value] + return ArrayVariable(name='anonymous', value=elements) + if isinstance(value, FileVar): + return FileVariable(name='anonymous', value=value) + raise ValueError(f'not supported value {value}') + + +def build_segment(value: Any, /) -> Segment: + if isinstance(value, str): + return StringSegment(value=value) + raise ValueError(f'not supported value {value}') diff --git a/api/core/app/segments/parser.py b/api/core/app/segments/parser.py new file mode 100644 index 00000000000000..21d1b8954117ac --- /dev/null +++ b/api/core/app/segments/parser.py @@ -0,0 +1,17 @@ +import re + +from core.app.segments import SegmentGroup, factory +from core.workflow.entities.variable_pool import VariablePool + +VARIABLE_PATTERN = re.compile(r'\{\{#([a-zA-Z0-9_]{1,50}(?:\.[a-zA-Z_][a-zA-Z0-9_]{0,29}){1,10})#\}\}') + + +def convert_template(*, template: str, variable_pool: VariablePool): + parts = re.split(VARIABLE_PATTERN, template) + segments = [] + for part in parts: + if '.' in part and (value := variable_pool.get(part.split('.'))): + segments.append(value) + else: + segments.append(factory.build_segment(part)) + return SegmentGroup(segments=segments) diff --git a/api/core/app/segments/segment_group.py b/api/core/app/segments/segment_group.py new file mode 100644 index 00000000000000..0d5176b88586d6 --- /dev/null +++ b/api/core/app/segments/segment_group.py @@ -0,0 +1,19 @@ +from pydantic import BaseModel + +from .segments import Segment + + +class SegmentGroup(BaseModel): + segments: list[Segment] + + @property + def text(self): + return ''.join([segment.text for segment in self.segments]) + + @property + def log(self): + return ''.join([segment.log for segment in self.segments]) + + @property + def markdown(self): + return ''.join([segment.markdown for segment in self.segments]) \ No newline at end of file diff --git a/api/core/app/segments/segments.py b/api/core/app/segments/segments.py new file mode 100644 index 00000000000000..a6e953829e182b --- /dev/null +++ b/api/core/app/segments/segments.py @@ -0,0 +1,39 @@ +from typing import Any + +from pydantic import BaseModel, ConfigDict, field_validator + +from .types import SegmentType + + +class Segment(BaseModel): + model_config = ConfigDict(frozen=True) + + value_type: SegmentType + value: Any + + @field_validator('value_type') + def validate_value_type(cls, value): + """ + This validator checks if the provided value is equal to the default value of the 'value_type' field. + If the value is different, a ValueError is raised. + """ + if value != cls.model_fields['value_type'].default: + raise ValueError("Cannot modify 'value_type'") + return value + + @property + def text(self) -> str: + return str(self.value) + + @property + def log(self) -> str: + return str(self.value) + + @property + def markdown(self) -> str: + return str(self.value) + + +class StringSegment(Segment): + value_type: SegmentType = SegmentType.STRING + value: str diff --git a/api/core/app/segments/types.py b/api/core/app/segments/types.py new file mode 100644 index 00000000000000..517f210533e991 --- /dev/null +++ b/api/core/app/segments/types.py @@ -0,0 +1,17 @@ +from enum import Enum + + +class SegmentType(str, Enum): + STRING = 'string' + NUMBER = 'number' + FILE = 'file' + + SECRET = 'secret' + + OBJECT = 'object' + + ARRAY = 'array' + ARRAY_STRING = 'array[string]' + ARRAY_NUMBER = 'array[number]' + ARRAY_OBJECT = 'array[object]' + ARRAY_FILE = 'array[file]' \ No newline at end of file diff --git a/api/core/app/segments/variables.py b/api/core/app/segments/variables.py new file mode 100644 index 00000000000000..e600b442d69d73 --- /dev/null +++ b/api/core/app/segments/variables.py @@ -0,0 +1,83 @@ +import json +from collections.abc import Mapping, Sequence + +from pydantic import Field + +from core.file.file_obj import FileVar +from core.helper import encrypter + +from .segments import Segment, StringSegment +from .types import SegmentType + + +class Variable(Segment): + """ + A variable is a segment that has a name. + """ + + id: str = Field( + default='', + description="Unique identity for variable. It's only used by environment variables now.", + ) + name: str + + +class StringVariable(StringSegment, Variable): + pass + + +class FloatVariable(Variable): + value_type: SegmentType = SegmentType.NUMBER + value: float + + +class IntegerVariable(Variable): + value_type: SegmentType = SegmentType.NUMBER + value: int + + +class ObjectVariable(Variable): + value_type: SegmentType = SegmentType.OBJECT + value: Mapping[str, Variable] + + @property + def text(self) -> str: + # TODO: Process variables. + return json.dumps(self.model_dump()['value'], ensure_ascii=False) + + @property + def log(self) -> str: + # TODO: Process variables. + return json.dumps(self.model_dump()['value'], ensure_ascii=False, indent=2) + + @property + def markdown(self) -> str: + # TODO: Use markdown code block + return json.dumps(self.model_dump()['value'], ensure_ascii=False, indent=2) + + +class ArrayVariable(Variable): + value_type: SegmentType = SegmentType.ARRAY + value: Sequence[Variable] + + @property + def markdown(self) -> str: + return '\n'.join(['- ' + item.markdown for item in self.value]) + + +class FileVariable(Variable): + value_type: SegmentType = SegmentType.FILE + # TODO: embed FileVar in this model. + value: FileVar + + @property + def markdown(self) -> str: + return self.value.to_markdown() + + +class SecretVariable(StringVariable): + value_type: SegmentType = SegmentType.SECRET + + @property + def log(self) -> str: + return encrypter.obfuscated_token(self.value) diff --git a/api/core/callback_handler/agent_tool_callback_handler.py b/api/core/callback_handler/agent_tool_callback_handler.py index f973b7e1cec511..03f8244bab212e 100644 --- a/api/core/callback_handler/agent_tool_callback_handler.py +++ b/api/core/callback_handler/agent_tool_callback_handler.py @@ -1,9 +1,11 @@ import os +from collections.abc import Mapping, Sequence from typing import Any, Optional, TextIO, Union from pydantic import BaseModel from core.ops.ops_trace_manager import TraceQueueManager, TraceTask, TraceTaskName +from core.tools.entities.tool_entities import ToolInvokeMessage _TEXT_COLOR_MAPPING = { "blue": "36;1", @@ -43,7 +45,7 @@ def __init__(self, color: Optional[str] = None) -> None: def on_tool_start( self, tool_name: str, - tool_inputs: dict[str, Any], + tool_inputs: Mapping[str, Any], ) -> None: """Do nothing.""" print_text("\n[on_tool_start] ToolCall:" + tool_name + "\n" + str(tool_inputs) + "\n", color=self.color) @@ -51,8 +53,8 @@ def on_tool_start( def on_tool_end( self, tool_name: str, - tool_inputs: dict[str, Any], - tool_outputs: str, + tool_inputs: Mapping[str, Any], + tool_outputs: Sequence[ToolInvokeMessage], message_id: Optional[str] = None, timer: Optional[Any] = None, trace_manager: Optional[TraceQueueManager] = None diff --git a/api/core/file/message_file_parser.py b/api/core/file/message_file_parser.py index 842b539ad1b03b..7b2f8217f9231f 100644 --- a/api/core/file/message_file_parser.py +++ b/api/core/file/message_file_parser.py @@ -1,4 +1,5 @@ -from typing import Union +from collections.abc import Mapping, Sequence +from typing import Any, Union import requests @@ -16,7 +17,7 @@ def __init__(self, tenant_id: str, app_id: str) -> None: self.tenant_id = tenant_id self.app_id = app_id - def validate_and_transform_files_arg(self, files: list[dict], file_extra_config: FileExtraConfig, + def validate_and_transform_files_arg(self, files: Sequence[Mapping[str, Any]], file_extra_config: FileExtraConfig, user: Union[Account, EndUser]) -> list[FileVar]: """ validate and transform files arg diff --git a/api/core/helper/code_executor/code_executor.py b/api/core/helper/code_executor/code_executor.py index f094f7d79b58e0..5b69d3af4be350 100644 --- a/api/core/helper/code_executor/code_executor.py +++ b/api/core/helper/code_executor/code_executor.py @@ -21,7 +21,7 @@ CODE_EXECUTION_ENDPOINT = dify_config.CODE_EXECUTION_ENDPOINT CODE_EXECUTION_API_KEY = dify_config.CODE_EXECUTION_API_KEY -CODE_EXECUTION_TIMEOUT= (10, 60) +CODE_EXECUTION_TIMEOUT = (10, 60) class CodeExecutionException(Exception): pass @@ -64,7 +64,7 @@ class CodeExecutor: @classmethod def execute_code(cls, - language: Literal['python3', 'javascript', 'jinja2'], + language: CodeLanguage, preload: str, code: str, dependencies: Optional[list[CodeDependency]] = None) -> str: @@ -119,7 +119,7 @@ def execute_code(cls, return response.data.stdout @classmethod - def execute_workflow_code_template(cls, language: Literal['python3', 'javascript', 'jinja2'], code: str, inputs: dict, dependencies: Optional[list[CodeDependency]] = None) -> dict: + def execute_workflow_code_template(cls, language: CodeLanguage, code: str, inputs: dict, dependencies: Optional[list[CodeDependency]] = None) -> dict: """ Execute code :param language: code language diff --git a/api/core/helper/encrypter.py b/api/core/helper/encrypter.py index fcf293dc1cd78c..bf87a842c00bf4 100644 --- a/api/core/helper/encrypter.py +++ b/api/core/helper/encrypter.py @@ -6,11 +6,16 @@ def obfuscated_token(token: str): - return token[:6] + '*' * (len(token) - 8) + token[-2:] + if not token: + return token + if len(token) <= 8: + return '*' * 20 + return token[:6] + '*' * 12 + token[-2:] def encrypt_token(tenant_id: str, token: str): - tenant = db.session.query(Tenant).filter(Tenant.id == tenant_id).first() + if not (tenant := db.session.query(Tenant).filter(Tenant.id == tenant_id).first()): + raise ValueError(f'Tenant with id {tenant_id} not found') encrypted_token = rsa.encrypt(token, tenant.encrypt_public_key) return base64.b64encode(encrypted_token).decode() diff --git a/api/core/model_manager.py b/api/core/model_manager.py index d64db890f90f4c..dc7556f09a5917 100644 --- a/api/core/model_manager.py +++ b/api/core/model_manager.py @@ -413,6 +413,7 @@ def __init__(self, tenant_id: str, for load_balancing_config in self._load_balancing_configs: if load_balancing_config.name == "__inherit__": if not managed_credentials: + # FIXME: Mutation to loop iterable `self._load_balancing_configs` during iteration # remove __inherit__ if managed credentials is not provided self._load_balancing_configs.remove(load_balancing_config) else: diff --git a/api/core/model_runtime/model_providers/azure_openai/llm/llm.py b/api/core/model_runtime/model_providers/azure_openai/llm/llm.py index 25bc94cde6ed52..34d1f64210953a 100644 --- a/api/core/model_runtime/model_providers/azure_openai/llm/llm.py +++ b/api/core/model_runtime/model_providers/azure_openai/llm/llm.py @@ -501,7 +501,7 @@ def _convert_prompt_message_to_dict(message: PromptMessage): sub_messages.append(sub_message_dict) message_dict = {"role": "user", "content": sub_messages} elif isinstance(message, AssistantPromptMessage): - message = cast(AssistantPromptMessage, message) + # message = cast(AssistantPromptMessage, message) message_dict = {"role": "assistant", "content": message.content} if message.tool_calls: message_dict["tool_calls"] = [helper.dump_model(tool_call) for tool_call in message.tool_calls] diff --git a/api/core/rag/datasource/keyword/keyword_base.py b/api/core/rag/datasource/keyword/keyword_base.py index 02838cb1bd5254..67bc6df6fd4a32 100644 --- a/api/core/rag/datasource/keyword/keyword_base.py +++ b/api/core/rag/datasource/keyword/keyword_base.py @@ -42,6 +42,7 @@ def _filter_duplicate_texts(self, texts: list[Document]) -> list[Document]: doc_id = text.metadata['doc_id'] exists_duplicate_node = self.text_exists(doc_id) if exists_duplicate_node: + # FIXME: Mutation to loop iterable `texts` during iteration texts.remove(text) return texts diff --git a/api/core/rag/datasource/vdb/vector_base.py b/api/core/rag/datasource/vdb/vector_base.py index 17768ab042dd93..0b1d58856c4077 100644 --- a/api/core/rag/datasource/vdb/vector_base.py +++ b/api/core/rag/datasource/vdb/vector_base.py @@ -61,6 +61,7 @@ def _filter_duplicate_texts(self, texts: list[Document]) -> list[Document]: doc_id = text.metadata['doc_id'] exists_duplicate_node = self.text_exists(doc_id) if exists_duplicate_node: + # FIXME: Mutation to loop iterable `texts` during iteration texts.remove(text) return texts diff --git a/api/core/rag/datasource/vdb/vector_factory.py b/api/core/rag/datasource/vdb/vector_factory.py index 256abd28afbe53..509273e8eab6f0 100644 --- a/api/core/rag/datasource/vdb/vector_factory.py +++ b/api/core/rag/datasource/vdb/vector_factory.py @@ -157,6 +157,7 @@ def _filter_duplicate_texts(self, texts: list[Document]) -> list[Document]: doc_id = text.metadata['doc_id'] exists_duplicate_node = self.text_exists(doc_id) if exists_duplicate_node: + # FIXME: Mutation to loop iterable `texts` during iteration texts.remove(text) return texts diff --git a/api/core/tools/tool/tool.py b/api/core/tools/tool/tool.py index 04c09c7f5b8238..5d561911d12564 100644 --- a/api/core/tools/tool/tool.py +++ b/api/core/tools/tool/tool.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from collections.abc import Mapping from copy import deepcopy from enum import Enum from typing import Any, Optional, Union @@ -190,8 +191,9 @@ def list_default_image_variables(self) -> list[ToolRuntimeVariable]: return result - def invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> list[ToolInvokeMessage]: + def invoke(self, user_id: str, tool_parameters: Mapping[str, Any]) -> list[ToolInvokeMessage]: # update tool_parameters + # TODO: Fix type error. if self.runtime.runtime_parameters: tool_parameters.update(self.runtime.runtime_parameters) @@ -208,7 +210,7 @@ def invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> list[ToolInvo return result - def _transform_tool_parameters_type(self, tool_parameters: dict[str, Any]) -> dict[str, Any]: + def _transform_tool_parameters_type(self, tool_parameters: Mapping[str, Any]) -> dict[str, Any]: """ Transform tool parameters type """ @@ -241,7 +243,7 @@ def get_runtime_parameters(self) -> list[ToolParameter]: :return: the runtime parameters """ - return self.parameters + return self.parameters or [] def get_all_runtime_parameters(self) -> list[ToolParameter]: """ diff --git a/api/core/tools/tool_engine.py b/api/core/tools/tool_engine.py index 7615368934bfe5..0e15151aa49cba 100644 --- a/api/core/tools/tool_engine.py +++ b/api/core/tools/tool_engine.py @@ -1,4 +1,5 @@ import json +from collections.abc import Mapping from copy import deepcopy from datetime import datetime, timezone from mimetypes import guess_type @@ -46,7 +47,7 @@ def agent_invoke( if isinstance(tool_parameters, str): # check if this tool has only one parameter parameters = [ - parameter for parameter in tool.get_runtime_parameters() + parameter for parameter in tool.get_runtime_parameters() or [] if parameter.form == ToolParameter.ToolParameterForm.LLM ] if parameters and len(parameters) == 1: @@ -123,8 +124,8 @@ def agent_invoke( return error_response, [], ToolInvokeMeta.error_instance(error_response) @staticmethod - def workflow_invoke(tool: Tool, tool_parameters: dict, - user_id: str, workflow_id: str, + def workflow_invoke(tool: Tool, tool_parameters: Mapping[str, Any], + user_id: str, workflow_tool_callback: DifyWorkflowCallbackHandler, workflow_call_depth: int, ) -> list[ToolInvokeMessage]: @@ -141,7 +142,9 @@ def workflow_invoke(tool: Tool, tool_parameters: dict, if isinstance(tool, WorkflowTool): tool.workflow_call_depth = workflow_call_depth + 1 - response = tool.invoke(user_id, tool_parameters) + if tool.runtime and tool.runtime.runtime_parameters: + tool_parameters = {**tool.runtime.runtime_parameters, **tool_parameters} + response = tool.invoke(user_id=user_id, tool_parameters=tool_parameters) # hit the callback handler workflow_tool_callback.on_tool_end( diff --git a/api/core/workflow/callbacks/base_workflow_callback.py b/api/core/workflow/callbacks/base_workflow_callback.py index 3b0d51d86893a9..6db8adf4c21d72 100644 --- a/api/core/workflow/callbacks/base_workflow_callback.py +++ b/api/core/workflow/callbacks/base_workflow_callback.py @@ -6,7 +6,7 @@ from core.workflow.entities.node_entities import NodeType -class BaseWorkflowCallback(ABC): +class WorkflowCallback(ABC): @abstractmethod def on_workflow_run_started(self) -> None: """ @@ -78,7 +78,7 @@ def on_workflow_iteration_started(self, node_type: NodeType, node_run_index: int = 1, node_data: Optional[BaseNodeData] = None, - inputs: dict = None, + inputs: Optional[dict] = None, predecessor_node_id: Optional[str] = None, metadata: Optional[dict] = None) -> None: """ diff --git a/api/core/workflow/entities/node_entities.py b/api/core/workflow/entities/node_entities.py index ae86463407d5af..996aae94c20e27 100644 --- a/api/core/workflow/entities/node_entities.py +++ b/api/core/workflow/entities/node_entities.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from enum import Enum from typing import Any, Optional @@ -82,9 +83,9 @@ class NodeRunResult(BaseModel): """ status: WorkflowNodeExecutionStatus = WorkflowNodeExecutionStatus.RUNNING - inputs: Optional[dict] = None # node inputs + inputs: Optional[Mapping[str, Any]] = None # node inputs process_data: Optional[dict] = None # process data - outputs: Optional[dict] = None # node outputs + outputs: Optional[Mapping[str, Any]] = None # node outputs metadata: Optional[dict[NodeRunMetadataKey, Any]] = None # node metadata edge_source_handle: Optional[str] = None # source handle id of node with multiple branches diff --git a/api/core/workflow/entities/variable_pool.py b/api/core/workflow/entities/variable_pool.py index c04770616c321c..23076d5ca46b84 100644 --- a/api/core/workflow/entities/variable_pool.py +++ b/api/core/workflow/entities/variable_pool.py @@ -1,101 +1,141 @@ -from enum import Enum -from typing import Any, Optional, Union +from collections import defaultdict +from collections.abc import Mapping, Sequence +from typing import Any, Union +from typing_extensions import deprecated + +from core.app.segments import ArrayVariable, ObjectVariable, Variable, factory from core.file.file_obj import FileVar from core.workflow.entities.node_entities import SystemVariable VariableValue = Union[str, int, float, dict, list, FileVar] -class ValueType(Enum): - """ - Value Type Enum - """ - STRING = "string" - NUMBER = "number" - OBJECT = "object" - ARRAY_STRING = "array[string]" - ARRAY_NUMBER = "array[number]" - ARRAY_OBJECT = "array[object]" - ARRAY_FILE = "array[file]" - FILE = "file" +SYSTEM_VARIABLE_NODE_ID = 'sys' +ENVIRONMENT_VARIABLE_NODE_ID = 'env' class VariablePool: - - def __init__(self, system_variables: dict[SystemVariable, Any], - user_inputs: dict) -> None: + def __init__( + self, + system_variables: Mapping[SystemVariable, Any], + user_inputs: Mapping[str, Any], + environment_variables: Sequence[Variable], + ) -> None: # system variables # for example: # { # 'query': 'abc', # 'files': [] # } - self.variables_mapping = {} + + # Varaible dictionary is a dictionary for looking up variables by their selector. + # The first element of the selector is the node id, it's the first-level key in the dictionary. + # Other elements of the selector are the keys in the second-level dictionary. To get the key, we hash the + # elements of the selector except the first one. + self._variable_dictionary: dict[str, dict[int, Variable]] = defaultdict(dict) + + # TODO: This user inputs is not used for pool. self.user_inputs = user_inputs + + # Add system variables to the variable pool self.system_variables = system_variables - for system_variable, value in system_variables.items(): - self.append_variable('sys', [system_variable.value], value) + for key, value in system_variables.items(): + self.add((SYSTEM_VARIABLE_NODE_ID, key.value), value) - def append_variable(self, node_id: str, variable_key_list: list[str], value: VariableValue) -> None: + # Add environment variables to the variable pool + for var in environment_variables or []: + self.add((ENVIRONMENT_VARIABLE_NODE_ID, var.name), var) + + def add(self, selector: Sequence[str], value: Any, /) -> None: """ - Append variable - :param node_id: node id - :param variable_key_list: variable key list, like: ['result', 'text'] - :param value: value - :return: + Adds a variable to the variable pool. + + Args: + selector (Sequence[str]): The selector for the variable. + value (VariableValue): The value of the variable. + + Raises: + ValueError: If the selector is invalid. + + Returns: + None """ - if node_id not in self.variables_mapping: - self.variables_mapping[node_id] = {} + if len(selector) < 2: + raise ValueError('Invalid selector') + + if value is None: + return - variable_key_list_hash = hash(tuple(variable_key_list)) + if not isinstance(value, Variable): + v = factory.build_anonymous_variable(value) + else: + v = value - self.variables_mapping[node_id][variable_key_list_hash] = value + hash_key = hash(tuple(selector[1:])) + self._variable_dictionary[selector[0]][hash_key] = v - def get_variable_value(self, variable_selector: list[str], - target_value_type: Optional[ValueType] = None) -> Optional[VariableValue]: + def get(self, selector: Sequence[str], /) -> Variable | None: """ - Get variable - :param variable_selector: include node_id and variables - :param target_value_type: target value type - :return: + Retrieves the value from the variable pool based on the given selector. + + Args: + selector (Sequence[str]): The selector used to identify the variable. + + Returns: + Any: The value associated with the given selector. + + Raises: + ValueError: If the selector is invalid. """ - if len(variable_selector) < 2: - raise ValueError('Invalid value selector') - - node_id = variable_selector[0] - if node_id not in self.variables_mapping: - return None - - # fetch variable keys, pop node_id - variable_key_list = variable_selector[1:] - - variable_key_list_hash = hash(tuple(variable_key_list)) - - value = self.variables_mapping[node_id].get(variable_key_list_hash) - - if target_value_type: - if target_value_type == ValueType.STRING: - return str(value) - elif target_value_type == ValueType.NUMBER: - return int(value) - elif target_value_type == ValueType.OBJECT: - if not isinstance(value, dict): - raise ValueError('Invalid value type: object') - elif target_value_type in [ValueType.ARRAY_STRING, - ValueType.ARRAY_NUMBER, - ValueType.ARRAY_OBJECT, - ValueType.ARRAY_FILE]: - if not isinstance(value, list): - raise ValueError(f'Invalid value type: {target_value_type.value}') + if len(selector) < 2: + raise ValueError('Invalid selector') + hash_key = hash(tuple(selector[1:])) + value = self._variable_dictionary[selector[0]].get(hash_key) return value - def clear_node_variables(self, node_id: str) -> None: + @deprecated('This method is deprecated, use `get` instead.') + def get_any(self, selector: Sequence[str], /) -> Any | None: + """ + Retrieves the value from the variable pool based on the given selector. + + Args: + selector (Sequence[str]): The selector used to identify the variable. + + Returns: + Any: The value associated with the given selector. + + Raises: + ValueError: If the selector is invalid. + """ + if len(selector) < 2: + raise ValueError('Invalid selector') + hash_key = hash(tuple(selector[1:])) + value = self._variable_dictionary[selector[0]].get(hash_key) + + if value is None: + return value + if isinstance(value, ArrayVariable): + return [element.value for element in value.value] + if isinstance(value, ObjectVariable): + return {k: v.value for k, v in value.value.items()} + return value.value if value else None + + def remove(self, selector: Sequence[str], /): """ - Clear node variables - :param node_id: node id - :return: + Remove variables from the variable pool based on the given selector. + + Args: + selector (Sequence[str]): A sequence of strings representing the selector. + + Returns: + None """ - if node_id in self.variables_mapping: - self.variables_mapping.pop(node_id) \ No newline at end of file + if not selector: + return + if len(selector) == 1: + self._variable_dictionary[selector[0]] = {} + return + hash_key = hash(tuple(selector[1:])) + self._variable_dictionary[selector[0]].pop(hash_key, None) diff --git a/api/core/workflow/nodes/answer/answer_node.py b/api/core/workflow/nodes/answer/answer_node.py index e8f1678ecb234a..5bae27092f920d 100644 --- a/api/core/workflow/nodes/answer/answer_node.py +++ b/api/core/workflow/nodes/answer/answer_node.py @@ -1,7 +1,5 @@ -import json from typing import cast -from core.file.file_obj import FileVar from core.prompt.utils.prompt_template_parser import PromptTemplateParser from core.workflow.entities.base_node_data_entities import BaseNodeData from core.workflow.entities.node_entities import NodeRunResult, NodeType @@ -19,7 +17,7 @@ class AnswerNode(BaseNode): _node_data_cls = AnswerNodeData - node_type = NodeType.ANSWER + _node_type: NodeType = NodeType.ANSWER def _run(self, variable_pool: VariablePool) -> NodeRunResult: """ @@ -28,7 +26,7 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: :return: """ node_data = self.node_data - node_data = cast(self._node_data_cls, node_data) + node_data = cast(AnswerNodeData, node_data) # generate routes generate_routes = self.extract_generate_route_from_node_data(node_data) @@ -38,31 +36,9 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: if part.type == "var": part = cast(VarGenerateRouteChunk, part) value_selector = part.value_selector - value = variable_pool.get_variable_value( - variable_selector=value_selector - ) - - text = '' - if isinstance(value, str | int | float): - text = str(value) - elif isinstance(value, dict): - # other types - text = json.dumps(value, ensure_ascii=False) - elif isinstance(value, FileVar): - # convert file to markdown - text = value.to_markdown() - elif isinstance(value, list): - for item in value: - if isinstance(item, FileVar): - text += item.to_markdown() + ' ' - - text = text.strip() - - if not text and value: - # other types - text = json.dumps(value, ensure_ascii=False) - - answer += text + value = variable_pool.get(value_selector) + if value: + answer += value.markdown else: part = cast(TextGenerateRouteChunk, part) answer += part.text @@ -82,7 +58,7 @@ def extract_generate_route_selectors(cls, config: dict) -> list[GenerateRouteChu :return: """ node_data = cls._node_data_cls(**config.get("data", {})) - node_data = cast(cls._node_data_cls, node_data) + node_data = cast(AnswerNodeData, node_data) return cls.extract_generate_route_from_node_data(node_data) @@ -143,7 +119,7 @@ def _extract_variable_selector_to_variable_mapping(cls, node_data: BaseNodeData) :return: """ node_data = node_data - node_data = cast(cls._node_data_cls, node_data) + node_data = cast(AnswerNodeData, node_data) variable_template_parser = VariableTemplateParser(template=node_data.answer) variable_selectors = variable_template_parser.extract_variable_selectors() diff --git a/api/core/workflow/nodes/base_node.py b/api/core/workflow/nodes/base_node.py index fa7d6424f1cbb9..b83a0ae19b3cdd 100644 --- a/api/core/workflow/nodes/base_node.py +++ b/api/core/workflow/nodes/base_node.py @@ -1,9 +1,10 @@ from abc import ABC, abstractmethod +from collections.abc import Mapping, Sequence from enum import Enum -from typing import Optional +from typing import Any, Optional from core.app.entities.app_invoke_entities import InvokeFrom -from core.workflow.callbacks.base_workflow_callback import BaseWorkflowCallback +from core.workflow.callbacks.base_workflow_callback import WorkflowCallback from core.workflow.entities.base_node_data_entities import BaseIterationState, BaseNodeData from core.workflow.entities.node_entities import NodeRunResult, NodeType from core.workflow.entities.variable_pool import VariablePool @@ -46,7 +47,7 @@ class BaseNode(ABC): node_data: BaseNodeData node_run_result: Optional[NodeRunResult] = None - callbacks: list[BaseWorkflowCallback] + callbacks: Sequence[WorkflowCallback] def __init__(self, tenant_id: str, app_id: str, @@ -54,8 +55,8 @@ def __init__(self, tenant_id: str, user_id: str, user_from: UserFrom, invoke_from: InvokeFrom, - config: dict, - callbacks: list[BaseWorkflowCallback] = None, + config: Mapping[str, Any], + callbacks: Sequence[WorkflowCallback] | None = None, workflow_call_depth: int = 0) -> None: self.tenant_id = tenant_id self.app_id = app_id @@ -65,7 +66,8 @@ def __init__(self, tenant_id: str, self.invoke_from = invoke_from self.workflow_call_depth = workflow_call_depth - self.node_id = config.get("id") + # TODO: May need to check if key exists. + self.node_id = config["id"] if not self.node_id: raise ValueError("Node ID is required.") diff --git a/api/core/workflow/nodes/code/code_node.py b/api/core/workflow/nodes/code/code_node.py index e15c1c6f8750cd..b2c362b983cebd 100644 --- a/api/core/workflow/nodes/code/code_node.py +++ b/api/core/workflow/nodes/code/code_node.py @@ -59,11 +59,8 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: variables = {} for variable_selector in node_data.variables: variable = variable_selector.variable - value = variable_pool.get_variable_value( - variable_selector=variable_selector.value_selector - ) - - variables[variable] = value + value = variable_pool.get(variable_selector.value_selector) + variables[variable] = value.value if value else None # Run code try: result = CodeExecutor.execute_workflow_code_template( diff --git a/api/core/workflow/nodes/end/end_node.py b/api/core/workflow/nodes/end/end_node.py index 08d55d55762b41..17488f72531d6c 100644 --- a/api/core/workflow/nodes/end/end_node.py +++ b/api/core/workflow/nodes/end/end_node.py @@ -10,7 +10,7 @@ class EndNode(BaseNode): _node_data_cls = EndNodeData - node_type = NodeType.END + _node_type = NodeType.END def _run(self, variable_pool: VariablePool) -> NodeRunResult: """ @@ -19,16 +19,13 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: :return: """ node_data = self.node_data - node_data = cast(self._node_data_cls, node_data) + node_data = cast(EndNodeData, node_data) output_variables = node_data.outputs outputs = {} for variable_selector in output_variables: - value = variable_pool.get_variable_value( - variable_selector=variable_selector.value_selector - ) - - outputs[variable_selector.variable] = value + value = variable_pool.get(variable_selector.value_selector) + outputs[variable_selector.variable] = value.value if value else None return NodeRunResult( status=WorkflowNodeExecutionStatus.SUCCEEDED, @@ -45,7 +42,7 @@ def extract_generate_nodes(cls, graph: dict, config: dict) -> list[str]: :return: """ node_data = cls._node_data_cls(**config.get("data", {})) - node_data = cast(cls._node_data_cls, node_data) + node_data = cast(EndNodeData, node_data) return cls.extract_generate_nodes_from_node_data(graph, node_data) @@ -57,7 +54,7 @@ def extract_generate_nodes_from_node_data(cls, graph: dict, node_data: EndNodeDa :param node_data: node data object :return: """ - nodes = graph.get('nodes') + nodes = graph.get('nodes', []) node_mapping = {node.get('id'): node for node in nodes} variable_selectors = node_data.outputs diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index fc32ee7a0cc025..473d85f073d9bd 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -9,7 +9,7 @@ import core.helper.ssrf_proxy as ssrf_proxy from configs import dify_config from core.workflow.entities.variable_entities import VariableSelector -from core.workflow.entities.variable_pool import ValueType, VariablePool +from core.workflow.entities.variable_pool import VariablePool from core.workflow.nodes.http_request.entities import ( HttpRequestNodeAuthorization, HttpRequestNodeBody, @@ -212,13 +212,11 @@ def _assembling_headers(self) -> dict[str, Any]: raise ValueError('self.authorization config is required') if authorization.config is None: raise ValueError('authorization config is required') - if authorization.config.type != 'bearer' and authorization.config.header is None: - raise ValueError('authorization config header is required') if self.authorization.config.api_key is None: raise ValueError('api_key is required') - if not self.authorization.config.header: + if not authorization.config.header: authorization.config.header = 'Authorization' if self.authorization.config.type == 'bearer': @@ -335,16 +333,13 @@ def _format_template( if variable_pool: variable_value_mapping = {} for variable_selector in variable_selectors: - value = variable_pool.get_variable_value( - variable_selector=variable_selector.value_selector, target_value_type=ValueType.STRING - ) - - if value is None: + variable = variable_pool.get(variable_selector.value_selector) + if variable is None: raise ValueError(f'Variable {variable_selector.variable} not found') - - if escape_quotes and isinstance(value, str): - value = value.replace('"', '\\"') - + if escape_quotes and isinstance(variable.value, str): + value = variable.value.replace('"', '\\"') + else: + value = variable.value variable_value_mapping[variable_selector.variable] = value return variable_template_parser.format(variable_value_mapping), variable_selectors diff --git a/api/core/workflow/nodes/http_request/http_request_node.py b/api/core/workflow/nodes/http_request/http_request_node.py index 86b17c9b33fbeb..bbe5f9ad43f561 100644 --- a/api/core/workflow/nodes/http_request/http_request_node.py +++ b/api/core/workflow/nodes/http_request/http_request_node.py @@ -3,6 +3,7 @@ from os import path from typing import cast +from core.app.segments import parser from core.file.file_obj import FileTransferMethod, FileType, FileVar from core.tools.tool_file_manager import ToolFileManager from core.workflow.entities.base_node_data_entities import BaseNodeData @@ -51,6 +52,9 @@ def get_default_config(cls, filters: dict | None = None) -> dict: def _run(self, variable_pool: VariablePool) -> NodeRunResult: node_data: HttpRequestNodeData = cast(HttpRequestNodeData, self.node_data) + # TODO: Switch to use segment directly + if node_data.authorization.config and node_data.authorization.config.api_key: + node_data.authorization.config.api_key = parser.convert_template(template=node_data.authorization.config.api_key, variable_pool=variable_pool).text # init http executor http_executor = None diff --git a/api/core/workflow/nodes/if_else/if_else_node.py b/api/core/workflow/nodes/if_else/if_else_node.py index 6176a752014141..c6d235627f04b0 100644 --- a/api/core/workflow/nodes/if_else/if_else_node.py +++ b/api/core/workflow/nodes/if_else/if_else_node.py @@ -1,3 +1,4 @@ +from collections.abc import Sequence from typing import Optional, cast from core.workflow.entities.base_node_data_entities import BaseNodeData @@ -11,7 +12,7 @@ class IfElseNode(BaseNode): _node_data_cls = IfElseNodeData - node_type = NodeType.IF_ELSE + _node_type = NodeType.IF_ELSE def _run(self, variable_pool: VariablePool) -> NodeRunResult: """ @@ -20,7 +21,7 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: :return: """ node_data = self.node_data - node_data = cast(self._node_data_cls, node_data) + node_data = cast(IfElseNodeData, node_data) node_inputs = { "conditions": [] @@ -138,14 +139,12 @@ def evaluate_condition( else: raise ValueError(f"Invalid comparison operator: {comparison_operator}") - def process_conditions(self, variable_pool: VariablePool, conditions: list[Condition]): + def process_conditions(self, variable_pool: VariablePool, conditions: Sequence[Condition]): input_conditions = [] group_result = [] for condition in conditions: - actual_value = variable_pool.get_variable_value( - variable_selector=condition.variable_selector - ) + actual_variable = variable_pool.get_any(condition.variable_selector) if condition.value is not None: variable_template_parser = VariableTemplateParser(template=condition.value) @@ -153,9 +152,7 @@ def process_conditions(self, variable_pool: VariablePool, conditions: list[Condi variable_selectors = variable_template_parser.extract_variable_selectors() if variable_selectors: for variable_selector in variable_selectors: - value = variable_pool.get_variable_value( - variable_selector=variable_selector.value_selector - ) + value = variable_pool.get_any(variable_selector.value_selector) expected_value = variable_template_parser.format({variable_selector.variable: value}) else: expected_value = condition.value @@ -165,13 +162,13 @@ def process_conditions(self, variable_pool: VariablePool, conditions: list[Condi comparison_operator = condition.comparison_operator input_conditions.append( { - "actual_value": actual_value, + "actual_value": actual_variable, "expected_value": expected_value, "comparison_operator": comparison_operator } ) - result = self.evaluate_condition(actual_value, expected_value, comparison_operator) + result = self.evaluate_condition(actual_variable, expected_value, comparison_operator) group_result.append(result) return input_conditions, group_result diff --git a/api/core/workflow/nodes/iteration/iteration_node.py b/api/core/workflow/nodes/iteration/iteration_node.py index 12d792f297dbd6..fcd853a4bda33a 100644 --- a/api/core/workflow/nodes/iteration/iteration_node.py +++ b/api/core/workflow/nodes/iteration/iteration_node.py @@ -20,7 +20,8 @@ def _run(self, variable_pool: VariablePool) -> BaseIterationState: """ Run the node. """ - iterator = variable_pool.get_variable_value(cast(IterationNodeData, self.node_data).iterator_selector) + self.node_data = cast(IterationNodeData, self.node_data) + iterator = variable_pool.get_any(self.node_data.iterator_selector) if not isinstance(iterator, list): raise ValueError(f"Invalid iterator value: {iterator}, please provide a list.") @@ -63,15 +64,15 @@ def _set_current_iteration_variable(self, variable_pool: VariablePool, state: It """ node_data = cast(IterationNodeData, self.node_data) - variable_pool.append_variable(self.node_id, ['index'], state.index) + variable_pool.add((self.node_id, 'index'), state.index) # get the iterator value - iterator = variable_pool.get_variable_value(node_data.iterator_selector) + iterator = variable_pool.get_any(node_data.iterator_selector) if iterator is None or not isinstance(iterator, list): return if state.index < len(iterator): - variable_pool.append_variable(self.node_id, ['item'], iterator[state.index]) + variable_pool.add((self.node_id, 'item'), iterator[state.index]) def _next_iteration(self, variable_pool: VariablePool, state: IterationState): """ @@ -87,7 +88,7 @@ def _reached_iteration_limit(self, variable_pool: VariablePool, state: Iteration :return: True if iteration limit is reached, False otherwise """ node_data = cast(IterationNodeData, self.node_data) - iterator = variable_pool.get_variable_value(node_data.iterator_selector) + iterator = variable_pool.get_any(node_data.iterator_selector) if iterator is None or not isinstance(iterator, list): return True @@ -100,9 +101,9 @@ def _resolve_current_output(self, variable_pool: VariablePool, state: IterationS :param variable_pool: variable pool """ output_selector = cast(IterationNodeData, self.node_data).output_selector - output = variable_pool.get_variable_value(output_selector) + output = variable_pool.get_any(output_selector) # clear the output for this iteration - variable_pool.append_variable(self.node_id, output_selector[1:], None) + variable_pool.remove([self.node_id] + output_selector[1:]) state.current_output = output if output is not None: state.outputs.append(output) diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index 12fe4dfa84b1a6..6e1534b3c2c93e 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -41,7 +41,8 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: node_data: KnowledgeRetrievalNodeData = cast(self._node_data_cls, self.node_data) # extract variables - query = variable_pool.get_variable_value(variable_selector=node_data.query_variable_selector) + variable = variable_pool.get(node_data.query_variable_selector) + query = variable.value if variable else None variables = { 'query': query } diff --git a/api/core/workflow/nodes/llm/llm_node.py b/api/core/workflow/nodes/llm/llm_node.py index af928517d942d1..4431259a57543b 100644 --- a/api/core/workflow/nodes/llm/llm_node.py +++ b/api/core/workflow/nodes/llm/llm_node.py @@ -41,7 +41,7 @@ class LLMNode(BaseNode): _node_data_cls = LLMNodeData - node_type = NodeType.LLM + _node_type = NodeType.LLM def _run(self, variable_pool: VariablePool) -> NodeRunResult: """ @@ -90,7 +90,7 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: # fetch prompt messages prompt_messages, stop = self._fetch_prompt_messages( node_data=node_data, - query=variable_pool.get_variable_value(['sys', SystemVariable.QUERY.value]) + query=variable_pool.get_any(['sys', SystemVariable.QUERY.value]) if node_data.memory else None, query_prompt_template=node_data.memory.query_prompt_template if node_data.memory else None, inputs=inputs, @@ -238,8 +238,8 @@ def _fetch_jinja_inputs(self, node_data: LLMNodeData, variable_pool: VariablePoo for variable_selector in node_data.prompt_config.jinja2_variables or []: variable = variable_selector.variable - value = variable_pool.get_variable_value( - variable_selector=variable_selector.value_selector + value = variable_pool.get_any( + variable_selector.value_selector ) def parse_dict(d: dict) -> str: @@ -302,7 +302,7 @@ def _fetch_inputs(self, node_data: LLMNodeData, variable_pool: VariablePool) -> variable_selectors = variable_template_parser.extract_variable_selectors() for variable_selector in variable_selectors: - variable_value = variable_pool.get_variable_value(variable_selector.value_selector) + variable_value = variable_pool.get_any(variable_selector.value_selector) if variable_value is None: raise ValueError(f'Variable {variable_selector.variable} not found') @@ -313,7 +313,7 @@ def _fetch_inputs(self, node_data: LLMNodeData, variable_pool: VariablePool) -> query_variable_selectors = (VariableTemplateParser(template=memory.query_prompt_template) .extract_variable_selectors()) for variable_selector in query_variable_selectors: - variable_value = variable_pool.get_variable_value(variable_selector.value_selector) + variable_value = variable_pool.get_any(variable_selector.value_selector) if variable_value is None: raise ValueError(f'Variable {variable_selector.variable} not found') @@ -331,7 +331,7 @@ def _fetch_files(self, node_data: LLMNodeData, variable_pool: VariablePool) -> l if not node_data.vision.enabled: return [] - files = variable_pool.get_variable_value(['sys', SystemVariable.FILES.value]) + files = variable_pool.get_any(['sys', SystemVariable.FILES.value]) if not files: return [] @@ -350,7 +350,7 @@ def _fetch_context(self, node_data: LLMNodeData, variable_pool: VariablePool) -> if not node_data.context.variable_selector: return None - context_value = variable_pool.get_variable_value(node_data.context.variable_selector) + context_value = variable_pool.get_any(node_data.context.variable_selector) if context_value: if isinstance(context_value, str): return context_value @@ -496,7 +496,7 @@ def _fetch_memory(self, node_data_memory: Optional[MemoryConfig], return None # get conversation id - conversation_id = variable_pool.get_variable_value(['sys', SystemVariable.CONVERSATION_ID.value]) + conversation_id = variable_pool.get_any(['sys', SystemVariable.CONVERSATION_ID.value]) if conversation_id is None: return None diff --git a/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py b/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py index d219156026e1de..1fe6896e891160 100644 --- a/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py +++ b/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py @@ -71,9 +71,10 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: Run the node. """ node_data = cast(ParameterExtractorNodeData, self.node_data) - query = variable_pool.get_variable_value(node_data.query) - if not query: + variable = variable_pool.get(node_data.query) + if not variable: raise ValueError("Input variable content not found or is empty") + query = variable.value inputs = { 'query': query, @@ -564,7 +565,8 @@ def _render_instruction(self, instruction: str, variable_pool: VariablePool) -> variable_template_parser = VariableTemplateParser(instruction) inputs = {} for selector in variable_template_parser.extract_variable_selectors(): - inputs[selector.variable] = variable_pool.get_variable_value(selector.value_selector) + variable = variable_pool.get(selector.value_selector) + inputs[selector.variable] = variable.value if variable else None return variable_template_parser.format(inputs) diff --git a/api/core/workflow/nodes/question_classifier/question_classifier_node.py b/api/core/workflow/nodes/question_classifier/question_classifier_node.py index 76f3dec8366801..2e1464efcef1e6 100644 --- a/api/core/workflow/nodes/question_classifier/question_classifier_node.py +++ b/api/core/workflow/nodes/question_classifier/question_classifier_node.py @@ -41,7 +41,8 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: node_data = cast(QuestionClassifierNodeData, node_data) # extract variables - query = variable_pool.get_variable_value(variable_selector=node_data.query_variable_selector) + variable = variable_pool.get(node_data.query_variable_selector) + query = variable.value if variable else None variables = { 'query': query } @@ -294,7 +295,8 @@ def _format_instruction(self, instruction: str, variable_pool: VariablePool) -> variable_template_parser = VariableTemplateParser(template=instruction) variable_selectors.extend(variable_template_parser.extract_variable_selectors()) for variable_selector in variable_selectors: - variable_value = variable_pool.get_variable_value(variable_selector.value_selector) + variable = variable_pool.get(variable_selector.value_selector) + variable_value = variable.value if variable else None if variable_value is None: raise ValueError(f'Variable {variable_selector.variable} not found') diff --git a/api/core/workflow/nodes/start/start_node.py b/api/core/workflow/nodes/start/start_node.py index fd51a6c476c482..661b403d32f8ca 100644 --- a/api/core/workflow/nodes/start/start_node.py +++ b/api/core/workflow/nodes/start/start_node.py @@ -9,7 +9,7 @@ class StartNode(BaseNode): _node_data_cls = StartNodeData - node_type = NodeType.START + _node_type = NodeType.START def _run(self, variable_pool: VariablePool) -> NodeRunResult: """ @@ -18,7 +18,7 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: :return: """ # Get cleaned inputs - cleaned_inputs = variable_pool.user_inputs + cleaned_inputs = dict(variable_pool.user_inputs) for var in variable_pool.system_variables: cleaned_inputs['sys.' + var.value] = variable_pool.system_variables[var] diff --git a/api/core/workflow/nodes/template_transform/template_transform_node.py b/api/core/workflow/nodes/template_transform/template_transform_node.py index 2c4a2257f599f3..21f71db6c549aa 100644 --- a/api/core/workflow/nodes/template_transform/template_transform_node.py +++ b/api/core/workflow/nodes/template_transform/template_transform_node.py @@ -44,12 +44,9 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: # Get variables variables = {} for variable_selector in node_data.variables: - variable = variable_selector.variable - value = variable_pool.get_variable_value( - variable_selector=variable_selector.value_selector - ) - - variables[variable] = value + variable_name = variable_selector.variable + value = variable_pool.get_any(variable_selector.value_selector) + variables[variable_name] = value # Run code try: result = CodeExecutor.execute_workflow_code_template( diff --git a/api/core/workflow/nodes/tool/entities.py b/api/core/workflow/nodes/tool/entities.py index 2e4743c483229f..5da5cd07271bae 100644 --- a/api/core/workflow/nodes/tool/entities.py +++ b/api/core/workflow/nodes/tool/entities.py @@ -29,6 +29,7 @@ def validate_tool_configurations(cls, value, values: ValidationInfo): class ToolNodeData(BaseNodeData, ToolEntity): class ToolInput(BaseModel): + # TODO: check this type value: Union[Any, list[str]] type: Literal['mixed', 'variable', 'constant'] diff --git a/api/core/workflow/nodes/tool/tool_node.py b/api/core/workflow/nodes/tool/tool_node.py index cddea03bf862ad..1bd126f8422ff4 100644 --- a/api/core/workflow/nodes/tool/tool_node.py +++ b/api/core/workflow/nodes/tool/tool_node.py @@ -1,10 +1,11 @@ +from collections.abc import Mapping, Sequence from os import path -from typing import Optional, cast +from typing import Any, cast +from core.app.segments import parser from core.callback_handler.workflow_tool_callback_handler import DifyWorkflowCallbackHandler from core.file.file_obj import FileTransferMethod, FileType, FileVar from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter -from core.tools.tool.tool import Tool from core.tools.tool_engine import ToolEngine from core.tools.tool_manager import ToolManager from core.tools.utils.message_transformer import ToolFileMessageTransformer @@ -20,6 +21,7 @@ class ToolNode(BaseNode): """ Tool Node """ + _node_data_cls = ToolNodeData _node_type = NodeType.TOOL @@ -50,23 +52,24 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: }, error=f'Failed to get tool runtime: {str(e)}' ) - + # get parameters - parameters = self._generate_parameters(variable_pool, node_data, tool_runtime) + tool_parameters = tool_runtime.get_runtime_parameters() or [] + parameters = self._generate_parameters(tool_parameters=tool_parameters, variable_pool=variable_pool, node_data=node_data) + parameters_for_log = self._generate_parameters(tool_parameters=tool_parameters, variable_pool=variable_pool, node_data=node_data, for_log=True) try: messages = ToolEngine.workflow_invoke( tool=tool_runtime, tool_parameters=parameters, user_id=self.user_id, - workflow_id=self.workflow_id, workflow_tool_callback=DifyWorkflowCallbackHandler(), workflow_call_depth=self.workflow_call_depth, ) except Exception as e: return NodeRunResult( status=WorkflowNodeExecutionStatus.FAILED, - inputs=parameters, + inputs=parameters_for_log, metadata={ NodeRunMetadataKey.TOOL_INFO: tool_info }, @@ -86,21 +89,34 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: metadata={ NodeRunMetadataKey.TOOL_INFO: tool_info }, - inputs=parameters + inputs=parameters_for_log ) - def _generate_parameters(self, variable_pool: VariablePool, node_data: ToolNodeData, tool_runtime: Tool) -> dict: - """ - Generate parameters + def _generate_parameters( + self, + *, + tool_parameters: Sequence[ToolParameter], + variable_pool: VariablePool, + node_data: ToolNodeData, + for_log: bool = False, + ) -> Mapping[str, Any]: """ - tool_parameters = tool_runtime.get_all_runtime_parameters() + Generate parameters based on the given tool parameters, variable pool, and node data. - def fetch_parameter(name: str) -> Optional[ToolParameter]: - return next((parameter for parameter in tool_parameters if parameter.name == name), None) + Args: + tool_parameters (Sequence[ToolParameter]): The list of tool parameters. + variable_pool (VariablePool): The variable pool containing the variables. + node_data (ToolNodeData): The data associated with the tool node. + + Returns: + Mapping[str, Any]: A dictionary containing the generated parameters. + + """ + tool_parameters_dictionary = {parameter.name: parameter for parameter in tool_parameters} result = {} for parameter_name in node_data.tool_parameters: - parameter = fetch_parameter(parameter_name) + parameter = tool_parameters_dictionary.get(parameter_name) if not parameter: continue if parameter.type == ToolParameter.ToolParameterType.FILE: @@ -108,35 +124,21 @@ def fetch_parameter(name: str) -> Optional[ToolParameter]: v.to_dict() for v in self._fetch_files(variable_pool) ] else: - input = node_data.tool_parameters[parameter_name] - if input.type == 'mixed': - result[parameter_name] = self._format_variable_template(input.value, variable_pool) - elif input.type == 'variable': - result[parameter_name] = variable_pool.get_variable_value(input.value) - elif input.type == 'constant': - result[parameter_name] = input.value + tool_input = node_data.tool_parameters[parameter_name] + segment_group = parser.convert_template( + template=str(tool_input.value), + variable_pool=variable_pool, + ) + result[parameter_name] = segment_group.log if for_log else segment_group.text return result - - def _format_variable_template(self, template: str, variable_pool: VariablePool) -> str: - """ - Format variable template - """ - inputs = {} - template_parser = VariableTemplateParser(template) - for selector in template_parser.extract_variable_selectors(): - inputs[selector.variable] = variable_pool.get_variable_value(selector.value_selector) - - return template_parser.format(inputs) - + def _fetch_files(self, variable_pool: VariablePool) -> list[FileVar]: - files = variable_pool.get_variable_value(['sys', SystemVariable.FILES.value]) - if not files: - return [] - - return files + # FIXME: ensure this is a ArrayVariable contains FileVariable. + variable = variable_pool.get(['sys', SystemVariable.FILES.value]) + return [file_var.value for file_var in variable.value] if variable else [] - def _convert_tool_messages(self, messages: list[ToolInvokeMessage]) -> tuple[str, list[FileVar]]: + def _convert_tool_messages(self, messages: list[ToolInvokeMessage]): """ Convert ToolInvokeMessages into tuple[plain_text, files] """ diff --git a/api/core/workflow/nodes/variable_aggregator/variable_aggregator_node.py b/api/core/workflow/nodes/variable_aggregator/variable_aggregator_node.py index 63ce790625d131..1e4947606688f7 100644 --- a/api/core/workflow/nodes/variable_aggregator/variable_aggregator_node.py +++ b/api/core/workflow/nodes/variable_aggregator/variable_aggregator_node.py @@ -19,28 +19,27 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: inputs = {} if not node_data.advanced_settings or not node_data.advanced_settings.group_enabled: - for variable in node_data.variables: - value = variable_pool.get_variable_value(variable) - - if value is not None: + for selector in node_data.variables: + variable = variable_pool.get(selector) + if variable is not None: outputs = { - "output": value + "output": variable.value } inputs = { - '.'.join(variable[1:]): value + '.'.join(selector[1:]): variable.value } break else: for group in node_data.advanced_settings.groups: - for variable in group.variables: - value = variable_pool.get_variable_value(variable) + for selector in group.variables: + variable = variable_pool.get(selector) - if value is not None: + if variable is not None: outputs[group.group_name] = { - 'output': value + 'output': variable.value } - inputs['.'.join(variable[1:])] = value + inputs['.'.join(selector[1:])] = variable.value break return NodeRunResult( diff --git a/api/core/workflow/utils/variable_template_parser.py b/api/core/workflow/utils/variable_template_parser.py index 076a370da25386..c43fde172c7b7a 100644 --- a/api/core/workflow/utils/variable_template_parser.py +++ b/api/core/workflow/utils/variable_template_parser.py @@ -3,12 +3,46 @@ from typing import Any from core.workflow.entities.variable_entities import VariableSelector +from core.workflow.entities.variable_pool import VariablePool -REGEX = re.compile(r"\{\{(#[a-zA-Z0-9_]{1,50}(\.[a-zA-Z_][a-zA-Z0-9_]{0,29}){1,10}#)\}\}") +REGEX = re.compile(r'\{\{(#[a-zA-Z0-9_]{1,50}(\.[a-zA-Z_][a-zA-Z0-9_]{0,29}){1,10}#)\}\}') + + +def parse_mixed_template(*, template: str, variable_pool: VariablePool) -> str: + """ + This is an alternative to the VariableTemplateParser class, + offering the same functionality but with better readability and ease of use. + """ + variable_keys = [match[0] for match in re.findall(REGEX, template)] + variable_keys = list(set(variable_keys)) + + # This key_selector is a tuple of (key, selector) where selector is a list of keys + # e.g. ('#node_id.query.name#', ['node_id', 'query', 'name']) + key_selectors = filter( + lambda t: len(t[1]) >= 2, + ((key, selector.replace('#', '').split('.')) for key, selector in zip(variable_keys, variable_keys)), + ) + inputs = {key: variable_pool.get_any(selector) for key, selector in key_selectors} + + def replacer(match): + key = match.group(1) + # return original matched string if key not found + value = inputs.get(key, match.group(0)) + if value is None: + value = '' + value = str(value) + # remove template variables if required + return re.sub(REGEX, r'{\1}', value) + + result = re.sub(REGEX, replacer, template) + result = re.sub(r'<\|.*?\|>', '', result) + return result class VariableTemplateParser: """ + !NOTE: Consider to use the new `segments` module instead of this class. + A class for parsing and manipulating template variables in a string. Rules: @@ -72,14 +106,11 @@ def extract_variable_selectors(self) -> list[VariableSelector]: if len(split_result) < 2: continue - variable_selectors.append(VariableSelector( - variable=variable_key, - value_selector=split_result - )) + variable_selectors.append(VariableSelector(variable=variable_key, value_selector=split_result)) return variable_selectors - def format(self, inputs: Mapping[str, Any], remove_template_variables: bool = True) -> str: + def format(self, inputs: Mapping[str, Any]) -> str: """ Formats the template string by replacing the template variables with their corresponding values. @@ -90,6 +121,7 @@ def format(self, inputs: Mapping[str, Any], remove_template_variables: bool = Tr Returns: The formatted string with template variables replaced by their values. """ + def replacer(match): key = match.group(1) value = inputs.get(key, match.group(0)) # return original matched string if key not found @@ -99,11 +131,9 @@ def replacer(match): # convert the value to string if isinstance(value, list | dict | bool | int | float): value = str(value) - + # remove template variables if required - if remove_template_variables: - return VariableTemplateParser.remove_template_variables(value) - return value + return VariableTemplateParser.remove_template_variables(value) prompt = re.sub(REGEX, replacer, self.template) return re.sub(r'<\|.*?\|>', '', prompt) diff --git a/api/core/workflow/workflow_engine_manager.py b/api/core/workflow/workflow_engine_manager.py index e81bf684a96a14..24e38e97308fb3 100644 --- a/api/core/workflow/workflow_engine_manager.py +++ b/api/core/workflow/workflow_engine_manager.py @@ -1,14 +1,15 @@ import logging import time -from typing import Optional, cast +from collections.abc import Mapping, Sequence +from typing import Any, Optional, cast from configs import dify_config from core.app.app_config.entities import FileExtraConfig from core.app.apps.base_app_queue_manager import GenerateTaskStoppedException from core.app.entities.app_invoke_entities import InvokeFrom from core.file.file_obj import FileTransferMethod, FileType, FileVar -from core.workflow.callbacks.base_workflow_callback import BaseWorkflowCallback -from core.workflow.entities.node_entities import NodeRunMetadataKey, NodeRunResult, NodeType +from core.workflow.callbacks.base_workflow_callback import WorkflowCallback +from core.workflow.entities.node_entities import NodeRunMetadataKey, NodeRunResult, NodeType, SystemVariable from core.workflow.entities.variable_pool import VariablePool, VariableValue from core.workflow.entities.workflow_entities import WorkflowNodeAndResult, WorkflowRunState from core.workflow.errors import WorkflowNodeRunFailedError @@ -35,7 +36,7 @@ WorkflowNodeExecutionStatus, ) -node_classes = { +node_classes: Mapping[NodeType, type[BaseNode]] = { NodeType.START: StartNode, NodeType.END: EndNode, NodeType.ANSWER: AnswerNode, @@ -86,14 +87,14 @@ def get_default_config(self, node_type: NodeType, filters: Optional[dict] = None return default_config - def run_workflow(self, workflow: Workflow, + def run_workflow(self, *, workflow: Workflow, user_id: str, user_from: UserFrom, invoke_from: InvokeFrom, - user_inputs: dict, - system_inputs: Optional[dict] = None, - callbacks: list[BaseWorkflowCallback] = None, - call_depth: Optional[int] = 0, + user_inputs: Mapping[str, Any], + system_inputs: Mapping[SystemVariable, Any], + callbacks: Sequence[WorkflowCallback], + call_depth: int = 0, variable_pool: Optional[VariablePool] = None) -> None: """ :param workflow: Workflow instance @@ -122,7 +123,8 @@ def run_workflow(self, workflow: Workflow, if not variable_pool: variable_pool = VariablePool( system_variables=system_inputs, - user_inputs=user_inputs + user_inputs=user_inputs, + environment_variables=workflow.environment_variables, ) workflow_call_max_depth = dify_config.WORKFLOW_CALL_MAX_DEPTH @@ -154,7 +156,7 @@ def run_workflow(self, workflow: Workflow, def _run_workflow(self, workflow: Workflow, workflow_run_state: WorkflowRunState, - callbacks: list[BaseWorkflowCallback] = None, + callbacks: Sequence[WorkflowCallback], start_at: Optional[str] = None, end_at: Optional[str] = None) -> None: """ @@ -173,8 +175,8 @@ def _run_workflow(self, workflow: Workflow, graph = workflow.graph_dict try: - predecessor_node: BaseNode = None - current_iteration_node: BaseIterationNode = None + predecessor_node: BaseNode | None = None + current_iteration_node: BaseIterationNode | None = None has_entry_node = False max_execution_steps = dify_config.WORKFLOW_MAX_EXECUTION_STEPS max_execution_time = dify_config.WORKFLOW_MAX_EXECUTION_TIME @@ -235,7 +237,7 @@ def _run_workflow(self, workflow: Workflow, # move to next iteration next_node_id = next_iteration # get next id - next_node = self._get_node(workflow_run_state, graph, next_node_id, callbacks) + next_node = self._get_node(workflow_run_state=workflow_run_state, graph=graph, node_id=next_node_id, callbacks=callbacks) if not next_node: break @@ -295,7 +297,7 @@ def _run_workflow(self, workflow: Workflow, workflow_run_state.current_iteration_state = None continue else: - next_node = self._get_node(workflow_run_state, graph, next_node_id, callbacks) + next_node = self._get_node(workflow_run_state=workflow_run_state, graph=graph, node_id=next_node_id, callbacks=callbacks) # run workflow, run multiple target nodes in the future self._run_workflow_node( @@ -381,7 +383,8 @@ def single_step_run_workflow_node(self, workflow: Workflow, # init variable pool variable_pool = VariablePool( system_variables={}, - user_inputs={} + user_inputs={}, + environment_variables=workflow.environment_variables, ) # variable selector to variable mapping @@ -419,7 +422,7 @@ def single_step_run_iteration_workflow_node(self, workflow: Workflow, node_id: str, user_id: str, user_inputs: dict, - callbacks: list[BaseWorkflowCallback] = None, + callbacks: Sequence[WorkflowCallback], ) -> None: """ Single iteration run workflow node @@ -446,7 +449,8 @@ def single_step_run_iteration_workflow_node(self, workflow: Workflow, # init variable pool variable_pool = VariablePool( system_variables={}, - user_inputs={} + user_inputs={}, + environment_variables=workflow.environment_variables, ) # variable selector to variable mapping @@ -535,7 +539,7 @@ def single_step_run_iteration_workflow_node(self, workflow: Workflow, end_at=end_node_id ) - def _workflow_run_success(self, callbacks: list[BaseWorkflowCallback] = None) -> None: + def _workflow_run_success(self, callbacks: Sequence[WorkflowCallback]) -> None: """ Workflow run success :param callbacks: workflow callbacks @@ -547,7 +551,7 @@ def _workflow_run_success(self, callbacks: list[BaseWorkflowCallback] = None) -> callback.on_workflow_run_succeeded() def _workflow_run_failed(self, error: str, - callbacks: list[BaseWorkflowCallback] = None) -> None: + callbacks: Sequence[WorkflowCallback]) -> None: """ Workflow run failed :param error: error message @@ -560,11 +564,11 @@ def _workflow_run_failed(self, error: str, error=error ) - def _workflow_iteration_started(self, graph: dict, + def _workflow_iteration_started(self, *, graph: Mapping[str, Any], current_iteration_node: BaseIterationNode, workflow_run_state: WorkflowRunState, predecessor_node_id: Optional[str] = None, - callbacks: list[BaseWorkflowCallback] = None) -> None: + callbacks: Sequence[WorkflowCallback]) -> None: """ Workflow iteration started :param current_iteration_node: current iteration node @@ -597,10 +601,10 @@ def _workflow_iteration_started(self, graph: dict, # add steps workflow_run_state.workflow_node_steps += 1 - def _workflow_iteration_next(self, graph: dict, + def _workflow_iteration_next(self, *, graph: Mapping[str, Any], current_iteration_node: BaseIterationNode, workflow_run_state: WorkflowRunState, - callbacks: list[BaseWorkflowCallback] = None) -> None: + callbacks: Sequence[WorkflowCallback]) -> None: """ Workflow iteration next :param workflow_run_state: workflow run state @@ -627,11 +631,11 @@ def _workflow_iteration_next(self, graph: dict, nodes = [node for node in nodes if node.get('data', {}).get('iteration_id') == current_iteration_node.node_id] for node in nodes: - workflow_run_state.variable_pool.clear_node_variables(node_id=node.get('id')) + workflow_run_state.variable_pool.remove((node.get('id'),)) - def _workflow_iteration_completed(self, current_iteration_node: BaseIterationNode, + def _workflow_iteration_completed(self, *, current_iteration_node: BaseIterationNode, workflow_run_state: WorkflowRunState, - callbacks: list[BaseWorkflowCallback] = None) -> None: + callbacks: Sequence[WorkflowCallback]) -> None: if callbacks: if isinstance(workflow_run_state.current_iteration_state, IterationState): for callback in callbacks: @@ -644,10 +648,10 @@ def _workflow_iteration_completed(self, current_iteration_node: BaseIterationNod } ) - def _get_next_overall_node(self, workflow_run_state: WorkflowRunState, - graph: dict, + def _get_next_overall_node(self, *, workflow_run_state: WorkflowRunState, + graph: Mapping[str, Any], predecessor_node: Optional[BaseNode] = None, - callbacks: list[BaseWorkflowCallback] = None, + callbacks: Sequence[WorkflowCallback], start_at: Optional[str] = None, end_at: Optional[str] = None) -> Optional[BaseNode]: """ @@ -739,9 +743,9 @@ def _get_next_overall_node(self, workflow_run_state: WorkflowRunState, ) def _get_node(self, workflow_run_state: WorkflowRunState, - graph: dict, + graph: Mapping[str, Any], node_id: str, - callbacks: list[BaseWorkflowCallback]) -> Optional[BaseNode]: + callbacks: Sequence[WorkflowCallback]): """ Get node from graph by node id """ @@ -752,7 +756,7 @@ def _get_node(self, workflow_run_state: WorkflowRunState, for node_config in nodes: if node_config.get('id') == node_id: node_type = NodeType.value_of(node_config.get('data', {}).get('type')) - node_cls = node_classes.get(node_type) + node_cls = node_classes[node_type] return node_cls( tenant_id=workflow_run_state.tenant_id, app_id=workflow_run_state.app_id, @@ -765,8 +769,6 @@ def _get_node(self, workflow_run_state: WorkflowRunState, workflow_call_depth=workflow_run_state.workflow_call_depth ) - return None - def _is_timed_out(self, start_at: float, max_execution_time: int) -> bool: """ Check timeout @@ -785,10 +787,10 @@ def _check_node_has_ran(self, workflow_run_state: WorkflowRunState, node_id: str if node_and_result.node_id == node_id ]) - def _run_workflow_node(self, workflow_run_state: WorkflowRunState, + def _run_workflow_node(self, *, workflow_run_state: WorkflowRunState, node: BaseNode, predecessor_node: Optional[BaseNode] = None, - callbacks: list[BaseWorkflowCallback] = None) -> None: + callbacks: Sequence[WorkflowCallback]) -> None: if callbacks: for callback in callbacks: callback.on_workflow_node_execute_started( @@ -894,10 +896,8 @@ def _append_variables_recursively(self, variable_pool: VariablePool, :param variable_value: variable value :return: """ - variable_pool.append_variable( - node_id=node_id, - variable_key_list=variable_key_list, - value=variable_value + variable_pool.add( + [node_id] + variable_key_list, variable_value ) # if variable_value is a dict, then recursively append variables @@ -946,7 +946,7 @@ def _mapping_user_inputs_to_variable_pool(self, tenant_id: str, node_instance: BaseNode): for variable_key, variable_selector in variable_mapping.items(): - if variable_key not in user_inputs: + if variable_key not in user_inputs and not variable_pool.get(variable_selector): raise ValueError(f'Variable key {variable_key} not found in user inputs.') # fetch variable node id from variable selector @@ -956,7 +956,7 @@ def _mapping_user_inputs_to_variable_pool(self, # get value value = user_inputs.get(variable_key) - # temp fix for image type + # FIXME: temp fix for image type if node_instance.node_type == NodeType.LLM: new_value = [] if isinstance(value, list): @@ -983,8 +983,4 @@ def _mapping_user_inputs_to_variable_pool(self, value = new_value # append variable and value to variable pool - variable_pool.append_variable( - node_id=variable_node_id, - variable_key_list=variable_key_list, - value=value - ) + variable_pool.add([variable_node_id]+variable_key_list, value) diff --git a/api/fields/workflow_fields.py b/api/fields/workflow_fields.py index 54d7ed55f85f9e..c98c332021dde5 100644 --- a/api/fields/workflow_fields.py +++ b/api/fields/workflow_fields.py @@ -1,8 +1,38 @@ from flask_restful import fields +from core.app.segments import SecretVariable, Variable +from core.helper import encrypter from fields.member_fields import simple_account_fields from libs.helper import TimestampField + +class EnvironmentVariableField(fields.Raw): + def format(self, value): + # Mask secret variables values in environment_variables + if isinstance(value, SecretVariable): + return { + 'id': value.id, + 'name': value.name, + 'value': encrypter.obfuscated_token(value.value), + 'value_type': value.value_type.value, + } + elif isinstance(value, Variable): + return { + 'id': value.id, + 'name': value.name, + 'value': value.value, + 'value_type': value.value_type.value, + } + return value + + +environment_variable_fields = { + 'id': fields.String, + 'name': fields.String, + 'value': fields.Raw, + 'value_type': fields.String(attribute='value_type.value'), +} + workflow_fields = { 'id': fields.String, 'graph': fields.Raw(attribute='graph_dict'), @@ -13,4 +43,5 @@ 'updated_by': fields.Nested(simple_account_fields, attribute='updated_by_account', allow_null=True), 'updated_at': TimestampField, 'tool_published': fields.Boolean, + 'environment_variables': fields.List(EnvironmentVariableField()), } diff --git a/api/migrations/versions/8e5588e6412e_add_environment_variable_to_workflow_.py b/api/migrations/versions/8e5588e6412e_add_environment_variable_to_workflow_.py new file mode 100644 index 00000000000000..ec2336da4dec71 --- /dev/null +++ b/api/migrations/versions/8e5588e6412e_add_environment_variable_to_workflow_.py @@ -0,0 +1,33 @@ +"""add environment variable to workflow model + +Revision ID: 8e5588e6412e +Revises: 6e957a32015b +Create Date: 2024-07-22 03:27:16.042533 + +""" +import sqlalchemy as sa +from alembic import op + +import models as models + +# revision identifiers, used by Alembic. +revision = '8e5588e6412e' +down_revision = '6e957a32015b' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('workflows', schema=None) as batch_op: + batch_op.add_column(sa.Column('environment_variables', sa.Text(), server_default='{}', nullable=False)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('workflows', schema=None) as batch_op: + batch_op.drop_column('environment_variables') + + # ### end Alembic commands ### diff --git a/api/models/account.py b/api/models/account.py index 23e7528d22fa67..d36b2b9fda3278 100644 --- a/api/models/account.py +++ b/api/models/account.py @@ -48,7 +48,7 @@ def current_tenant(self): return self._current_tenant @current_tenant.setter - def current_tenant(self, value): + def current_tenant(self, value: "Tenant"): tenant = value ta = TenantAccountJoin.query.filter_by(tenant_id=tenant.id, account_id=self.id).first() if ta: @@ -62,7 +62,7 @@ def current_tenant_id(self): return self._current_tenant.id @current_tenant_id.setter - def current_tenant_id(self, value): + def current_tenant_id(self, value: str): try: tenant_account_join = db.session.query(Tenant, TenantAccountJoin) \ .filter(Tenant.id == value) \ diff --git a/api/models/workflow.py b/api/models/workflow.py index 16e9d88ca15897..df2269cd0fb6cc 100644 --- a/api/models/workflow.py +++ b/api/models/workflow.py @@ -1,7 +1,16 @@ import json +from collections.abc import Mapping, Sequence from enum import Enum -from typing import Optional, Union - +from typing import Any, Optional, Union + +import contexts +from constants import HIDDEN_VALUE +from core.app.segments import ( + SecretVariable, + Variable, + factory, +) +from core.helper import encrypter from extensions.ext_database import db from libs import helper from models import StringUUID @@ -112,6 +121,7 @@ class Workflow(db.Model): created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) updated_by = db.Column(StringUUID) updated_at = db.Column(db.DateTime) + _environment_variables = db.Column('environment_variables', db.Text, nullable=False, server_default='{}') @property def created_by_account(self): @@ -122,11 +132,11 @@ def updated_by_account(self): return db.session.get(Account, self.updated_by) if self.updated_by else None @property - def graph_dict(self): - return json.loads(self.graph) if self.graph else None + def graph_dict(self) -> Mapping[str, Any]: + return json.loads(self.graph) if self.graph else {} @property - def features_dict(self): + def features_dict(self) -> Mapping[str, Any]: return json.loads(self.features) if self.features else {} def user_input_form(self, to_old_structure: bool = False) -> list: @@ -177,6 +187,72 @@ def tool_published(self) -> bool: WorkflowToolProvider.app_id == self.app_id ).first() is not None + @property + def environment_variables(self) -> Sequence[Variable]: + # TODO: find some way to init `self._environment_variables` when instance created. + if self._environment_variables is None: + self._environment_variables = '{}' + + tenant_id = contexts.tenant_id.get() + + environment_variables_dict: dict[str, Any] = json.loads(self._environment_variables) + results = [factory.build_variable_from_mapping(v) for v in environment_variables_dict.values()] + + # decrypt secret variables value + decrypt_func = ( + lambda var: var.model_copy( + update={'value': encrypter.decrypt_token(tenant_id=tenant_id, token=var.value)} + ) + if isinstance(var, SecretVariable) + else var + ) + results = list(map(decrypt_func, results)) + return results + + @environment_variables.setter + def environment_variables(self, value: Sequence[Variable]): + tenant_id = contexts.tenant_id.get() + + value = list(value) + if any(var for var in value if not var.id): + raise ValueError('environment variable require a unique id') + + # Compare inputs and origin variables, if the value is HIDDEN_VALUE, use the origin variable value (only update `name`). + origin_variables_dictionary = {var.id: var for var in self.environment_variables} + for i, variable in enumerate(value): + if variable.id in origin_variables_dictionary and variable.value == HIDDEN_VALUE: + value[i] = origin_variables_dictionary[variable.id].model_copy(update={'name': variable.name}) + + # encrypt secret variables value + encrypt_func = ( + lambda var: var.model_copy( + update={'value': encrypter.encrypt_token(tenant_id=tenant_id, token=var.value)} + ) + if isinstance(var, SecretVariable) + else var + ) + encrypted_vars = list(map(encrypt_func, value)) + environment_variables_json = json.dumps( + {var.name: var.model_dump() for var in encrypted_vars}, + ensure_ascii=False, + ) + self._environment_variables = environment_variables_json + + def to_dict(self, *, include_secret: bool = False) -> Mapping[str, Any]: + environment_variables = list(self.environment_variables) + environment_variables = [ + v if not isinstance(v, SecretVariable) or include_secret else v.model_copy(update={'value': ''}) + for v in environment_variables + ] + + result = { + 'graph': self.graph_dict, + 'features': self.features_dict, + 'environment_variables': [var.model_dump(mode='json') for var in environment_variables], + } + return result + + class WorkflowRunTriggeredFrom(Enum): """ Workflow Run Triggered From Enum diff --git a/api/schedule/clean_unused_datasets_task.py b/api/schedule/clean_unused_datasets_task.py index 9cbee6e81e4b8e..b2b2f82b786f5e 100644 --- a/api/schedule/clean_unused_datasets_task.py +++ b/api/schedule/clean_unused_datasets_task.py @@ -15,7 +15,7 @@ @app.celery.task(queue='dataset') def clean_unused_datasets_task(): click.echo(click.style('Start clean unused datasets indexes.', fg='green')) - clean_days = int(dify_config.CLEAN_DAY_SETTING) + clean_days = dify_config.CLEAN_DAY_SETTING start_at = time.perf_counter() thirty_days_ago = datetime.datetime.now() - datetime.timedelta(days=clean_days) page = 1 diff --git a/api/services/account_service.py b/api/services/account_service.py index 0bcbe8b2c03557..d73cec2697f975 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -47,7 +47,7 @@ class AccountService: ) @staticmethod - def load_user(user_id: str) -> Account: + def load_user(user_id: str) -> None | Account: account = Account.query.filter_by(id=user_id).first() if not account: return None @@ -55,7 +55,7 @@ def load_user(user_id: str) -> Account: if account.status in [AccountStatus.BANNED.value, AccountStatus.CLOSED.value]: raise Unauthorized("Account is banned or closed.") - current_tenant = TenantAccountJoin.query.filter_by(account_id=account.id, current=True).first() + current_tenant: TenantAccountJoin = TenantAccountJoin.query.filter_by(account_id=account.id, current=True).first() if current_tenant: account.current_tenant_id = current_tenant.tenant_id else: diff --git a/api/services/app_dsl_service.py b/api/services/app_dsl_service.py index 050295002e7634..3764166333255d 100644 --- a/api/services/app_dsl_service.py +++ b/api/services/app_dsl_service.py @@ -3,6 +3,7 @@ import httpx import yaml # type: ignore +from core.app.segments import factory from events.app_event import app_model_config_was_updated, app_was_created from extensions.ext_database import db from models.account import Account @@ -150,7 +151,7 @@ def import_and_overwrite_workflow(cls, app_model: App, data: str, account: Accou ) @classmethod - def export_dsl(cls, app_model: App) -> str: + def export_dsl(cls, app_model: App, include_secret:bool = False) -> str: """ Export app :param app_model: App instance @@ -171,7 +172,7 @@ def export_dsl(cls, app_model: App) -> str: } if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]: - cls._append_workflow_export_data(export_data, app_model) + cls._append_workflow_export_data(export_data=export_data, app_model=app_model, include_secret=include_secret) else: cls._append_model_config_export_data(export_data, app_model) @@ -235,13 +236,16 @@ def _import_and_create_new_workflow_based_app(cls, ) # init draft workflow + environment_variables_list = workflow_data.get('environment_variables') or [] + environment_variables = [factory.build_variable_from_mapping(obj) for obj in environment_variables_list] workflow_service = WorkflowService() draft_workflow = workflow_service.sync_draft_workflow( app_model=app, graph=workflow_data.get('graph', {}), features=workflow_data.get('../core/app/features', {}), unique_hash=None, - account=account + account=account, + environment_variables=environment_variables, ) workflow_service.publish_workflow( app_model=app, @@ -276,12 +280,15 @@ def _import_and_overwrite_workflow_based_app(cls, unique_hash = None # sync draft workflow + environment_variables_list = workflow_data.get('environment_variables') or [] + environment_variables = [factory.build_variable_from_mapping(obj) for obj in environment_variables_list] draft_workflow = workflow_service.sync_draft_workflow( app_model=app_model, graph=workflow_data.get('graph', {}), features=workflow_data.get('features', {}), unique_hash=unique_hash, - account=account + account=account, + environment_variables=environment_variables, ) return draft_workflow @@ -377,7 +384,7 @@ def _create_app(cls, return app @classmethod - def _append_workflow_export_data(cls, export_data: dict, app_model: App) -> None: + def _append_workflow_export_data(cls, *, export_data: dict, app_model: App, include_secret: bool) -> None: """ Append workflow export data :param export_data: export data @@ -388,10 +395,7 @@ def _append_workflow_export_data(cls, export_data: dict, app_model: App) -> None if not workflow: raise ValueError("Missing draft workflow configuration, please check.") - export_data['workflow'] = { - "graph": workflow.graph_dict, - "features": workflow.features_dict - } + export_data['workflow'] = workflow.to_dict(include_secret=include_secret) @classmethod def _append_model_config_export_data(cls, export_data: dict, app_model: App) -> None: diff --git a/api/services/model_load_balancing_service.py b/api/services/model_load_balancing_service.py index c684c2862b9745..4f59b86c12a222 100644 --- a/api/services/model_load_balancing_service.py +++ b/api/services/model_load_balancing_service.py @@ -133,6 +133,7 @@ def get_load_balancing_configs(self, tenant_id: str, provider: str, model: str, # move the inherit configuration to the first for i, load_balancing_config in enumerate(load_balancing_configs): if load_balancing_config.name == '__inherit__': + # FIXME: Mutation to loop iterable `load_balancing_configs` during iteration inherit_config = load_balancing_configs.pop(i) load_balancing_configs.insert(0, inherit_config) diff --git a/api/services/recommended_app_service.py b/api/services/recommended_app_service.py index 1c1c5be17c64b3..20d21c22a912c5 100644 --- a/api/services/recommended_app_service.py +++ b/api/services/recommended_app_service.py @@ -4,7 +4,6 @@ from typing import Optional import requests -from flask import current_app from configs import dify_config from constants.languages import languages diff --git a/api/services/workflow/workflow_converter.py b/api/services/workflow/workflow_converter.py index 010d53389aef28..06b129be691010 100644 --- a/api/services/workflow/workflow_converter.py +++ b/api/services/workflow/workflow_converter.py @@ -199,7 +199,8 @@ def convert_app_model_config_to_workflow(self, app_model: App, version='draft', graph=json.dumps(graph), features=json.dumps(features), - created_by=account_id + created_by=account_id, + environment_variables=[], ) db.session.add(workflow) diff --git a/api/services/workflow_service.py b/api/services/workflow_service.py index 6235ecf0a36543..d868255f96e419 100644 --- a/api/services/workflow_service.py +++ b/api/services/workflow_service.py @@ -1,10 +1,12 @@ import json import time +from collections.abc import Sequence from datetime import datetime, timezone from typing import Optional from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager +from core.app.segments import Variable from core.model_runtime.utils.encoders import jsonable_encoder from core.workflow.entities.node_entities import NodeType from core.workflow.errors import WorkflowNodeRunFailedError @@ -61,11 +63,16 @@ def get_published_workflow(self, app_model: App) -> Optional[Workflow]: return workflow - def sync_draft_workflow(self, app_model: App, - graph: dict, - features: dict, - unique_hash: Optional[str], - account: Account) -> Workflow: + def sync_draft_workflow( + self, + *, + app_model: App, + graph: dict, + features: dict, + unique_hash: Optional[str], + account: Account, + environment_variables: Sequence[Variable], + ) -> Workflow: """ Sync draft workflow :raises WorkflowHashNotEqualError @@ -73,10 +80,8 @@ def sync_draft_workflow(self, app_model: App, # fetch draft workflow by app_model workflow = self.get_draft_workflow(app_model=app_model) - if workflow: - # validate unique hash - if workflow.unique_hash != unique_hash: - raise WorkflowHashNotEqualError() + if workflow and workflow.unique_hash != unique_hash: + raise WorkflowHashNotEqualError() # validate features structure self.validate_features_structure( @@ -93,7 +98,8 @@ def sync_draft_workflow(self, app_model: App, version='draft', graph=json.dumps(graph), features=json.dumps(features), - created_by=account.id + created_by=account.id, + environment_variables=environment_variables ) db.session.add(workflow) # update draft workflow if found @@ -102,6 +108,7 @@ def sync_draft_workflow(self, app_model: App, workflow.features = json.dumps(features) workflow.updated_by = account.id workflow.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) + workflow.environment_variables = environment_variables # commit db session changes db.session.commit() @@ -137,7 +144,8 @@ def publish_workflow(self, app_model: App, version=str(datetime.now(timezone.utc).replace(tzinfo=None)), graph=draft_workflow.graph, features=draft_workflow.features, - created_by=account.id + created_by=account.id, + environment_variables=draft_workflow.environment_variables ) # commit db session changes diff --git a/api/tests/integration_tests/workflow/nodes/test_code.py b/api/tests/integration_tests/workflow/nodes/test_code.py index 15cf5367d38d66..5c952585208d7b 100644 --- a/api/tests/integration_tests/workflow/nodes/test_code.py +++ b/api/tests/integration_tests/workflow/nodes/test_code.py @@ -55,9 +55,9 @@ def main(args1: int, args2: int) -> dict: ) # construct variable pool - pool = VariablePool(system_variables={}, user_inputs={}) - pool.append_variable(node_id='1', variable_key_list=['123', 'args1'], value=1) - pool.append_variable(node_id='1', variable_key_list=['123', 'args2'], value=2) + pool = VariablePool(system_variables={}, user_inputs={}, environment_variables=[]) + pool.add(['1', '123', 'args1'], 1) + pool.add(['1', '123', 'args2'], 2) # execute node result = node.run(pool) @@ -109,9 +109,9 @@ def main(args1: int, args2: int) -> dict: ) # construct variable pool - pool = VariablePool(system_variables={}, user_inputs={}) - pool.append_variable(node_id='1', variable_key_list=['123', 'args1'], value=1) - pool.append_variable(node_id='1', variable_key_list=['123', 'args2'], value=2) + pool = VariablePool(system_variables={}, user_inputs={}, environment_variables=[]) + pool.add(['1', '123', 'args1'], 1) + pool.add(['1', '123', 'args2'], 2) # execute node result = node.run(pool) diff --git a/api/tests/integration_tests/workflow/nodes/test_http.py b/api/tests/integration_tests/workflow/nodes/test_http.py index 2ab2eb7ecf7036..a1354bd6a5f220 100644 --- a/api/tests/integration_tests/workflow/nodes/test_http.py +++ b/api/tests/integration_tests/workflow/nodes/test_http.py @@ -18,9 +18,9 @@ } # construct variable pool -pool = VariablePool(system_variables={}, user_inputs={}) -pool.append_variable(node_id='a', variable_key_list=['b123', 'args1'], value=1) -pool.append_variable(node_id='a', variable_key_list=['b123', 'args2'], value=2) +pool = VariablePool(system_variables={}, user_inputs={}, environment_variables=[]) +pool.add(['a', 'b123', 'args1'], 1) +pool.add(['a', 'b123', 'args2'], 2) @pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True) diff --git a/api/tests/integration_tests/workflow/nodes/test_llm.py b/api/tests/integration_tests/workflow/nodes/test_llm.py index d7a6c1224f5c5a..ac704e4eaf54df 100644 --- a/api/tests/integration_tests/workflow/nodes/test_llm.py +++ b/api/tests/integration_tests/workflow/nodes/test_llm.py @@ -70,8 +70,8 @@ def test_execute_llm(setup_openai_mock): SystemVariable.FILES: [], SystemVariable.CONVERSATION_ID: 'abababa', SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) - pool.append_variable(node_id='abc', variable_key_list=['output'], value='sunny') + }, user_inputs={}, environment_variables=[]) + pool.add(['abc', 'output'], 'sunny') credentials = { 'openai_api_key': os.environ.get('OPENAI_API_KEY') @@ -185,8 +185,8 @@ def test_execute_llm_with_jinja2(setup_code_executor_mock, setup_openai_mock): SystemVariable.FILES: [], SystemVariable.CONVERSATION_ID: 'abababa', SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) - pool.append_variable(node_id='abc', variable_key_list=['output'], value='sunny') + }, user_inputs={}, environment_variables=[]) + pool.add(['abc', 'output'], 'sunny') credentials = { 'openai_api_key': os.environ.get('OPENAI_API_KEY') diff --git a/api/tests/integration_tests/workflow/nodes/test_parameter_extractor.py b/api/tests/integration_tests/workflow/nodes/test_parameter_extractor.py index e5fd2bc1fd2ed9..312ad47026beb5 100644 --- a/api/tests/integration_tests/workflow/nodes/test_parameter_extractor.py +++ b/api/tests/integration_tests/workflow/nodes/test_parameter_extractor.py @@ -123,7 +123,7 @@ def test_function_calling_parameter_extractor(setup_openai_mock): SystemVariable.FILES: [], SystemVariable.CONVERSATION_ID: 'abababa', SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) + }, user_inputs={}, environment_variables=[]) result = node.run(pool) @@ -181,7 +181,7 @@ def test_instructions(setup_openai_mock): SystemVariable.FILES: [], SystemVariable.CONVERSATION_ID: 'abababa', SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) + }, user_inputs={}, environment_variables=[]) result = node.run(pool) @@ -247,7 +247,7 @@ def test_chat_parameter_extractor(setup_anthropic_mock): SystemVariable.FILES: [], SystemVariable.CONVERSATION_ID: 'abababa', SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) + }, user_inputs={}, environment_variables=[]) result = node.run(pool) @@ -311,7 +311,7 @@ def test_completion_parameter_extractor(setup_openai_mock): SystemVariable.FILES: [], SystemVariable.CONVERSATION_ID: 'abababa', SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) + }, user_inputs={}, environment_variables=[]) result = node.run(pool) @@ -424,7 +424,7 @@ def test_chat_parameter_extractor_with_memory(setup_anthropic_mock): SystemVariable.FILES: [], SystemVariable.CONVERSATION_ID: 'abababa', SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) + }, user_inputs={}, environment_variables=[]) result = node.run(pool) diff --git a/api/tests/integration_tests/workflow/nodes/test_template_transform.py b/api/tests/integration_tests/workflow/nodes/test_template_transform.py index 02999bf0a2615e..781dfbc50fdba3 100644 --- a/api/tests/integration_tests/workflow/nodes/test_template_transform.py +++ b/api/tests/integration_tests/workflow/nodes/test_template_transform.py @@ -38,9 +38,9 @@ def test_execute_code(setup_code_executor_mock): ) # construct variable pool - pool = VariablePool(system_variables={}, user_inputs={}) - pool.append_variable(node_id='1', variable_key_list=['123', 'args1'], value=1) - pool.append_variable(node_id='1', variable_key_list=['123', 'args2'], value=3) + pool = VariablePool(system_variables={}, user_inputs={}, environment_variables=[]) + pool.add(['1', '123', 'args1'], 1) + pool.add(['1', '123', 'args2'], 3) # execute node result = node.run(pool) diff --git a/api/tests/integration_tests/workflow/nodes/test_tool.py b/api/tests/integration_tests/workflow/nodes/test_tool.py index fffd074457e454..01d62280e837b3 100644 --- a/api/tests/integration_tests/workflow/nodes/test_tool.py +++ b/api/tests/integration_tests/workflow/nodes/test_tool.py @@ -6,8 +6,8 @@ def test_tool_variable_invoke(): - pool = VariablePool(system_variables={}, user_inputs={}) - pool.append_variable(node_id='1', variable_key_list=['123', 'args1'], value='1+1') + pool = VariablePool(system_variables={}, user_inputs={}, environment_variables=[]) + pool.add(['1', '123', 'args1'], '1+1') node = ToolNode( tenant_id='1', @@ -45,8 +45,8 @@ def test_tool_variable_invoke(): assert result.outputs['files'] == [] def test_tool_mixed_invoke(): - pool = VariablePool(system_variables={}, user_inputs={}) - pool.append_variable(node_id='1', variable_key_list=['args1'], value='1+1') + pool = VariablePool(system_variables={}, user_inputs={}, environment_variables=[]) + pool.add(['1', 'args1'], '1+1') node = ToolNode( tenant_id='1', diff --git a/api/tests/unit_tests/app/test_segment.py b/api/tests/unit_tests/app/test_segment.py new file mode 100644 index 00000000000000..7ef37ff64668f6 --- /dev/null +++ b/api/tests/unit_tests/app/test_segment.py @@ -0,0 +1,53 @@ +from core.app.segments import SecretVariable, parser +from core.helper import encrypter +from core.workflow.entities.node_entities import SystemVariable +from core.workflow.entities.variable_pool import VariablePool + + +def test_segment_group_to_text(): + variable_pool = VariablePool( + system_variables={ + SystemVariable('user_id'): 'fake-user-id', + }, + user_inputs={}, + environment_variables=[ + SecretVariable(name='secret_key', value='fake-secret-key'), + ], + ) + variable_pool.add(('node_id', 'custom_query'), 'fake-user-query') + template = ( + 'Hello, {{#sys.user_id#}}! Your query is {{#node_id.custom_query#}}. And your key is {{#env.secret_key#}}.' + ) + segments_group = parser.convert_template(template=template, variable_pool=variable_pool) + + assert segments_group.text == 'Hello, fake-user-id! Your query is fake-user-query. And your key is fake-secret-key.' + assert ( + segments_group.log + == f"Hello, fake-user-id! Your query is fake-user-query. And your key is {encrypter.obfuscated_token('fake-secret-key')}." + ) + + +def test_convert_constant_to_segment_group(): + variable_pool = VariablePool( + system_variables={}, + user_inputs={}, + environment_variables=[], + ) + template = 'Hello, world!' + segments_group = parser.convert_template(template=template, variable_pool=variable_pool) + assert segments_group.text == 'Hello, world!' + assert segments_group.log == 'Hello, world!' + + +def test_convert_variable_to_segment_group(): + variable_pool = VariablePool( + system_variables={ + SystemVariable('user_id'): 'fake-user-id', + }, + user_inputs={}, + environment_variables=[], + ) + template = '{{#sys.user_id#}}' + segments_group = parser.convert_template(template=template, variable_pool=variable_pool) + assert segments_group.text == 'fake-user-id' + assert segments_group.log == 'fake-user-id' diff --git a/api/tests/unit_tests/app/test_variables.py b/api/tests/unit_tests/app/test_variables.py new file mode 100644 index 00000000000000..65db88a4a84676 --- /dev/null +++ b/api/tests/unit_tests/app/test_variables.py @@ -0,0 +1,91 @@ +import pytest +from pydantic import ValidationError + +from core.app.segments import ( + FloatVariable, + IntegerVariable, + SecretVariable, + SegmentType, + StringVariable, + factory, +) + + +def test_string_variable(): + test_data = {'value_type': 'string', 'name': 'test_text', 'value': 'Hello, World!'} + result = factory.build_variable_from_mapping(test_data) + assert isinstance(result, StringVariable) + + +def test_integer_variable(): + test_data = {'value_type': 'number', 'name': 'test_int', 'value': 42} + result = factory.build_variable_from_mapping(test_data) + assert isinstance(result, IntegerVariable) + + +def test_float_variable(): + test_data = {'value_type': 'number', 'name': 'test_float', 'value': 3.14} + result = factory.build_variable_from_mapping(test_data) + assert isinstance(result, FloatVariable) + + +def test_secret_variable(): + test_data = {'value_type': 'secret', 'name': 'test_secret', 'value': 'secret_value'} + result = factory.build_variable_from_mapping(test_data) + assert isinstance(result, SecretVariable) + + +def test_invalid_value_type(): + test_data = {'value_type': 'unknown', 'name': 'test_invalid', 'value': 'value'} + with pytest.raises(ValueError): + factory.build_variable_from_mapping(test_data) + + +def test_frozen_variables(): + var = StringVariable(name='text', value='text') + with pytest.raises(ValidationError): + var.value = 'new value' + + int_var = IntegerVariable(name='integer', value=42) + with pytest.raises(ValidationError): + int_var.value = 100 + + float_var = FloatVariable(name='float', value=3.14) + with pytest.raises(ValidationError): + float_var.value = 2.718 + + secret_var = SecretVariable(name='secret', value='secret_value') + with pytest.raises(ValidationError): + secret_var.value = 'new_secret_value' + + +def test_variable_value_type_immutable(): + with pytest.raises(ValidationError): + StringVariable(value_type=SegmentType.ARRAY, name='text', value='text') + + with pytest.raises(ValidationError): + StringVariable.model_validate({'value_type': 'not text', 'name': 'text', 'value': 'text'}) + + var = IntegerVariable(name='integer', value=42) + with pytest.raises(ValidationError): + IntegerVariable(value_type=SegmentType.ARRAY, name=var.name, value=var.value) + + var = FloatVariable(name='float', value=3.14) + with pytest.raises(ValidationError): + FloatVariable(value_type=SegmentType.ARRAY, name=var.name, value=var.value) + + var = SecretVariable(name='secret', value='secret_value') + with pytest.raises(ValidationError): + SecretVariable(value_type=SegmentType.ARRAY, name=var.name, value=var.value) + + +def test_build_a_blank_string(): + result = factory.build_variable_from_mapping( + { + 'value_type': 'string', + 'name': 'blank', + 'value': '', + } + ) + assert isinstance(result, StringVariable) + assert result.value == '' diff --git a/api/tests/unit_tests/configs/test_dify_config.py b/api/tests/unit_tests/configs/test_dify_config.py index 50bb2b75ac2fbd..949a5a17693457 100644 --- a/api/tests/unit_tests/configs/test_dify_config.py +++ b/api/tests/unit_tests/configs/test_dify_config.py @@ -1,3 +1,4 @@ +import os from textwrap import dedent import pytest @@ -48,7 +49,9 @@ def test_dify_config(example_env_file): # This is due to `pymilvus` loading all the variables from the `.env` file into `os.environ`. def test_flask_configs(example_env_file): flask_app = Flask('app') - flask_app.config.from_mapping(DifyConfig(_env_file=example_env_file).model_dump()) + # clear system environment variables + os.environ.clear() + flask_app.config.from_mapping(DifyConfig(_env_file=example_env_file).model_dump()) # pyright: ignore config = flask_app.config # configs read from pydantic-settings diff --git a/api/tests/unit_tests/core/workflow/nodes/test_answer.py b/api/tests/unit_tests/core/workflow/nodes/test_answer.py index 102711b4b642d1..3a32829e373c28 100644 --- a/api/tests/unit_tests/core/workflow/nodes/test_answer.py +++ b/api/tests/unit_tests/core/workflow/nodes/test_answer.py @@ -31,9 +31,9 @@ def test_execute_answer(): pool = VariablePool(system_variables={ SystemVariable.FILES: [], SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) - pool.append_variable(node_id='start', variable_key_list=['weather'], value='sunny') - pool.append_variable(node_id='llm', variable_key_list=['text'], value='You are a helpful AI.') + }, user_inputs={}, environment_variables=[]) + pool.add(['start', 'weather'], 'sunny') + pool.add(['llm', 'text'], 'You are a helpful AI.') # Mock db.session.close() db.session.close = MagicMock() diff --git a/api/tests/unit_tests/core/workflow/nodes/test_if_else.py b/api/tests/unit_tests/core/workflow/nodes/test_if_else.py index 6860b2fd97740b..4662c5ff2b26d8 100644 --- a/api/tests/unit_tests/core/workflow/nodes/test_if_else.py +++ b/api/tests/unit_tests/core/workflow/nodes/test_if_else.py @@ -121,24 +121,24 @@ def test_execute_if_else_result_true(): pool = VariablePool(system_variables={ SystemVariable.FILES: [], SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) - pool.append_variable(node_id='start', variable_key_list=['array_contains'], value=['ab', 'def']) - pool.append_variable(node_id='start', variable_key_list=['array_not_contains'], value=['ac', 'def']) - pool.append_variable(node_id='start', variable_key_list=['contains'], value='cabcde') - pool.append_variable(node_id='start', variable_key_list=['not_contains'], value='zacde') - pool.append_variable(node_id='start', variable_key_list=['start_with'], value='abc') - pool.append_variable(node_id='start', variable_key_list=['end_with'], value='zzab') - pool.append_variable(node_id='start', variable_key_list=['is'], value='ab') - pool.append_variable(node_id='start', variable_key_list=['is_not'], value='aab') - pool.append_variable(node_id='start', variable_key_list=['empty'], value='') - pool.append_variable(node_id='start', variable_key_list=['not_empty'], value='aaa') - pool.append_variable(node_id='start', variable_key_list=['equals'], value=22) - pool.append_variable(node_id='start', variable_key_list=['not_equals'], value=23) - pool.append_variable(node_id='start', variable_key_list=['greater_than'], value=23) - pool.append_variable(node_id='start', variable_key_list=['less_than'], value=21) - pool.append_variable(node_id='start', variable_key_list=['greater_than_or_equal'], value=22) - pool.append_variable(node_id='start', variable_key_list=['less_than_or_equal'], value=21) - pool.append_variable(node_id='start', variable_key_list=['not_null'], value='1212') + }, user_inputs={}, environment_variables=[]) + pool.add(['start', 'array_contains'], ['ab', 'def']) + pool.add(['start', 'array_not_contains'], ['ac', 'def']) + pool.add(['start', 'contains'], 'cabcde') + pool.add(['start', 'not_contains'], 'zacde') + pool.add(['start', 'start_with'], 'abc') + pool.add(['start', 'end_with'], 'zzab') + pool.add(['start', 'is'], 'ab') + pool.add(['start', 'is_not'], 'aab') + pool.add(['start', 'empty'], '') + pool.add(['start', 'not_empty'], 'aaa') + pool.add(['start', 'equals'], 22) + pool.add(['start', 'not_equals'], 23) + pool.add(['start', 'greater_than'], 23) + pool.add(['start', 'less_than'], 21) + pool.add(['start', 'greater_than_or_equal'], 22) + pool.add(['start', 'less_than_or_equal'], 21) + pool.add(['start', 'not_null'], '1212') # Mock db.session.close() db.session.close = MagicMock() @@ -184,9 +184,9 @@ def test_execute_if_else_result_false(): pool = VariablePool(system_variables={ SystemVariable.FILES: [], SystemVariable.USER_ID: 'aaa' - }, user_inputs={}) - pool.append_variable(node_id='start', variable_key_list=['array_contains'], value=['1ab', 'def']) - pool.append_variable(node_id='start', variable_key_list=['array_not_contains'], value=['ab', 'def']) + }, user_inputs={}, environment_variables=[]) + pool.add(['start', 'array_contains'], ['1ab', 'def']) + pool.add(['start', 'array_not_contains'], ['ab', 'def']) # Mock db.session.close() db.session.close = MagicMock() diff --git a/api/tests/unit_tests/models/test_workflow.py b/api/tests/unit_tests/models/test_workflow.py new file mode 100644 index 00000000000000..facea34b5b4c9c --- /dev/null +++ b/api/tests/unit_tests/models/test_workflow.py @@ -0,0 +1,95 @@ +from unittest import mock +from uuid import uuid4 + +import contexts +from constants import HIDDEN_VALUE +from core.app.segments import FloatVariable, IntegerVariable, SecretVariable, StringVariable +from models.workflow import Workflow + + +def test_environment_variables(): + contexts.tenant_id.set('tenant_id') + + # Create a Workflow instance + workflow = Workflow() + + # Create some EnvironmentVariable instances + variable1 = StringVariable.model_validate({'name': 'var1', 'value': 'value1', 'id': str(uuid4())}) + variable2 = IntegerVariable.model_validate({'name': 'var2', 'value': 123, 'id': str(uuid4())}) + variable3 = SecretVariable.model_validate({'name': 'var3', 'value': 'secret', 'id': str(uuid4())}) + variable4 = FloatVariable.model_validate({'name': 'var4', 'value': 3.14, 'id': str(uuid4())}) + + with ( + mock.patch('core.helper.encrypter.encrypt_token', return_value='encrypted_token'), + mock.patch('core.helper.encrypter.decrypt_token', return_value='secret'), + ): + # Set the environment_variables property of the Workflow instance + variables = [variable1, variable2, variable3, variable4] + workflow.environment_variables = variables + + # Get the environment_variables property and assert its value + assert workflow.environment_variables == variables + + +def test_update_environment_variables(): + contexts.tenant_id.set('tenant_id') + + # Create a Workflow instance + workflow = Workflow() + + # Create some EnvironmentVariable instances + variable1 = StringVariable.model_validate({'name': 'var1', 'value': 'value1', 'id': str(uuid4())}) + variable2 = IntegerVariable.model_validate({'name': 'var2', 'value': 123, 'id': str(uuid4())}) + variable3 = SecretVariable.model_validate({'name': 'var3', 'value': 'secret', 'id': str(uuid4())}) + variable4 = FloatVariable.model_validate({'name': 'var4', 'value': 3.14, 'id': str(uuid4())}) + + with ( + mock.patch('core.helper.encrypter.encrypt_token', return_value='encrypted_token'), + mock.patch('core.helper.encrypter.decrypt_token', return_value='secret'), + ): + variables = [variable1, variable2, variable3, variable4] + + # Set the environment_variables property of the Workflow instance + workflow.environment_variables = variables + assert workflow.environment_variables == [variable1, variable2, variable3, variable4] + + # Update the name of variable3 and keep the value as it is + variables[2] = variable3.model_copy( + update={ + 'name': 'new name', + 'value': HIDDEN_VALUE, + } + ) + + workflow.environment_variables = variables + assert workflow.environment_variables[2].name == 'new name' + assert workflow.environment_variables[2].value == variable3.value + + +def test_to_dict(): + contexts.tenant_id.set('tenant_id') + + # Create a Workflow instance + workflow = Workflow() + workflow.graph = '{}' + workflow.features = '{}' + + # Create some EnvironmentVariable instances + + with ( + mock.patch('core.helper.encrypter.encrypt_token', return_value='encrypted_token'), + mock.patch('core.helper.encrypter.decrypt_token', return_value='secret'), + ): + # Set the environment_variables property of the Workflow instance + workflow.environment_variables = [ + SecretVariable.model_validate({'name': 'secret', 'value': 'secret', 'id': str(uuid4())}), + StringVariable.model_validate({'name': 'text', 'value': 'text', 'id': str(uuid4())}), + ] + + workflow_dict = workflow.to_dict() + assert workflow_dict['environment_variables'][0]['value'] == '' + assert workflow_dict['environment_variables'][1]['value'] == 'text' + + workflow_dict = workflow.to_dict(include_secret=True) + assert workflow_dict['environment_variables'][0]['value'] == 'secret' + assert workflow_dict['environment_variables'][1]['value'] == 'text' diff --git a/web/app/(commonLayout)/apps/AppCard.tsx b/web/app/(commonLayout)/apps/AppCard.tsx index 53b31af7f0a91e..34279f816ebdc0 100644 --- a/web/app/(commonLayout)/apps/AppCard.tsx +++ b/web/app/(commonLayout)/apps/AppCard.tsx @@ -28,6 +28,9 @@ import EditAppModal from '@/app/components/explore/create-app-modal' import SwitchAppModal from '@/app/components/app/switch-app-modal' import type { Tag } from '@/app/components/base/tag-management/constant' import TagSelector from '@/app/components/base/tag-management/selector' +import type { EnvironmentVariable } from '@/app/components/workflow/types' +import DSLExportConfirmModal from '@/app/components/workflow/dsl-export-confirm-modal' +import { fetchWorkflowDraft } from '@/service/workflow' export type AppCardProps = { app: App @@ -50,6 +53,7 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => { const [showDuplicateModal, setShowDuplicateModal] = useState(false) const [showSwitchModal, setShowSwitchModal] = useState(false) const [showConfirmDelete, setShowConfirmDelete] = useState(false) + const [secretEnvList, setSecretEnvList] = useState([]) const onConfirmDelete = useCallback(async () => { try { @@ -123,9 +127,12 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => { } } - const onExport = async () => { + const onExport = async (include = false) => { try { - const { data } = await exportAppConfig(app.id) + const { data } = await exportAppConfig({ + appID: app.id, + include, + }) const a = document.createElement('a') const file = new Blob([data], { type: 'application/yaml' }) a.href = URL.createObjectURL(file) @@ -137,6 +144,25 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => { } } + const exportCheck = async () => { + if (app.mode !== 'workflow' && app.mode !== 'advanced-chat') { + onExport() + return + } + try { + const workflowDraft = await fetchWorkflowDraft(`/apps/${app.id}/workflows/draft`) + const list = (workflowDraft.environment_variables || []).filter(env => env.value_type === 'secret') + if (list.length === 0) { + onExport() + return + } + setSecretEnvList(list) + } + catch (e) { + notify({ type: 'error', message: t('app.exportFailed') }) + } + } + const onSwitch = () => { if (onRefresh) onRefresh() @@ -164,7 +190,7 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => { e.stopPropagation() props.onClick?.() e.preventDefault() - onExport() + exportCheck() } const onClickSwitch = async (e: React.MouseEvent) => { e.stopPropagation() @@ -371,6 +397,13 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => { onCancel={() => setShowConfirmDelete(false)} /> )} + {secretEnvList.length > 0 && ( + setSecretEnvList([])} + /> + )} ) } diff --git a/web/app/components/app-sidebar/app-info.tsx b/web/app/components/app-sidebar/app-info.tsx index c931afbe7fcfe4..ef37ff3c78cd07 100644 --- a/web/app/components/app-sidebar/app-info.tsx +++ b/web/app/components/app-sidebar/app-info.tsx @@ -28,6 +28,9 @@ import type { CreateAppModalProps } from '@/app/components/explore/create-app-mo import { NEED_REFRESH_APP_LIST_KEY } from '@/config' import { getRedirection } from '@/utils/app-redirection' import UpdateDSLModal from '@/app/components/workflow/update-dsl-modal' +import type { EnvironmentVariable } from '@/app/components/workflow/types' +import DSLExportConfirmModal from '@/app/components/workflow/dsl-export-confirm-modal' +import { fetchWorkflowDraft } from '@/service/workflow' export type IAppInfoProps = { expand: boolean @@ -47,6 +50,7 @@ const AppInfo = ({ expand }: IAppInfoProps) => { const [showSwitchTip, setShowSwitchTip] = useState('') const [showSwitchModal, setShowSwitchModal] = useState(false) const [showImportDSLModal, setShowImportDSLModal] = useState(false) + const [secretEnvList, setSecretEnvList] = useState([]) const mutateApps = useContextSelector( AppsContext, @@ -108,11 +112,14 @@ const AppInfo = ({ expand }: IAppInfoProps) => { } } - const onExport = async () => { + const onExport = async (include = false) => { if (!appDetail) return try { - const { data } = await exportAppConfig(appDetail.id) + const { data } = await exportAppConfig({ + appID: appDetail.id, + include, + }) const a = document.createElement('a') const file = new Blob([data], { type: 'application/yaml' }) a.href = URL.createObjectURL(file) @@ -124,6 +131,27 @@ const AppInfo = ({ expand }: IAppInfoProps) => { } } + const exportCheck = async () => { + if (!appDetail) + return + if (appDetail.mode !== 'workflow' && appDetail.mode !== 'advanced-chat') { + onExport() + return + } + try { + const workflowDraft = await fetchWorkflowDraft(`/apps/${appDetail.id}/workflows/draft`) + const list = (workflowDraft.environment_variables || []).filter(env => env.value_type === 'secret') + if (list.length === 0) { + onExport() + return + } + setSecretEnvList(list) + } + catch (e) { + notify({ type: 'error', message: t('app.exportFailed') }) + } + } + const onConfirmDelete = useCallback(async () => { if (!appDetail) return @@ -314,7 +342,7 @@ const AppInfo = ({ expand }: IAppInfoProps) => { )} -
+
{t('app.export')}
{ @@ -403,14 +431,19 @@ const AppInfo = ({ expand }: IAppInfoProps) => { onCancel={() => setShowConfirmDelete(false)} /> )} - { - showImportDSLModal && ( - setShowImportDSLModal(false)} - onBackup={onExport} - /> - ) - } + {showImportDSLModal && ( + setShowImportDSLModal(false)} + onBackup={onExport} + /> + )} + {secretEnvList.length > 0 && ( + setSecretEnvList([])} + /> + )}
) diff --git a/web/app/components/app/app-publisher/index.tsx b/web/app/components/app/app-publisher/index.tsx index d7e9856ce46cce..e971274a71b3bc 100644 --- a/web/app/components/app/app-publisher/index.tsx +++ b/web/app/components/app/app-publisher/index.tsx @@ -119,11 +119,11 @@ const AppPublisher = ({ diff --git a/web/app/components/base/icons/assets/vender/line/communication/message-play.svg b/web/app/components/base/icons/assets/vender/line/communication/message-play.svg deleted file mode 100644 index 79460f56549add..00000000000000 --- a/web/app/components/base/icons/assets/vender/line/communication/message-play.svg +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/web/app/components/base/icons/assets/vender/line/others/env.svg b/web/app/components/base/icons/assets/vender/line/others/env.svg new file mode 100644 index 00000000000000..b183d406806e2b --- /dev/null +++ b/web/app/components/base/icons/assets/vender/line/others/env.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/web/app/components/base/icons/src/vender/line/communication/MessagePlay.json b/web/app/components/base/icons/src/vender/line/communication/MessagePlay.json deleted file mode 100644 index 5624650604ab0a..00000000000000 --- a/web/app/components/base/icons/src/vender/line/communication/MessagePlay.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "icon": { - "type": "element", - "isRootNode": true, - "name": "svg", - "attributes": { - "width": "17", - "height": "16", - "viewBox": "0 0 17 16", - "fill": "none", - "xmlns": "http://www.w3.org/2000/svg" - }, - "children": [ - { - "type": "element", - "name": "g", - "attributes": { - "id": "Left Icon" - }, - "children": [ - { - "type": "element", - "name": "path", - "attributes": { - "id": "Vector", - "d": "M7.83333 2.66683H5.7C4.5799 2.66683 4.01984 2.66683 3.59202 2.88482C3.21569 3.07656 2.90973 3.38252 2.71799 3.75885C2.5 4.18667 2.5 4.74672 2.5 5.86683V9.3335C2.5 9.95348 2.5 10.2635 2.56815 10.5178C2.75308 11.208 3.29218 11.7471 3.98236 11.932C4.2367 12.0002 4.54669 12.0002 5.16667 12.0002V13.5572C5.16667 13.9124 5.16667 14.09 5.23949 14.1812C5.30282 14.2606 5.39885 14.3067 5.50036 14.3066C5.61708 14.3065 5.75578 14.1955 6.03317 13.9736L7.62348 12.7014C7.94834 12.4415 8.11078 12.3115 8.29166 12.2191C8.45213 12.1371 8.62295 12.0772 8.79948 12.041C8.99845 12.0002 9.20646 12.0002 9.6225 12.0002H10.6333C11.7534 12.0002 12.3135 12.0002 12.7413 11.7822C13.1176 11.5904 13.4236 11.2845 13.6153 10.9081C13.8333 10.4803 13.8333 9.92027 13.8333 8.80016V8.66683M11.6551 6.472L14.8021 4.44889C15.0344 4.29958 15.1505 4.22493 15.1906 4.13C15.2257 4.04706 15.2257 3.95347 15.1906 3.87052C15.1505 3.7756 15.0344 3.70094 14.8021 3.55163L11.6551 1.52852C11.3874 1.35646 11.2536 1.27043 11.1429 1.27833C11.0465 1.28522 10.9578 1.33365 10.8998 1.41105C10.8333 1.49987 10.8333 1.65896 10.8333 1.97715V6.02337C10.8333 6.34156 10.8333 6.50066 10.8998 6.58948C10.9578 6.66688 11.0465 6.71531 11.1429 6.72219C11.2536 6.7301 11.3874 6.64407 11.6551 6.472Z", - "stroke": "currentColor", - "stroke-width": "1.5", - "stroke-linecap": "round", - "stroke-linejoin": "round" - }, - "children": [] - } - ] - } - ] - }, - "name": "MessagePlay" -} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/line/communication/index.ts b/web/app/components/base/icons/src/vender/line/communication/index.ts index 27e646ef9836b1..3ab20e8bb431cc 100644 --- a/web/app/components/base/icons/src/vender/line/communication/index.ts +++ b/web/app/components/base/icons/src/vender/line/communication/index.ts @@ -4,4 +4,3 @@ export { default as ChatBot } from './ChatBot' export { default as CuteRobot } from './CuteRobot' export { default as MessageCheckRemove } from './MessageCheckRemove' export { default as MessageFastPlus } from './MessageFastPlus' -export { default as MessagePlay } from './MessagePlay' diff --git a/web/app/components/base/icons/src/vender/line/others/Env.json b/web/app/components/base/icons/src/vender/line/others/Env.json new file mode 100644 index 00000000000000..87a88edf3f431d --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/others/Env.json @@ -0,0 +1,90 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "16", + "height": "16", + "viewBox": "0 0 16 16", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "env" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Vector" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "fill-rule": "evenodd", + "clip-rule": "evenodd", + "d": "M1.33325 3.33325C1.33325 2.22868 2.22868 1.33325 3.33325 1.33325H12.6666C13.7712 1.33325 14.6666 2.22869 14.6666 3.33325V3.66659C14.6666 4.03478 14.3681 4.33325 13.9999 4.33325C13.6317 4.33325 13.3333 4.03478 13.3333 3.66659V3.33325C13.3333 2.96506 13.0348 2.66659 12.6666 2.66659H3.33325C2.96506 2.66659 2.66659 2.96506 2.66659 3.33325V3.66659C2.66659 4.03478 2.36811 4.33325 1.99992 4.33325C1.63173 4.33325 1.33325 4.03478 1.33325 3.66659V3.33325Z", + "fill": "currentColor" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "fill-rule": "evenodd", + "clip-rule": "evenodd", + "d": "M14.6666 12.6666C14.6666 13.7712 13.7712 14.6666 12.6666 14.6666L3.33325 14.6666C2.22866 14.6666 1.33325 13.7711 1.33325 12.6666L1.33325 12.3333C1.33325 11.9651 1.63173 11.6666 1.99992 11.6666C2.36811 11.6666 2.66659 11.9651 2.66659 12.3333V12.6666C2.66659 13.0348 2.96505 13.3333 3.33325 13.3333L12.6666 13.3333C13.0348 13.3333 13.3333 13.0348 13.3333 12.6666V12.3333C13.3333 11.9651 13.6317 11.6666 13.9999 11.6666C14.3681 11.6666 14.6666 11.9651 14.6666 12.3333V12.6666Z", + "fill": "currentColor" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "fill-rule": "evenodd", + "clip-rule": "evenodd", + "d": "M1.33325 5.99992C1.33325 5.63173 1.63173 5.33325 1.99992 5.33325H4.33325C4.70144 5.33325 4.99992 5.63173 4.99992 5.99992C4.99992 6.36811 4.70144 6.66658 4.33325 6.66658H2.66659V7.33325H3.99992C4.36811 7.33325 4.66659 7.63173 4.66659 7.99992C4.66659 8.36811 4.36811 8.66658 3.99992 8.66658H2.66659V9.33325H4.33325C4.70144 9.33325 4.99992 9.63173 4.99992 9.99992C4.99992 10.3681 4.70144 10.6666 4.33325 10.6666H1.99992C1.63173 10.6666 1.33325 10.3681 1.33325 9.99992V5.99992Z", + "fill": "currentColor" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "fill-rule": "evenodd", + "clip-rule": "evenodd", + "d": "M6.4734 5.36186C6.75457 5.27673 7.05833 5.38568 7.22129 5.63012L8.66659 7.79807V5.99992C8.66659 5.63173 8.96506 5.33325 9.33325 5.33325C9.70144 5.33325 9.99992 5.63173 9.99992 5.99992V9.99992C9.99992 10.2937 9.80761 10.5528 9.52644 10.638C9.24527 10.7231 8.94151 10.6142 8.77855 10.3697L7.33325 8.20177V9.99992C7.33325 10.3681 7.03478 10.6666 6.66659 10.6666C6.2984 10.6666 5.99992 10.3681 5.99992 9.99992V5.99992C5.99992 5.70614 6.19222 5.44699 6.4734 5.36186Z", + "fill": "currentColor" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "fill-rule": "evenodd", + "clip-rule": "evenodd", + "d": "M11.0768 5.38453C11.4167 5.24292 11.807 5.40364 11.9486 5.74351L12.9999 8.26658L14.0512 5.74351C14.1928 5.40364 14.5831 5.24292 14.923 5.38453C15.2629 5.52614 15.4236 5.91646 15.282 6.25633L13.6153 10.2563C13.5118 10.5048 13.2691 10.6666 12.9999 10.6666C12.7308 10.6666 12.488 10.5048 12.3845 10.2563L10.7179 6.25633C10.5763 5.91646 10.737 5.52614 11.0768 5.38453Z", + "fill": "currentColor" + }, + "children": [] + } + ] + } + ] + } + ] + }, + "name": "Env" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/line/communication/MessagePlay.tsx b/web/app/components/base/icons/src/vender/line/others/Env.tsx similarity index 85% rename from web/app/components/base/icons/src/vender/line/communication/MessagePlay.tsx rename to web/app/components/base/icons/src/vender/line/others/Env.tsx index e570e49c52adae..356a5f6fb4c569 100644 --- a/web/app/components/base/icons/src/vender/line/communication/MessagePlay.tsx +++ b/web/app/components/base/icons/src/vender/line/others/Env.tsx @@ -2,7 +2,7 @@ // DON NOT EDIT IT MANUALLY import * as React from 'react' -import data from './MessagePlay.json' +import data from './Env.json' import IconBase from '@/app/components/base/icons/IconBase' import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' @@ -11,6 +11,6 @@ const Icon = React.forwardRef, Omit ) -Icon.displayName = 'MessagePlay' +Icon.displayName = 'Env' export default Icon diff --git a/web/app/components/base/icons/src/vender/line/others/index.ts b/web/app/components/base/icons/src/vender/line/others/index.ts index 554f14b55bd650..282a39499f3651 100644 --- a/web/app/components/base/icons/src/vender/line/others/index.ts +++ b/web/app/components/base/icons/src/vender/line/others/index.ts @@ -1,6 +1,7 @@ export { default as Apps02 } from './Apps02' export { default as Colors } from './Colors' export { default as DragHandle } from './DragHandle' +export { default as Env } from './Env' export { default as Exchange02 } from './Exchange02' export { default as FileCode } from './FileCode' export { default as Icon3Dots } from './Icon3Dots' diff --git a/web/app/components/base/prompt-editor/plugins/workflow-variable-block/component.tsx b/web/app/components/base/prompt-editor/plugins/workflow-variable-block/component.tsx index a0743ddb9f9014..e149f5b75a198a 100644 --- a/web/app/components/base/prompt-editor/plugins/workflow-variable-block/component.tsx +++ b/web/app/components/base/prompt-editor/plugins/workflow-variable-block/component.tsx @@ -21,9 +21,10 @@ import { } from './index' import cn from '@/utils/classnames' import { Variable02 } from '@/app/components/base/icons/src/vender/solid/development' +import { Env } from '@/app/components/base/icons/src/vender/line/others' import { VarBlockIcon } from '@/app/components/workflow/block-icon' import { Line3 } from '@/app/components/base/icons/src/public/common' -import { isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils' +import { isENV, isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils' import TooltipPlus from '@/app/components/base/tooltip-plus' type WorkflowVariableBlockComponentProps = { @@ -50,6 +51,7 @@ const WorkflowVariableBlockComponent = ({ )() const [localWorkflowNodesMap, setLocalWorkflowNodesMap] = useState(workflowNodesMap) const node = localWorkflowNodesMap![variables[0]] + const isEnv = isENV(variables) useEffect(() => { if (!editor.hasNodes([WorkflowVariableBlockNode])) @@ -73,30 +75,33 @@ const WorkflowVariableBlockComponent = ({ className={cn( 'mx-0.5 relative group/wrap flex items-center h-[18px] pl-0.5 pr-[3px] rounded-[5px] border select-none', isSelected ? ' border-[#84ADFF] bg-[#F5F8FF]' : ' border-black/5 bg-white', - !node && '!border-[#F04438] !bg-[#FEF3F2]', + !node && !isEnv && '!border-[#F04438] !bg-[#FEF3F2]', )} ref={ref} > -
- { - node?.type && ( -
- -
- ) - } -
{node?.title}
- -
+ {!isEnv && ( +
+ { + node?.type && ( +
+ +
+ ) + } +
{node?.title}
+ +
+ )}
- -
{varName}
+ {!isEnv && } + {isEnv && } +
{varName}
{ - !node && ( + !node && !isEnv && ( ) } @@ -104,7 +109,7 @@ const WorkflowVariableBlockComponent = ({
) - if (!node) { + if (!node && !isEnv) { return ( {Item} diff --git a/web/app/components/workflow/constants.ts b/web/app/components/workflow/constants.ts index aa4545cefb6f85..46d12764f1b8ac 100644 --- a/web/app/components/workflow/constants.ts +++ b/web/app/components/workflow/constants.ts @@ -396,3 +396,4 @@ export const PARAMETER_EXTRACTOR_COMMON_STRUCT: Var[] = [ export const WORKFLOW_DATA_UPDATE = 'WORKFLOW_DATA_UPDATE' export const CUSTOM_NODE = 'custom' +export const DSL_EXPORT_CHECK = 'DSL_EXPORT_CHECK' diff --git a/web/app/components/workflow/dsl-export-confirm-modal.tsx b/web/app/components/workflow/dsl-export-confirm-modal.tsx new file mode 100644 index 00000000000000..69bf1abfe5dbc8 --- /dev/null +++ b/web/app/components/workflow/dsl-export-confirm-modal.tsx @@ -0,0 +1,85 @@ +'use client' +import React, { useState } from 'react' +import { useTranslation } from 'react-i18next' +import { RiCloseLine, RiLock2Line } from '@remixicon/react' +import cn from '@/utils/classnames' +import { Env } from '@/app/components/base/icons/src/vender/line/others' +import Modal from '@/app/components/base/modal' +import Checkbox from '@/app/components/base/checkbox' +import Button from '@/app/components/base/button' +import type { EnvironmentVariable } from '@/app/components/workflow/types' + +export type DSLExportConfirmModalProps = { + envList: EnvironmentVariable[] + onConfirm: (state: boolean) => void + onClose: () => void +} + +const DSLExportConfirmModal = ({ + envList = [], + onConfirm, + onClose, +}: DSLExportConfirmModalProps) => { + const { t } = useTranslation() + + const [exportSecrets, setExportSecrets] = useState(false) + + const submit = () => { + onConfirm(exportSecrets) + onClose() + } + + return ( + { }} + className={cn('max-w-[480px] w-[480px]')} + > +
{t('workflow.env.export.title')}
+
+ +
+
+ + + + + + + + + {envList.map((env, index) => ( + + + + + ))} + +
NAMEVALUE
+
+ +
{env.name}
+
Secret
+ +
+ 		+
{env.value}
+
+
+
+ setExportSecrets(!exportSecrets)} + /> +
setExportSecrets(!exportSecrets)}>{t('workflow.env.export.checkbox')}
+
+
+ + +
+ + ) +} + +export default DSLExportConfirmModal diff --git a/web/app/components/workflow/header/checklist.tsx b/web/app/components/workflow/header/checklist.tsx index ee2876acb69987..7de9cfa2f48ce0 100644 --- a/web/app/components/workflow/header/checklist.tsx +++ b/web/app/components/workflow/header/checklist.tsx @@ -57,22 +57,15 @@ const WorkflowChecklist = ({ !disabled && setOpen(v => !v)}>
{ diff --git a/web/app/components/workflow/header/env-button.tsx b/web/app/components/workflow/header/env-button.tsx new file mode 100644 index 00000000000000..f9327397164f74 --- /dev/null +++ b/web/app/components/workflow/header/env-button.tsx @@ -0,0 +1,22 @@ +import { memo } from 'react' +import { Env } from '@/app/components/base/icons/src/vender/line/others' +import { useStore } from '@/app/components/workflow/store' +import cn from '@/utils/classnames' + +const EnvButton = () => { + const setShowEnvPanel = useStore(s => s.setShowEnvPanel) + const setShowDebugAndPreviewPanel = useStore(s => s.setShowDebugAndPreviewPanel) + + const handleClick = () => { + setShowEnvPanel(true) + setShowDebugAndPreviewPanel(false) + } + + return ( +
+ +
+ ) +} + +export default memo(EnvButton) diff --git a/web/app/components/workflow/header/index.tsx b/web/app/components/workflow/header/index.tsx index 1623eb6196f311..75d5b29a834698 100644 --- a/web/app/components/workflow/header/index.tsx +++ b/web/app/components/workflow/header/index.tsx @@ -4,6 +4,7 @@ import { useCallback, useMemo, } from 'react' +import { RiApps2AddLine } from '@remixicon/react' import { useNodes } from 'reactflow' import { useTranslation } from 'react-i18next' import { useContext } from 'use-context-selector' @@ -30,8 +31,7 @@ import EditingTitle from './editing-title' import RunningTitle from './running-title' import RestoringTitle from './restoring-title' import ViewHistory from './view-history' -import Checklist from './checklist' -import { Grid01 } from '@/app/components/base/icons/src/vender/line/layout' +import EnvButton from './env-button' import Button from '@/app/components/base/button' import { useStore as useAppStore } from '@/app/components/app/store' import { publishWorkflow } from '@/service/workflow' @@ -44,10 +44,7 @@ const Header: FC = () => { const appDetail = useAppStore(s => s.appDetail) const appSidebarExpand = useAppStore(s => s.appSidebarExpand) const appID = appDetail?.id - const { - nodesReadOnly, - getNodesReadOnly, - } = useNodesReadOnly() + const { getNodesReadOnly } = useNodesReadOnly() const publishedAt = useStore(s => s.publishedAt) const draftUpdatedAt = useStore(s => s.draftUpdatedAt) const toolPublished = useStore(s => s.toolPublished) @@ -167,14 +164,12 @@ const Header: FC = () => {
{ normal && ( -
+
+ +
-
- { onPublish, onRestore: onStartRestoring, onToggle: onPublisherToggle, - crossAxisOffset: 53, + crossAxisOffset: 4, }} /> -
-
) } @@ -215,10 +208,8 @@ const Header: FC = () => { { restoring && (
-
diff --git a/web/app/components/workflow/header/run-and-history.tsx b/web/app/components/workflow/header/run-and-history.tsx index bb3982ed32c8b8..047104912c358e 100644 --- a/web/app/components/workflow/header/run-and-history.tsx +++ b/web/app/components/workflow/header/run-and-history.tsx @@ -3,21 +3,22 @@ import { memo } from 'react' import { useTranslation } from 'react-i18next' import { RiLoader2Line, - RiPlayLargeFill, + RiPlayLargeLine, } from '@remixicon/react' import { useStore } from '../store' import { useIsChatMode, + useNodesReadOnly, useWorkflowRun, useWorkflowStartRun, } from '../hooks' import { WorkflowRunningStatus } from '../types' import ViewHistory from './view-history' +import Checklist from './checklist' import cn from '@/utils/classnames' import { StopCircle, } from '@/app/components/base/icons/src/vender/line/mediaAndDevices' -import { MessagePlay } from '@/app/components/base/icons/src/vender/line/communication' const RunMode = memo(() => { const { t } = useTranslation() @@ -30,9 +31,9 @@ const RunMode = memo(() => { <>
handleWorkflowStartRunInWorkflow()} > @@ -46,7 +47,7 @@ const RunMode = memo(() => { ) : ( <> - + {t('workflow.common.run')} ) @@ -58,7 +59,7 @@ const RunMode = memo(() => { className='flex items-center justify-center ml-0.5 w-7 h-7 cursor-pointer hover:bg-black/5 rounded-md' onClick={() => handleStopRun(workflowRunningData?.task_id || '')} > - +
) } @@ -74,12 +75,12 @@ const PreviewMode = memo(() => { return (
handleWorkflowStartRunInChatflow()} > - + {t('workflow.common.debugAndPreview')}
) @@ -88,17 +89,19 @@ PreviewMode.displayName = 'PreviewMode' const RunAndHistory: FC = () => { const isChatMode = useIsChatMode() + const { nodesReadOnly } = useNodesReadOnly() return ( -
+
{ !isChatMode && } { isChatMode && } -
+
+
) } diff --git a/web/app/components/workflow/header/view-history.tsx b/web/app/components/workflow/header/view-history.tsx index 44d572557f0295..6711ff2589e57a 100644 --- a/web/app/components/workflow/header/view-history.tsx +++ b/web/app/components/workflow/header/view-history.tsx @@ -103,16 +103,13 @@ const ViewHistory = ({ popupContent={t('workflow.common.viewRunHistory')} >
{ setCurrentLogItem() setShowMessageLogModal(false) }} > - +
) @@ -170,6 +167,7 @@ const ViewHistory = ({ workflowStore.setState({ historyWorkflowData: item, showInputsPanel: false, + showEnvPanel: false, }) handleBackupDraft() setOpen(false) diff --git a/web/app/components/workflow/hooks/index.ts b/web/app/components/workflow/hooks/index.ts index e355aece39d520..b2aa958025e26a 100644 --- a/web/app/components/workflow/hooks/index.ts +++ b/web/app/components/workflow/hooks/index.ts @@ -14,3 +14,4 @@ export * from './use-panel-interactions' export * from './use-workflow-start-run' export * from './use-nodes-layout' export * from './use-workflow-history' +export * from './use-workflow-variables' diff --git a/web/app/components/workflow/hooks/use-nodes-sync-draft.ts b/web/app/components/workflow/hooks/use-nodes-sync-draft.ts index 05a4dccfded823..78e065329508cf 100644 --- a/web/app/components/workflow/hooks/use-nodes-sync-draft.ts +++ b/web/app/components/workflow/hooks/use-nodes-sync-draft.ts @@ -31,6 +31,7 @@ export const useNodesSyncDraft = () => { const [x, y, zoom] = transform const { appId, + environmentVariables, syncWorkflowDraftHash, } = workflowStore.getState() @@ -80,6 +81,7 @@ export const useNodesSyncDraft = () => { sensitive_word_avoidance: features.moderation, file_upload: features.file, }, + environment_variables: environmentVariables, hash: syncWorkflowDraftHash, }, } diff --git a/web/app/components/workflow/hooks/use-workflow-interactions.ts b/web/app/components/workflow/hooks/use-workflow-interactions.ts index 88299af712be80..dd54ec7401a8f6 100644 --- a/web/app/components/workflow/hooks/use-workflow-interactions.ts +++ b/web/app/components/workflow/hooks/use-workflow-interactions.ts @@ -5,7 +5,7 @@ import { import { useTranslation } from 'react-i18next' import { useReactFlow } from 'reactflow' import { useWorkflowStore } from '../store' -import { WORKFLOW_DATA_UPDATE } from '../constants' +import { DSL_EXPORT_CHECK, WORKFLOW_DATA_UPDATE } from '../constants' import type { WorkflowDataUpdator } from '../types' import { initialEdges, @@ -66,11 +66,18 @@ export const useWorkflowUpdate = () => { appId, setSyncWorkflowDraftHash, setIsSyncingWorkflowDraft, + setEnvironmentVariables, + setEnvSecrets, } = workflowStore.getState() setIsSyncingWorkflowDraft(true) fetchWorkflowDraft(`/apps/${appId}/workflows/draft`).then((response) => { handleUpdateWorkflowCanvas(response.graph as WorkflowDataUpdator) setSyncWorkflowDraftHash(response.hash) + setEnvSecrets((response.environment_variables || []).filter(env => env.value_type === 'secret').reduce((acc, env) => { + acc[env.id] = env.value + return acc + }, {} as Record)) + setEnvironmentVariables(response.environment_variables?.map(env => env.value_type === 'secret' ? { ...env, value: '[__HIDDEN__]' } : env) || []) }).finally(() => setIsSyncingWorkflowDraft(false)) }, [handleUpdateWorkflowCanvas, workflowStore]) @@ -83,12 +90,13 @@ export const useWorkflowUpdate = () => { export const useDSL = () => { const { t } = useTranslation() const { notify } = useToastContext() + const { eventEmitter } = useEventEmitterContextContext() const [exporting, setExporting] = useState(false) const { doSyncWorkflowDraft } = useNodesSyncDraft() const appDetail = useAppStore(s => s.appDetail) - const handleExportDSL = useCallback(async () => { + const handleExportDSL = useCallback(async (include = false) => { if (!appDetail) return @@ -98,7 +106,10 @@ export const useDSL = () => { try { setExporting(true) await doSyncWorkflowDraft() - const { data } = await exportAppConfig(appDetail.id) + const { data } = await exportAppConfig({ + appID: appDetail.id, + include, + }) const a = document.createElement('a') const file = new Blob([data], { type: 'application/yaml' }) a.href = URL.createObjectURL(file) @@ -113,7 +124,30 @@ export const useDSL = () => { } }, [appDetail, notify, t, doSyncWorkflowDraft, exporting]) + const exportCheck = useCallback(async () => { + if (!appDetail) + return + try { + const workflowDraft = await fetchWorkflowDraft(`/apps/${appDetail?.id}/workflows/draft`) + const list = (workflowDraft.environment_variables || []).filter(env => env.value_type === 'secret') + if (list.length === 0) { + handleExportDSL() + return + } + eventEmitter?.emit({ + type: DSL_EXPORT_CHECK, + payload: { + data: list, + }, + } as any) + } + catch (e) { + notify({ type: 'error', message: t('app.exportFailed') }) + } + }, [appDetail, eventEmitter, handleExportDSL, notify, t]) + return { + exportCheck, handleExportDSL, } } diff --git a/web/app/components/workflow/hooks/use-workflow-run.ts b/web/app/components/workflow/hooks/use-workflow-run.ts index 8a7c15055699eb..96f6557fe08a2d 100644 --- a/web/app/components/workflow/hooks/use-workflow-run.ts +++ b/web/app/components/workflow/hooks/use-workflow-run.ts @@ -41,6 +41,7 @@ export const useWorkflowRun = () => { const { backupDraft, setBackupDraft, + environmentVariables, } = workflowStore.getState() const { features } = featuresStore!.getState() @@ -50,6 +51,7 @@ export const useWorkflowRun = () => { edges, viewport: getViewport(), features, + environmentVariables, }) doSyncWorkflowDraft() } @@ -59,6 +61,7 @@ export const useWorkflowRun = () => { const { backupDraft, setBackupDraft, + setEnvironmentVariables, } = workflowStore.getState() if (backupDraft) { @@ -67,12 +70,14 @@ export const useWorkflowRun = () => { edges, viewport, features, + environmentVariables, } = backupDraft handleUpdateWorkflowCanvas({ nodes, edges, viewport, }) + setEnvironmentVariables(environmentVariables) featuresStore!.setState({ features }) setBackupDraft(undefined) } @@ -522,6 +527,7 @@ export const useWorkflowRun = () => { }) featuresStore?.setState({ features: publishedWorkflow.features }) workflowStore.getState().setPublishedAt(publishedWorkflow.created_at) + workflowStore.getState().setEnvironmentVariables(publishedWorkflow.environment_variables || []) } }, [featuresStore, handleUpdateWorkflowCanvas, workflowStore]) diff --git a/web/app/components/workflow/hooks/use-workflow-start-run.tsx b/web/app/components/workflow/hooks/use-workflow-start-run.tsx index f80191cc2d3345..19dd94cf510633 100644 --- a/web/app/components/workflow/hooks/use-workflow-start-run.tsx +++ b/web/app/components/workflow/hooks/use-workflow-start-run.tsx @@ -39,8 +39,11 @@ export const useWorkflowStartRun = () => { showDebugAndPreviewPanel, setShowDebugAndPreviewPanel, setShowInputsPanel, + setShowEnvPanel, } = workflowStore.getState() + setShowEnvPanel(false) + if (showDebugAndPreviewPanel) { handleCancelDebugAndPreviewPanel() return @@ -63,8 +66,11 @@ export const useWorkflowStartRun = () => { showDebugAndPreviewPanel, setShowDebugAndPreviewPanel, setHistoryWorkflowData, + setShowEnvPanel, } = workflowStore.getState() + setShowEnvPanel(false) + if (showDebugAndPreviewPanel) handleCancelDebugAndPreviewPanel() else diff --git a/web/app/components/workflow/hooks/use-workflow-variables.ts b/web/app/components/workflow/hooks/use-workflow-variables.ts new file mode 100644 index 00000000000000..081e8e624280f2 --- /dev/null +++ b/web/app/components/workflow/hooks/use-workflow-variables.ts @@ -0,0 +1,69 @@ +import { useCallback } from 'react' +import { useTranslation } from 'react-i18next' +import { useStore } from '../store' +import { getVarType, toNodeAvailableVars } from '@/app/components/workflow/nodes/_base/components/variable/utils' +import type { + Node, + NodeOutPutVar, + ValueSelector, + Var, +} from '@/app/components/workflow/types' + +export const useWorkflowVariables = () => { + const { t } = useTranslation() + const environmentVariables = useStore(s => s.environmentVariables) + + const getNodeAvailableVars = useCallback(({ + parentNode, + beforeNodes, + isChatMode, + filterVar, + hideEnv, + }: { + parentNode?: Node | null + beforeNodes: Node[] + isChatMode: boolean + filterVar: (payload: Var, selector: ValueSelector) => boolean + hideEnv?: boolean + }): NodeOutPutVar[] => { + return toNodeAvailableVars({ + parentNode, + t, + beforeNodes, + isChatMode, + environmentVariables: hideEnv ? [] : environmentVariables, + filterVar, + }) + }, [environmentVariables, t]) + + const getCurrentVariableType = useCallback(({ + parentNode, + valueSelector, + isIterationItem, + availableNodes, + isChatMode, + isConstant, + }: { + valueSelector: ValueSelector + parentNode?: Node | null + isIterationItem?: boolean + availableNodes: any[] + isChatMode: boolean + isConstant?: boolean + }) => { + return getVarType({ + parentNode, + valueSelector, + isIterationItem, + availableNodes, + isChatMode, + isConstant, + environmentVariables, + }) + }, [environmentVariables]) + + return { + getNodeAvailableVars, + getCurrentVariableType, + } +} diff --git a/web/app/components/workflow/hooks/use-workflow.ts b/web/app/components/workflow/hooks/use-workflow.ts index effd6d757c8c23..b1c97585099c0f 100644 --- a/web/app/components/workflow/hooks/use-workflow.ts +++ b/web/app/components/workflow/hooks/use-workflow.ts @@ -471,8 +471,14 @@ export const useWorkflowInit = () => { const handleGetInitialWorkflowData = useCallback(async () => { try { const res = await fetchWorkflowDraft(`/apps/${appDetail.id}/workflows/draft`) - setData(res) + workflowStore.setState({ + envSecrets: (res.environment_variables || []).filter(env => env.value_type === 'secret').reduce((acc, env) => { + acc[env.id] = env.value + return acc + }, {} as Record), + environmentVariables: res.environment_variables?.map(env => env.value_type === 'secret' ? { ...env, value: '[__HIDDEN__]' } : env) || [], + }) setSyncWorkflowDraftHash(res.hash) setIsLoading(false) } @@ -491,6 +497,7 @@ export const useWorkflowInit = () => { features: { retriever_resource: { enabled: true }, }, + environment_variables: [], }, }).then((res) => { workflowStore.getState().setDraftUpdatedAt(res.updated_at) diff --git a/web/app/components/workflow/index.tsx b/web/app/components/workflow/index.tsx index aca8935f62b36e..92fed2defa43d0 100644 --- a/web/app/components/workflow/index.tsx +++ b/web/app/components/workflow/index.tsx @@ -7,6 +7,7 @@ import { useEffect, useMemo, useRef, + useState, } from 'react' import { setAutoFreeze } from 'immer' import { @@ -30,6 +31,7 @@ import 'reactflow/dist/style.css' import './style.css' import type { Edge, + EnvironmentVariable, Node, } from './types' import { WorkflowContextProvider } from './context' @@ -62,6 +64,7 @@ import PanelContextmenu from './panel-contextmenu' import NodeContextmenu from './node-contextmenu' import SyncingDataModal from './syncing-data-modal' import UpdateDSLModal from './update-dsl-modal' +import DSLExportConfirmModal from './dsl-export-confirm-modal' import { useStore, useWorkflowStore, @@ -74,6 +77,7 @@ import { } from './utils' import { CUSTOM_NODE, + DSL_EXPORT_CHECK, ITERATION_CHILDREN_Z_INDEX, WORKFLOW_DATA_UPDATE, } from './constants' @@ -114,6 +118,7 @@ const Workflow: FC = memo(({ const nodeAnimation = useStore(s => s.nodeAnimation) const showConfirm = useStore(s => s.showConfirm) const showImportDSLModal = useStore(s => s.showImportDSLModal) + const { setShowConfirm, setControlPromptEditorRerenderKey, @@ -127,6 +132,8 @@ const Workflow: FC = memo(({ const { workflowReadOnly } = useWorkflowReadOnly() const { nodesReadOnly } = useNodesReadOnly() + const [secretEnvList, setSecretEnvList] = useState([]) + const { eventEmitter } = useEventEmitterContextContext() eventEmitter?.useSubscription((v: any) => { @@ -148,6 +155,8 @@ const Workflow: FC = memo(({ setTimeout(() => setControlPromptEditorRerenderKey(Date.now())) } + if (v.type === DSL_EXPORT_CHECK) + setSecretEnvList(v.payload.data as EnvironmentVariable[]) }) useEffect(() => { @@ -330,6 +339,15 @@ const Workflow: FC = memo(({ /> ) } + { + secretEnvList.length > 0 && ( + setSecretEnvList([])} + /> + ) + } { - const { t } = useTranslation() const ref = useRef(null) const showAssignVariablePopup = useStore(s => s.showAssignVariablePopup) const setShowAssignVariablePopup = useStore(s => s.setShowAssignVariablePopup) @@ -38,6 +36,7 @@ const AddVariablePopupWithPosition = ({ const { handleAddVariableInAddVariablePopupWithPosition } = useVariableAssigner() const isChatMode = useIsChatMode() const { getBeforeNodesInSameBranch } = useWorkflow() + const { getNodeAvailableVars } = useWorkflowVariables() const outputType = useMemo(() => { if (!showAssignVariablePopup) @@ -55,9 +54,8 @@ const AddVariablePopupWithPosition = ({ if (!showAssignVariablePopup) return [] - return toNodeAvailableVars({ + return getNodeAvailableVars({ parentNode: showAssignVariablePopup.parentNode, - t, beforeNodes: [ ...getBeforeNodesInSameBranch(showAssignVariablePopup.nodeId), { @@ -65,10 +63,16 @@ const AddVariablePopupWithPosition = ({ data: showAssignVariablePopup.nodeData, } as any, ], + hideEnv: true, isChatMode, filterVar: filterVar(outputType as VarType), }) - }, [getBeforeNodesInSameBranch, isChatMode, showAssignVariablePopup, t, outputType]) + .map(node => ({ + ...node, + vars: node.isStartNode ? node.vars.filter(v => !v.variable.startsWith('sys.')) : node.vars, + })) + .filter(item => item.vars.length > 0) + }, [showAssignVariablePopup, getNodeAvailableVars, getBeforeNodesInSameBranch, isChatMode, outputType]) useClickAway(() => { if (nodeData._holdAddVariablePopup) { diff --git a/web/app/components/workflow/nodes/_base/components/readonly-input-with-select-var.tsx b/web/app/components/workflow/nodes/_base/components/readonly-input-with-select-var.tsx index 46b4c67fff75e6..b805de11586247 100644 --- a/web/app/components/workflow/nodes/_base/components/readonly-input-with-select-var.tsx +++ b/web/app/components/workflow/nodes/_base/components/readonly-input-with-select-var.tsx @@ -5,9 +5,10 @@ import cn from 'classnames' import { useWorkflow } from '../../../hooks' import { BlockEnum } from '../../../types' import { VarBlockIcon } from '../../../block-icon' -import { getNodeInfoById, isSystemVar } from './variable/utils' +import { getNodeInfoById, isENV, isSystemVar } from './variable/utils' import { Line3 } from '@/app/components/base/icons/src/public/common' import { Variable02 } from '@/app/components/base/icons/src/vender/solid/development' +import { Env } from '@/app/components/base/icons/src/vender/line/others' type Props = { nodeId: string value: string @@ -40,25 +41,29 @@ const ReadonlyInputWithSelectVar: FC = ({ const value = vars[index].split('.') const isSystem = isSystemVar(value) + const isEnv = isENV(value) const node = (isSystem ? startNode : getNodeInfoById(availableNodes, value[0]))?.data const varName = `${isSystem ? 'sys.' : ''}${value[value.length - 1]}` return ( {str}
-
-
- + {!isEnv && ( +
+
+ +
+
{node?.title}
+
-
{node?.title}
- -
+ )}
- -
{varName}
+ {!isEnv && } + {isEnv && } +
{varName}
) diff --git a/web/app/components/workflow/nodes/_base/components/variable-tag.tsx b/web/app/components/workflow/nodes/_base/components/variable-tag.tsx index a13e44097f4cbf..a144e7e1e93cfa 100644 --- a/web/app/components/workflow/nodes/_base/components/variable-tag.tsx +++ b/web/app/components/workflow/nodes/_base/components/variable-tag.tsx @@ -10,7 +10,9 @@ import type { import { BlockEnum } from '@/app/components/workflow/types' import { Line3 } from '@/app/components/base/icons/src/public/common' import { Variable02 } from '@/app/components/base/icons/src/vender/solid/development' -import { isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils' +import { Env } from '@/app/components/base/icons/src/vender/line/others' +import { isENV, isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils' +import cn from '@/utils/classnames' type VariableTagProps = { valueSelector: ValueSelector @@ -27,36 +29,40 @@ const VariableTag = ({ return nodes.find(node => node.id === valueSelector[0]) }, [nodes, valueSelector]) + const isEnv = isENV(valueSelector) const variableName = isSystemVar(valueSelector) ? valueSelector.slice(0).join('.') : valueSelector.slice(1).join('.') return (
- { - node && ( - - ) - } -
- {node?.data.title} -
- - + {!isEnv && ( + <> + {node && ( + + )} +
+ {node?.data.title} +
+ + + + )} + {isEnv && }
{variableName}
{ varType && ( -
{capitalize(varType)}
+
{capitalize(varType)}
) }
diff --git a/web/app/components/workflow/nodes/_base/components/variable/utils.ts b/web/app/components/workflow/nodes/_base/components/variable/utils.ts index b06f85ef5bd5ad..07c9a13dacc035 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/utils.ts +++ b/web/app/components/workflow/nodes/_base/components/variable/utils.ts @@ -15,7 +15,7 @@ import type { ParameterExtractorNodeType } from '../../../parameter-extractor/ty import type { IterationNodeType } from '../../../iteration/types' import { BlockEnum, InputVarType, VarType } from '@/app/components/workflow/types' import type { StartNodeType } from '@/app/components/workflow/nodes/start/types' -import type { Node, NodeOutPutVar, ValueSelector, Var } from '@/app/components/workflow/types' +import type { EnvironmentVariable, Node, NodeOutPutVar, ValueSelector, Var } from '@/app/components/workflow/types' import type { VariableAssignerNodeType } from '@/app/components/workflow/nodes/variable-assigner/types' import { HTTP_REQUEST_OUTPUT_STRUCT, @@ -34,6 +34,10 @@ export const isSystemVar = (valueSelector: ValueSelector) => { return valueSelector[0] === 'sys' || valueSelector[1] === 'sys' } +export const isENV = (valueSelector: ValueSelector) => { + return valueSelector[0] === 'env' +} + const inputVarTypeToVarType = (type: InputVarType): VarType => { if (type === InputVarType.number) return VarType.number @@ -59,7 +63,11 @@ const findExceptVarInObject = (obj: any, filterVar: (payload: Var, selector: Val return res } -const formatItem = (item: any, isChatMode: boolean, filterVar: (payload: Var, selector: ValueSelector) => boolean): NodeOutPutVar => { +const formatItem = ( + item: any, + isChatMode: boolean, + filterVar: (payload: Var, selector: ValueSelector) => boolean, +): NodeOutPutVar => { const { id, data } = item const res: NodeOutPutVar = { @@ -226,6 +234,16 @@ const formatItem = (item: any, isChatMode: boolean, filterVar: (payload: Var, se ] break } + + case 'env': { + res.vars = data.envList.map((env: EnvironmentVariable) => { + return { + variable: `env.${env.name}`, + type: env.value_type, + } + }) as Var[] + break + } } const selector = [id] @@ -246,16 +264,30 @@ const formatItem = (item: any, isChatMode: boolean, filterVar: (payload: Var, se return res } -export const toNodeOutputVars = (nodes: any[], isChatMode: boolean, filterVar = (_payload: Var, _selector: ValueSelector) => true): NodeOutPutVar[] => { - const res = nodes - .filter(node => SUPPORT_OUTPUT_VARS_NODE.includes(node.data.type)) - .map((node) => { - return { - ...formatItem(node, isChatMode, filterVar), - isStartNode: node.data.type === BlockEnum.Start, - } - }) - .filter(item => item.vars.length > 0) +export const toNodeOutputVars = ( + nodes: any[], + isChatMode: boolean, + filterVar = (_payload: Var, _selector: ValueSelector) => true, + environmentVariables: EnvironmentVariable[] = [], +): NodeOutPutVar[] => { + // ENV_NODE data format + const ENV_NODE = { + id: 'env', + data: { + title: 'ENVIRONMENT', + type: 'env', + envList: environmentVariables, + }, + } + const res = [ + ...nodes.filter(node => SUPPORT_OUTPUT_VARS_NODE.includes(node.data.type)), + ...(environmentVariables.length > 0 ? [ENV_NODE] : []), + ].map((node) => { + return { + ...formatItem(node, isChatMode, filterVar), + isStartNode: node.data.type === BlockEnum.Start, + } + }).filter(item => item.vars.length > 0) return res } @@ -313,6 +345,7 @@ export const getVarType = ({ availableNodes, isChatMode, isConstant, + environmentVariables = [], }: { valueSelector: ValueSelector @@ -321,11 +354,17 @@ export const getVarType = ({ availableNodes: any[] isChatMode: boolean isConstant?: boolean + environmentVariables?: EnvironmentVariable[] }): VarType => { if (isConstant) return VarType.string - const beforeNodesOutputVars = toNodeOutputVars(availableNodes, isChatMode) + const beforeNodesOutputVars = toNodeOutputVars( + availableNodes, + isChatMode, + undefined, + environmentVariables, + ) const isIterationInnerVar = parentNode?.data.type === BlockEnum.Iteration if (isIterationItem) { @@ -346,6 +385,7 @@ export const getVarType = ({ return VarType.number } const isSystem = isSystemVar(valueSelector) + const isEnv = isENV(valueSelector) const startNode = availableNodes.find((node: any) => { return node.data.type === BlockEnum.Start }) @@ -358,7 +398,7 @@ export const getVarType = ({ let type: VarType = VarType.string let curr: any = targetVar.vars - if (isSystem) { + if (isSystem || isEnv) { return curr.find((v: any) => v.variable === (valueSelector as ValueSelector).join('.'))?.type } else { @@ -383,6 +423,7 @@ export const toNodeAvailableVars = ({ t, beforeNodes, isChatMode, + environmentVariables, filterVar, }: { parentNode?: Node | null @@ -390,9 +431,16 @@ export const toNodeAvailableVars = ({ // to get those nodes output vars beforeNodes: Node[] isChatMode: boolean + // env + environmentVariables?: EnvironmentVariable[] filterVar: (payload: Var, selector: ValueSelector) => boolean }): NodeOutPutVar[] => { - const beforeNodesOutputVars = toNodeOutputVars(beforeNodes, isChatMode, filterVar) + const beforeNodesOutputVars = toNodeOutputVars( + beforeNodes, + isChatMode, + filterVar, + environmentVariables, + ) const isInIteration = parentNode?.data.type === BlockEnum.Iteration if (isInIteration) { const iterationNode: any = parentNode @@ -402,6 +450,7 @@ export const toNodeAvailableVars = ({ valueSelector: iterationNode?.data.iterator_selector || [], availableNodes: beforeNodes, isChatMode, + environmentVariables, }) const iterationVar = { nodeId: iterationNode?.id, @@ -493,7 +542,7 @@ export const getNodeUsedVars = (node: Node): ValueSelector[] => { case BlockEnum.IfElse: { res = (data as IfElseNodeType).conditions?.map((c) => { return c.variable_selector - }) + }) || [] break } case BlockEnum.Code: { diff --git a/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx b/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx index c868da85408053..1e041ba5b167ee 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx +++ b/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx @@ -9,12 +9,13 @@ import { import produce from 'immer' import { useStoreApi } from 'reactflow' import VarReferencePopup from './var-reference-popup' -import { getNodeInfoById, getVarType, isSystemVar, toNodeAvailableVars } from './utils' +import { getNodeInfoById, isENV, isSystemVar } from './utils' import cn from '@/utils/classnames' import type { Node, NodeOutPutVar, ValueSelector, Var } from '@/app/components/workflow/types' import { BlockEnum } from '@/app/components/workflow/types' import { VarBlockIcon } from '@/app/components/workflow/block-icon' import { Line3 } from '@/app/components/base/icons/src/public/common' +import { Env } from '@/app/components/base/icons/src/vender/line/others' import { Variable02 } from '@/app/components/base/icons/src/vender/solid/development' import { PortalToFollowElem, @@ -24,6 +25,7 @@ import { import { useIsChatMode, useWorkflow, + useWorkflowVariables, } from '@/app/components/workflow/hooks' import { VarType as VarKindType } from '@/app/components/workflow/nodes/tool/types' import TypeSelector from '@/app/components/workflow/nodes/_base/components/selector' @@ -71,6 +73,7 @@ const VarReferencePicker: FC = ({ const isChatMode = useIsChatMode() const { getTreeLeafNodes, getBeforeNodesInSameBranch } = useWorkflow() + const { getCurrentVariableType, getNodeAvailableVars } = useWorkflowVariables() const availableNodes = useMemo(() => { return passedInAvailableNodes || (onlyLeafNodeVar ? getTreeLeafNodes(nodeId) : getBeforeNodesInSameBranch(nodeId)) }, [getBeforeNodesInSameBranch, getTreeLeafNodes, nodeId, onlyLeafNodeVar, passedInAvailableNodes]) @@ -97,16 +100,15 @@ const VarReferencePicker: FC = ({ if (availableVars) return availableVars - const vars = toNodeAvailableVars({ + const vars = getNodeAvailableVars({ parentNode: iterationNode, - t, beforeNodes: availableNodes, isChatMode, filterVar, }) return vars - }, [iterationNode, availableNodes, isChatMode, filterVar, availableVars, t]) + }, [iterationNode, availableNodes, isChatMode, filterVar, availableVars, getNodeAvailableVars]) const [open, setOpen] = useState(false) useEffect(() => { @@ -201,7 +203,7 @@ const VarReferencePicker: FC = ({ onChange([], varKindType) }, [onChange, varKindType]) - const type = getVarType({ + const type = getCurrentVariableType({ parentNode: iterationNode, valueSelector: value as ValueSelector, availableNodes, @@ -209,6 +211,8 @@ const VarReferencePicker: FC = ({ isConstant: !!isConstant, }) + const isEnv = isENV(value as ValueSelector) + // 8(left/right-padding) + 14(icon) + 4 + 14 + 2 = 42 + 17 buff const availableWidth = triggerWidth - 56 const [maxNodeNameWidth, maxVarNameWidth, maxTypeWidth] = (() => { @@ -276,7 +280,7 @@ const VarReferencePicker: FC = ({ {hasValue ? ( <> - {isShowNodeName && ( + {isShowNodeName && !isEnv && (
= ({ )}
{!hasValue && } -
} +
{varName}
diff --git a/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx b/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx index 893cc2a6e04a27..d6e231c6f22e73 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx +++ b/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx @@ -16,6 +16,7 @@ import { PortalToFollowElemTrigger, } from '@/app/components/base/portal-to-follow-elem' import { XCircle } from '@/app/components/base/icons/src/vender/solid/general' +import { Env } from '@/app/components/base/icons/src/vender/line/others' import { checkKeys } from '@/utils/var' type ObjectChildrenProps = { @@ -48,6 +49,8 @@ const Item: FC = ({ itemWidth, }) => { const isObj = itemData.type === VarType.object && itemData.children && itemData.children.length > 0 + const isSys = itemData.variable.startsWith('sys.') + const isEnv = itemData.variable.startsWith('env.') const itemRef = useRef(null) const [isItemHovering, setIsItemHovering] = useState(false) const _ = useHover(itemRef, { @@ -76,7 +79,7 @@ const Item: FC = ({ }, [isHovering]) const handleChosen = (e: React.MouseEvent) => { e.stopPropagation() - if (itemData.variable.startsWith('sys.')) { // system variable + if (isSys || isEnv) { // system variable or environment variable onChange([...objPath, ...itemData.variable.split('.')], itemData) } else { @@ -101,8 +104,9 @@ const Item: FC = ({ onClick={handleChosen} >
- -
{itemData.variable}
+ {!isEnv && } + {isEnv && } +
{!isEnv ? itemData.variable : itemData.variable.replace('env.', '')}
{itemData.type}
{isObj && ( @@ -205,8 +209,9 @@ const VarReferenceVars: FC = ({ }) => { const { t } = useTranslation() const [searchText, setSearchText] = useState('') + const filteredVars = vars.filter((v) => { - const children = v.vars.filter(v => checkKeys([v.variable], false).isValid || v.variable.startsWith('sys.')) + const children = v.vars.filter(v => checkKeys([v.variable], false).isValid || v.variable.startsWith('sys.') || v.variable.startsWith('env.')) return children.length > 0 }).filter((node) => { if (!searchText) @@ -217,7 +222,7 @@ const VarReferenceVars: FC = ({ }) return children.length > 0 }).map((node) => { - let vars = node.vars.filter(v => checkKeys([v.variable], false).isValid || v.variable.startsWith('sys.')) + let vars = node.vars.filter(v => checkKeys([v.variable], false).isValid || v.variable.startsWith('sys.') || v.variable.startsWith('env.')) if (searchText) { const searchTextLower = searchText.toLowerCase() if (!node.title.toLowerCase().includes(searchTextLower)) @@ -229,6 +234,7 @@ const VarReferenceVars: FC = ({ vars, } }) + const [isFocus, { setFalse: setBlur, setTrue: setFocus, diff --git a/web/app/components/workflow/nodes/_base/hooks/use-available-var-list.ts b/web/app/components/workflow/nodes/_base/hooks/use-available-var-list.ts index 3d5937048fc8b3..ef3d6659102b42 100644 --- a/web/app/components/workflow/nodes/_base/hooks/use-available-var-list.ts +++ b/web/app/components/workflow/nodes/_base/hooks/use-available-var-list.ts @@ -1,10 +1,9 @@ -import { useTranslation } from 'react-i18next' import useNodeInfo from './use-node-info' import { useIsChatMode, useWorkflow, + useWorkflowVariables, } from '@/app/components/workflow/hooks' -import { toNodeAvailableVars } from '@/app/components/workflow/nodes/_base/components/variable/utils' import type { ValueSelector, Var } from '@/app/components/workflow/types' type Params = { onlyLeafNodeVar?: boolean @@ -18,9 +17,8 @@ const useAvailableVarList = (nodeId: string, { onlyLeafNodeVar: false, filterVar: () => true, }) => { - const { t } = useTranslation() - const { getTreeLeafNodes, getBeforeNodesInSameBranch } = useWorkflow() + const { getNodeAvailableVars } = useWorkflowVariables() const isChatMode = useIsChatMode() const availableNodes = onlyLeafNodeVar ? getTreeLeafNodes(nodeId) : getBeforeNodesInSameBranch(nodeId) @@ -29,9 +27,8 @@ const useAvailableVarList = (nodeId: string, { parentNode: iterationNode, } = useNodeInfo(nodeId) - const availableVars = toNodeAvailableVars({ + const availableVars = getNodeAvailableVars({ parentNode: iterationNode, - t, beforeNodes: availableNodes, isChatMode, filterVar, diff --git a/web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts b/web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts index 75fcb7dcc7961c..d14fc939dafd16 100644 --- a/web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts +++ b/web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts @@ -7,7 +7,7 @@ import { useNodeDataUpdate, useWorkflow, } from '@/app/components/workflow/hooks' -import { getNodeInfoById, isSystemVar, toNodeOutputVars } from '@/app/components/workflow/nodes/_base/components/variable/utils' +import { getNodeInfoById, isENV, isSystemVar, toNodeOutputVars } from '@/app/components/workflow/nodes/_base/components/variable/utils' import type { CommonNodeType, InputVar, ValueSelector, Var, Variable } from '@/app/components/workflow/types' import { BlockEnum, InputVarType, NodeRunningStatus, VarType } from '@/app/components/workflow/types' @@ -329,7 +329,7 @@ const useOneStepRun = ({ if (!variables) return [] - const varInputs = variables.map((item) => { + const varInputs = variables.filter(item => !isENV(item.value_selector)).map((item) => { const originalVar = getVar(item.value_selector) if (!originalVar) { return { diff --git a/web/app/components/workflow/nodes/code/use-config.ts b/web/app/components/workflow/nodes/code/use-config.ts index 59327504007a5f..db8e7c3c304fd7 100644 --- a/web/app/components/workflow/nodes/code/use-config.ts +++ b/web/app/components/workflow/nodes/code/use-config.ts @@ -161,7 +161,7 @@ const useConfig = (id: string, payload: CodeNodeType) => { }) const filterVar = useCallback((varPayload: Var) => { - return [VarType.string, VarType.number, VarType.object, VarType.array, VarType.arrayNumber, VarType.arrayString, VarType.arrayObject].includes(varPayload.type) + return [VarType.string, VarType.number, VarType.secret, VarType.object, VarType.array, VarType.arrayNumber, VarType.arrayString, VarType.arrayObject].includes(varPayload.type) }, []) // single run diff --git a/web/app/components/workflow/nodes/end/node.tsx b/web/app/components/workflow/nodes/end/node.tsx index af7e646bee03fd..cfcb2c12912be6 100644 --- a/web/app/components/workflow/nodes/end/node.tsx +++ b/web/app/components/workflow/nodes/end/node.tsx @@ -1,15 +1,18 @@ import type { FC } from 'react' import React from 'react' +import cn from 'classnames' import type { EndNodeType } from './types' import type { NodeProps, Variable } from '@/app/components/workflow/types' -import { getVarType, isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils' +import { isENV, isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils' import { useIsChatMode, useWorkflow, + useWorkflowVariables, } from '@/app/components/workflow/hooks' import { VarBlockIcon } from '@/app/components/workflow/block-icon' import { Line3 } from '@/app/components/base/icons/src/public/common' import { Variable02 } from '@/app/components/base/icons/src/vender/solid/development' +import { Env } from '@/app/components/base/icons/src/vender/line/others' import { BlockEnum } from '@/app/components/workflow/types' const Node: FC> = ({ @@ -18,6 +21,7 @@ const Node: FC> = ({ }) => { const { getBeforeNodesInSameBranch } = useWorkflow() const availableNodes = getBeforeNodesInSameBranch(id) + const { getCurrentVariableType } = useWorkflowVariables() const isChatMode = useIsChatMode() const startNode = availableNodes.find((node: any) => { @@ -39,8 +43,9 @@ const Node: FC> = ({ {filteredOutputs.map(({ value_selector }, index) => { const node = getNode(value_selector[0]) const isSystem = isSystemVar(value_selector) + const isEnv = isENV(value_selector) const varName = isSystem ? `sys.${value_selector[value_selector.length - 1]}` : value_selector[value_selector.length - 1] - const varType = getVarType({ + const varType = getCurrentVariableType({ valueSelector: value_selector, availableNodes, isChatMode, @@ -48,17 +53,22 @@ const Node: FC> = ({ return (
-
- -
-
{node?.data.title}
- + {!isEnv && ( + <> +
+ +
+
{node?.data.title}
+ + + )}
- -
{varName}
+ {!isEnv && } + {isEnv && } +
{varName}
diff --git a/web/app/components/workflow/nodes/http/components/api-input.tsx b/web/app/components/workflow/nodes/http/components/api-input.tsx index b5b9f81214c50b..1f418ac21dbb0d 100644 --- a/web/app/components/workflow/nodes/http/components/api-input.tsx +++ b/web/app/components/workflow/nodes/http/components/api-input.tsx @@ -42,7 +42,7 @@ const ApiInput: FC = ({ const { availableVars, availableNodesWithParent } = useAvailableVarList(nodeId, { onlyLeafNodeVar: false, filterVar: (varPayload: Var) => { - return [VarType.string, VarType.number].includes(varPayload.type) + return [VarType.string, VarType.number, VarType.secret].includes(varPayload.type) }, }) diff --git a/web/app/components/workflow/nodes/http/components/authorization/index.tsx b/web/app/components/workflow/nodes/http/components/authorization/index.tsx index 0d825bf4f0af30..4710c988831e90 100644 --- a/web/app/components/workflow/nodes/http/components/authorization/index.tsx +++ b/web/app/components/workflow/nodes/http/components/authorization/index.tsx @@ -1,17 +1,23 @@ 'use client' import type { FC } from 'react' import { useTranslation } from 'react-i18next' -import React, { useCallback } from 'react' +import React, { useCallback, useState } from 'react' import produce from 'immer' import type { Authorization as AuthorizationPayloadType } from '../../types' import { APIType, AuthorizationType } from '../../types' import RadioGroup from './radio-group' +import useAvailableVarList from '@/app/components/workflow/nodes/_base/hooks/use-available-var-list' +import { VarType } from '@/app/components/workflow/types' +import type { Var } from '@/app/components/workflow/types' import Modal from '@/app/components/base/modal' import Button from '@/app/components/base/button' +import Input from '@/app/components/workflow/nodes/_base/components/input-support-select-var' +import cn from '@/utils/classnames' const i18nPrefix = 'workflow.nodes.http.authorization' type Props = { + nodeId: string payload: AuthorizationPayloadType onChange: (payload: AuthorizationPayloadType) => void isShow: boolean @@ -31,6 +37,7 @@ const Field = ({ title, isRequired, children }: { title: string; isRequired?: bo } const Authorization: FC = ({ + nodeId, payload, onChange, isShow, @@ -38,6 +45,14 @@ const Authorization: FC = ({ }) => { const { t } = useTranslation() + const [isFocus, setIsFocus] = useState(false) + const { availableVars, availableNodesWithParent } = useAvailableVarList(nodeId, { + onlyLeafNodeVar: false, + filterVar: (varPayload: Var) => { + return [VarType.string, VarType.number, VarType.secret].includes(varPayload.type) + }, + }) + const [tempPayload, setTempPayload] = React.useState(payload) const handleAuthTypeChange = useCallback((type: string) => { const newPayload = produce(tempPayload, (draft: AuthorizationPayloadType) => { @@ -80,6 +95,19 @@ const Authorization: FC = ({ } }, [tempPayload, setTempPayload]) + const handleAPIKeyChange = useCallback((str: string) => { + const newPayload = produce(tempPayload, (draft: AuthorizationPayloadType) => { + if (!draft.config) { + draft.config = { + type: APIType.basic, + api_key: '', + } + } + draft.config.api_key = str + }) + setTempPayload(newPayload) + }, [tempPayload, setTempPayload]) + const handleConfirm = useCallback(() => { onChange(tempPayload) onHide() @@ -128,12 +156,19 @@ const Authorization: FC = ({ )} - +
+ +
)} diff --git a/web/app/components/workflow/nodes/http/components/edit-body/index.tsx b/web/app/components/workflow/nodes/http/components/edit-body/index.tsx index bcb9732e4b695a..645bcdcdf93983 100644 --- a/web/app/components/workflow/nodes/http/components/edit-body/index.tsx +++ b/web/app/components/workflow/nodes/http/components/edit-body/index.tsx @@ -44,7 +44,7 @@ const EditBody: FC = ({ const { availableVars, availableNodes } = useAvailableVarList(nodeId, { onlyLeafNodeVar: false, filterVar: (varPayload: Var) => { - return [VarType.string, VarType.number].includes(varPayload.type) + return [VarType.string, VarType.number, VarType.secret].includes(varPayload.type) }, }) diff --git a/web/app/components/workflow/nodes/http/components/key-value/key-value-edit/input-item.tsx b/web/app/components/workflow/nodes/http/components/key-value/key-value-edit/input-item.tsx index 0ba6a6921298fb..16b1674d5450dd 100644 --- a/web/app/components/workflow/nodes/http/components/key-value/key-value-edit/input-item.tsx +++ b/web/app/components/workflow/nodes/http/components/key-value/key-value-edit/input-item.tsx @@ -39,7 +39,7 @@ const InputItem: FC = ({ const { availableVars, availableNodesWithParent } = useAvailableVarList(nodeId, { onlyLeafNodeVar: false, filterVar: (varPayload: Var) => { - return [VarType.string, VarType.number].includes(varPayload.type) + return [VarType.string, VarType.number, VarType.secret].includes(varPayload.type) }, }) diff --git a/web/app/components/workflow/nodes/http/panel.tsx b/web/app/components/workflow/nodes/http/panel.tsx index 6a796bc9ace2ae..4401d6a7903cea 100644 --- a/web/app/components/workflow/nodes/http/panel.tsx +++ b/web/app/components/workflow/nodes/http/panel.tsx @@ -125,6 +125,7 @@ const Panel: FC> = ({
{(isShowAuthorization && !readOnly) && ( { }, [inputs, setInputs]) const filterVar = useCallback((varPayload: Var) => { - return [VarType.string, VarType.number].includes(varPayload.type) + return [VarType.string, VarType.number, VarType.secret].includes(varPayload.type) }, []) // single run diff --git a/web/app/components/workflow/nodes/if-else/components/condition-value.tsx b/web/app/components/workflow/nodes/if-else/components/condition-value.tsx index 904ecc8e81d0c0..eea3c583e5af3f 100644 --- a/web/app/components/workflow/nodes/if-else/components/condition-value.tsx +++ b/web/app/components/workflow/nodes/if-else/components/condition-value.tsx @@ -9,8 +9,9 @@ import { isComparisonOperatorNeedTranslate, } from '../utils' import { Variable02 } from '@/app/components/base/icons/src/vender/solid/development' +import { Env } from '@/app/components/base/icons/src/vender/line/others' import cn from '@/utils/classnames' -import { isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils' +import { isENV, isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils' type ConditionValueProps = { variableSelector: string[] @@ -32,7 +33,7 @@ const ConditionValue = ({ return '' return value.replace(/{{#([^#]*)#}}/g, (a, b) => { - const arr = b.split('.') + const arr: string[] = b.split('.') if (isSystemVar(arr)) return `{{${b}}}` @@ -42,7 +43,8 @@ const ConditionValue = ({ return (
- + {!isENV(variableSelector) && } + {isENV(variableSelector) && }
{ }, [inputs, setInputs]) const filterInputVar = useCallback((varPayload: Var) => { - return [VarType.number, VarType.string].includes(varPayload.type) + return [VarType.number, VarType.string, VarType.secret].includes(varPayload.type) }, []) const filterVar = useCallback((varPayload: Var) => { - return [VarType.arrayObject, VarType.array, VarType.string].includes(varPayload.type) + return [VarType.arrayObject, VarType.array, VarType.number, VarType.string, VarType.secret].includes(varPayload.type) }, []) const { diff --git a/web/app/components/workflow/nodes/tool/components/input-var-list.tsx b/web/app/components/workflow/nodes/tool/components/input-var-list.tsx index 07ba826221a9d9..3f8447a557125b 100644 --- a/web/app/components/workflow/nodes/tool/components/input-var-list.tsx +++ b/web/app/components/workflow/nodes/tool/components/input-var-list.tsx @@ -40,7 +40,7 @@ const InputVarList: FC = ({ const { availableVars, availableNodesWithParent } = useAvailableVarList(nodeId, { onlyLeafNodeVar: false, filterVar: (varPayload: Var) => { - return [VarType.string, VarType.number].includes(varPayload.type) + return [VarType.string, VarType.number, VarType.secret].includes(varPayload.type) }, }) const paramType = (type: string) => { diff --git a/web/app/components/workflow/nodes/variable-assigner/components/node-group-item.tsx b/web/app/components/workflow/nodes/variable-assigner/components/node-group-item.tsx index 337dcd246070b3..5f87dd6fecf3d1 100644 --- a/web/app/components/workflow/nodes/variable-assigner/components/node-group-item.tsx +++ b/web/app/components/workflow/nodes/variable-assigner/components/node-group-item.tsx @@ -19,8 +19,8 @@ import { import { filterVar } from '../utils' import AddVariable from './add-variable' import NodeVariableItem from './node-variable-item' +import { isENV, isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils' import cn from '@/utils/classnames' -import { isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils' const i18nPrefix = 'workflow.nodes.variableAssigner' type GroupItem = { @@ -55,7 +55,7 @@ const NodeGroupItem = ({ const group = item.variableAssignerNodeData.advanced_settings?.groups.find(group => group.groupId === item.targetHandleId) return group?.output_type || '' }, [item.variableAssignerNodeData, item.targetHandleId, groupEnabled]) - const availableVars = getAvailableVars(item.variableAssignerNodeId, item.targetHandleId, filterVar(outputType as VarType)) + const availableVars = getAvailableVars(item.variableAssignerNodeId, item.targetHandleId, filterVar(outputType as VarType), true) const showSelectionBorder = useMemo(() => { if (groupEnabled && enteringNodePayload?.nodeId === item.variableAssignerNodeId) { if (hoveringAssignVariableGroupId) @@ -123,12 +123,14 @@ const NodeGroupItem = ({ { !!item.variables.length && item.variables.map((variable = [], index) => { const isSystem = isSystemVar(variable) + const isEnv = isENV(variable) const node = isSystem ? nodes.find(node => node.data.type === BlockEnum.Start) : nodes.find(node => node.id === variable[0]) const varName = isSystem ? `sys.${variable[variable.length - 1]}` : variable.slice(1).join('.') return ( -
-
- + {!isEnv && ( +
+
+ +
+
{node?.data.title}
+
-
{node?.data.title}
- -
+ )}
- -
{varName}
+ {!isEnv && } + {isEnv && } +
{varName}
) diff --git a/web/app/components/workflow/nodes/variable-assigner/hooks.ts b/web/app/components/workflow/nodes/variable-assigner/hooks.ts index ccc74c8d89ce34..d9ae25416f7141 100644 --- a/web/app/components/workflow/nodes/variable-assigner/hooks.ts +++ b/web/app/components/workflow/nodes/variable-assigner/hooks.ts @@ -3,13 +3,13 @@ import { useNodes, useStoreApi, } from 'reactflow' -import { useTranslation } from 'react-i18next' import { uniqBy } from 'lodash-es' import produce from 'immer' import { useIsChatMode, useNodeDataUpdate, useWorkflow, + useWorkflowVariables, } from '../../hooks' import type { Node, @@ -21,7 +21,6 @@ import type { VarGroupItem, VariableAssignerNodeType, } from './types' -import { toNodeAvailableVars } from '@/app/components/workflow/nodes/_base/components/variable/utils' export const useVariableAssigner = () => { const store = useStoreApi() @@ -123,11 +122,11 @@ export const useVariableAssigner = () => { } export const useGetAvailableVars = () => { - const { t } = useTranslation() const nodes: Node[] = useNodes() const { getBeforeNodesInSameBranchIncludeParent } = useWorkflow() + const { getNodeAvailableVars } = useWorkflowVariables() const isChatMode = useIsChatMode() - const getAvailableVars = useCallback((nodeId: string, handleId: string, filterVar: (v: Var) => boolean) => { + const getAvailableVars = useCallback((nodeId: string, handleId: string, filterVar: (v: Var) => boolean, hideEnv = false) => { const availableNodes: Node[] = [] const currentNode = nodes.find(node => node.id === nodeId)! @@ -138,14 +137,28 @@ export const useGetAvailableVars = () => { availableNodes.push(...beforeNodes) const parentNode = nodes.find(node => node.id === currentNode.parentId) - return toNodeAvailableVars({ + if (hideEnv) { + return getNodeAvailableVars({ + parentNode, + beforeNodes: uniqBy(availableNodes, 'id').filter(node => node.id !== nodeId), + isChatMode, + hideEnv, + filterVar, + }) + .map(node => ({ + ...node, + vars: node.isStartNode ? node.vars.filter(v => !v.variable.startsWith('sys.')) : node.vars, + })) + .filter(item => item.vars.length > 0) + } + + return getNodeAvailableVars({ parentNode, - t, beforeNodes: uniqBy(availableNodes, 'id').filter(node => node.id !== nodeId), isChatMode, filterVar, }) - }, [nodes, t, isChatMode, getBeforeNodesInSameBranchIncludeParent]) + }, [nodes, getBeforeNodesInSameBranchIncludeParent, getNodeAvailableVars, isChatMode]) return getAvailableVars } diff --git a/web/app/components/workflow/nodes/variable-assigner/panel.tsx b/web/app/components/workflow/nodes/variable-assigner/panel.tsx index f94303a6eafacd..6152e0f5b822c5 100644 --- a/web/app/components/workflow/nodes/variable-assigner/panel.tsx +++ b/web/app/components/workflow/nodes/variable-assigner/panel.tsx @@ -51,7 +51,7 @@ const Panel: FC> = ({ }} onChange={handleListOrTypeChange} groupEnabled={false} - availableVars={getAvailableVars(id, 'target', filterVar(inputs.output_type))} + availableVars={getAvailableVars(id, 'target', filterVar(inputs.output_type), true)} /> ) : (
@@ -67,7 +67,7 @@ const Panel: FC> = ({ canRemove={!readOnly && inputs.advanced_settings?.groups.length > 1} onRemove={handleGroupRemoved(item.groupId)} onGroupNameChange={handleVarGroupNameChange(item.groupId)} - availableVars={getAvailableVars(id, item.groupId, filterVar(item.output_type))} + availableVars={getAvailableVars(id, item.groupId, filterVar(item.output_type), true)} /> {index !== inputs.advanced_settings?.groups.length - 1 && }
diff --git a/web/app/components/workflow/panel-contextmenu.tsx b/web/app/components/workflow/panel-contextmenu.tsx index 0ce9978984c1c9..502967ce2c48c0 100644 --- a/web/app/components/workflow/panel-contextmenu.tsx +++ b/web/app/components/workflow/panel-contextmenu.tsx @@ -26,7 +26,7 @@ const PanelContextmenu = () => { const { handlePaneContextmenuCancel } = usePanelInteractions() const { handleStartWorkflowRun } = useWorkflowStartRun() const { handleAddNote } = useOperator() - const { handleExportDSL } = useDSL() + const { exportCheck } = useDSL() useClickAway(() => { handlePaneContextmenuCancel() @@ -105,7 +105,7 @@ const PanelContextmenu = () => {
handleExportDSL()} + onClick={() => exportCheck()} > {t('app.export')}
diff --git a/web/app/components/workflow/panel/env-panel/index.tsx b/web/app/components/workflow/panel/env-panel/index.tsx new file mode 100644 index 00000000000000..66a3d0524df6a8 --- /dev/null +++ b/web/app/components/workflow/panel/env-panel/index.tsx @@ -0,0 +1,210 @@ +import { + memo, + useCallback, + useState, +} from 'react' +import { capitalize } from 'lodash-es' +import { + useStoreApi, +} from 'reactflow' +import { RiCloseLine, RiDeleteBinLine, RiEditLine, RiLock2Line } from '@remixicon/react' +import { useTranslation } from 'react-i18next' +import { useStore } from '@/app/components/workflow/store' +import { Env } from '@/app/components/base/icons/src/vender/line/others' +import VariableTrigger from '@/app/components/workflow/panel/env-panel/variable-trigger' +import type { + EnvironmentVariable, +} from '@/app/components/workflow/types' +import { findUsedVarNodes, updateNodeVars } from '@/app/components/workflow/nodes/_base/components/variable/utils' +import RemoveEffectVarConfirm from '@/app/components/workflow/nodes/_base/components/remove-effect-var-confirm' +import cn from '@/utils/classnames' +import { useNodesSyncDraft } from '@/app/components/workflow/hooks/use-nodes-sync-draft' + +const EnvPanel = () => { + const { t } = useTranslation() + const store = useStoreApi() + const setShowEnvPanel = useStore(s => s.setShowEnvPanel) + const envList = useStore(s => s.environmentVariables) as EnvironmentVariable[] + const envSecrets = useStore(s => s.envSecrets) + const updateEnvList = useStore(s => s.setEnvironmentVariables) + const setEnvSecrets = useStore(s => s.setEnvSecrets) + const { doSyncWorkflowDraft } = useNodesSyncDraft() + + const [showVariableModal, setShowVariableModal] = useState(false) + const [currentVar, setCurrentVar] = useState() + + const [showRemoveVarConfirm, setShowRemoveConfirm] = useState(false) + const [cacheForDelete, setCacheForDelete] = useState() + + const formatSecret = (s: string) => { + return s.length > 8 ? `${s.slice(0, 6)}************${s.slice(-2)}` : '********************' + } + + const getEffectedNodes = useCallback((env: EnvironmentVariable) => { + const { getNodes } = store.getState() + const allNodes = getNodes() + return findUsedVarNodes( + ['env', env.name], + allNodes, + ) + }, [store]) + + const removeUsedVarInNodes = useCallback((env: EnvironmentVariable) => { + const { getNodes, setNodes } = store.getState() + const effectedNodes = getEffectedNodes(env) + const newNodes = getNodes().map((node) => { + if (effectedNodes.find(n => n.id === node.id)) + return updateNodeVars(node, ['env', env.name], []) + + return node + }) + setNodes(newNodes) + }, [getEffectedNodes, store]) + + const handleDelete = useCallback((env: EnvironmentVariable) => { + removeUsedVarInNodes(env) + updateEnvList(envList.filter(e => e.id !== env.id)) + setCacheForDelete(undefined) + setShowRemoveConfirm(false) + doSyncWorkflowDraft() + if (env.value_type === 'secret') { + const newMap = { ...envSecrets } + delete newMap[env.id] + setEnvSecrets(newMap) + } + }, [doSyncWorkflowDraft, envList, envSecrets, removeUsedVarInNodes, setEnvSecrets, updateEnvList]) + + const deleteCheck = useCallback((env: EnvironmentVariable) => { + const effectedNodes = getEffectedNodes(env) + if (effectedNodes.length > 0) { + setCacheForDelete(env) + setShowRemoveConfirm(true) + } + else { + handleDelete(env) + } + }, [getEffectedNodes, handleDelete]) + + const handleSave = useCallback(async (env: EnvironmentVariable) => { + // add env + let newEnv = env + if (!currentVar) { + if (env.value_type === 'secret') { + setEnvSecrets({ + ...envSecrets, + [env.id]: formatSecret(env.value), + }) + } + const newList = [env, ...envList] + updateEnvList(newList) + await doSyncWorkflowDraft() + updateEnvList(newList.map(e => (e.id === env.id && env.value_type === 'secret') ? { ...e, value: '[__HIDDEN__]' } : e)) + return + } + else if (currentVar.value_type === 'secret') { + if (env.value_type === 'secret') { + if (envSecrets[currentVar.id] !== env.value) { + newEnv = env + setEnvSecrets({ + ...envSecrets, + [env.id]: formatSecret(env.value), + }) + } + else { + newEnv = { ...env, value: '[__HIDDEN__]' } + } + } + } + else { + if (env.value_type === 'secret') { + newEnv = env + setEnvSecrets({ + ...envSecrets, + [env.id]: formatSecret(env.value), + }) + } + } + const newList = envList.map(e => e.id === currentVar.id ? newEnv : e) + updateEnvList(newList) + // side effects of rename env + if (currentVar.name !== env.name) { + const { getNodes, setNodes } = store.getState() + const effectedNodes = getEffectedNodes(currentVar) + const newNodes = getNodes().map((node) => { + if (effectedNodes.find(n => n.id === node.id)) + return updateNodeVars(node, ['env', currentVar.name], ['env', env.name]) + + return node + }) + setNodes(newNodes) + } + await doSyncWorkflowDraft() + updateEnvList(newList.map(e => (e.id === env.id && env.value_type === 'secret') ? { ...e, value: '[__HIDDEN__]' } : e)) + }, [currentVar, doSyncWorkflowDraft, envList, envSecrets, getEffectedNodes, setEnvSecrets, store, updateEnvList]) + + return ( +
+
+ {t('workflow.env.envPanelTitle')} +
+
setShowEnvPanel(false)} + > + +
+
+
+
{t('workflow.env.envDescription')}
+
+ setCurrentVar(undefined)} + /> +
+
+ {envList.map(env => ( +
+
+
+ +
{env.name}
+
{capitalize(env.value_type)}
+ {env.value_type === 'secret' && } +
+
+
+ { + setCurrentVar(env) + setShowVariableModal(true) + }}/> +
+
+ deleteCheck(env)} /> +
+
+
+
{env.value_type === 'secret' ? envSecrets[env.id] : env.value}
+
+ ))} +
+ setShowRemoveConfirm(false)} + onConfirm={() => cacheForDelete && handleDelete(cacheForDelete)} + /> +
+ ) +} + +export default memo(EnvPanel) diff --git a/web/app/components/workflow/panel/env-panel/variable-modal.tsx b/web/app/components/workflow/panel/env-panel/variable-modal.tsx new file mode 100644 index 00000000000000..4a5aff01ecb4df --- /dev/null +++ b/web/app/components/workflow/panel/env-panel/variable-modal.tsx @@ -0,0 +1,151 @@ +import React, { useEffect } from 'react' +import { useTranslation } from 'react-i18next' +import { v4 as uuid4 } from 'uuid' +import { RiCloseLine, RiQuestionLine } from '@remixicon/react' +import { useContext } from 'use-context-selector' +import Button from '@/app/components/base/button' +import TooltipPlus from '@/app/components/base/tooltip-plus' +import { ToastContext } from '@/app/components/base/toast' +import { useStore } from '@/app/components/workflow/store' +import type { EnvironmentVariable } from '@/app/components/workflow/types' +import cn from '@/utils/classnames' + +export type ModalPropsType = { + env?: EnvironmentVariable + onClose: () => void + onSave: (env: EnvironmentVariable) => void +} +const VariableModal = ({ + env, + onClose, + onSave, +}: ModalPropsType) => { + const { t } = useTranslation() + const { notify } = useContext(ToastContext) + const envList = useStore(s => s.environmentVariables) + const envSecrets = useStore(s => s.envSecrets) + const [type, setType] = React.useState<'string' | 'number' | 'secret'>('string') + const [name, setName] = React.useState('') + const [value, setValue] = React.useState() + + const handleNameChange = (v: string) => { + if (!v) + return setName('') + if (!/^[a-zA-Z0-9_]+$/.test(v)) + return notify({ type: 'error', message: 'name is can only contain letters, numbers and underscores' }) + if (/^[0-9]/.test(v)) + return notify({ type: 'error', message: 'name can not start with a number' }) + setName(v) + } + + const handleSave = () => { + if (!name) + return notify({ type: 'error', message: 'name can not be empty' }) + if (!value) + return notify({ type: 'error', message: 'value can not be empty' }) + if (!env && envList.some(env => env.name === name)) + return notify({ type: 'error', message: 'name is existed' }) + onSave({ + id: env ? env.id : uuid4(), + value_type: type, + name, + value: type === 'number' ? Number(value) : value, + }) + onClose() + } + + useEffect(() => { + if (env) { + setType(env.value_type) + setName(env.name) + setValue(env.value_type === 'secret' ? envSecrets[env.id] : env.value) + } + }, [env, envSecrets]) + + return ( +
+
+ {!env ? t('workflow.env.modal.title') : t('workflow.env.modal.editTitle')} +
+
+ +
+
+
+
+ {/* type */} +
+
{t('workflow.env.modal.type')}
+
+
setType('string')}>String
+
{ + setType('number') + if (!(/^[0-9]$/).test(value)) + setValue('') + }}>Number
+
setType('secret')}> + Secret + + {t('workflow.env.modal.secretTip')} +
+ }> + + +
+
+
+ {/* name */} +
+
{t('workflow.env.modal.name')}
+
+ handleNameChange(e.target.value)} + type='text' + /> +
+
+ {/* value */} +
+
{t('workflow.env.modal.value')}
+
+ setValue(e.target.value)} + type={type !== 'number' ? 'text' : 'number'} + /> +
+
+
+
+
+ + +
+
+
+ ) +} + +export default VariableModal diff --git a/web/app/components/workflow/panel/env-panel/variable-trigger.tsx b/web/app/components/workflow/panel/env-panel/variable-trigger.tsx new file mode 100644 index 00000000000000..95706798e5639e --- /dev/null +++ b/web/app/components/workflow/panel/env-panel/variable-trigger.tsx @@ -0,0 +1,68 @@ +'use client' +import React from 'react' +import { useTranslation } from 'react-i18next' +import { RiAddLine } from '@remixicon/react' +import Button from '@/app/components/base/button' +import VariableModal from '@/app/components/workflow/panel/env-panel/variable-modal' +// import cn from '@/utils/classnames' +import { + PortalToFollowElem, + PortalToFollowElemContent, + PortalToFollowElemTrigger, +} from '@/app/components/base/portal-to-follow-elem' +import type { EnvironmentVariable } from '@/app/components/workflow/types' + +type Props = { + open: boolean + setOpen: (value: React.SetStateAction) => void + env?: EnvironmentVariable + onClose: () => void + onSave: (env: EnvironmentVariable) => void +} + +const VariableTrigger = ({ + open, + setOpen, + env, + onClose, + onSave, +}: Props) => { + const { t } = useTranslation() + + return ( + { + setOpen(v => !v) + open && onClose() + }} + placement='left-start' + offset={{ + mainAxis: 8, + alignmentAxis: -104, + }} + > + { + setOpen(v => !v) + open && onClose() + }}> + + + + { + onClose() + setOpen(false) + }} + /> + + + ) +} + +export default VariableTrigger diff --git a/web/app/components/workflow/panel/index.tsx b/web/app/components/workflow/panel/index.tsx index 7983b36615503d..6cd4d4f7b5f52c 100644 --- a/web/app/components/workflow/panel/index.tsx +++ b/web/app/components/workflow/panel/index.tsx @@ -13,6 +13,7 @@ import DebugAndPreview from './debug-and-preview' import Record from './record' import WorkflowPreview from './workflow-preview' import ChatRecord from './chat-record' +import EnvPanel from './env-panel' import cn from '@/utils/classnames' import { useStore as useAppStore } from '@/app/components/app/store' import MessageLogModal from '@/app/components/base/message-log-modal' @@ -23,6 +24,7 @@ const Panel: FC = () => { const selectedNode = nodes.find(node => node.data.selected) const historyWorkflowData = useStore(s => s.historyWorkflowData) const showDebugAndPreviewPanel = useStore(s => s.showDebugAndPreviewPanel) + const showEnvPanel = useStore(s => s.showEnvPanel) const isRestoring = useStore(s => s.isRestoring) const { enableShortcuts, @@ -39,9 +41,7 @@ const Panel: FC = () => { return (
{ ) } + { + showEnvPanel && ( + + ) + }
) } diff --git a/web/app/components/workflow/panel/workflow-preview.tsx b/web/app/components/workflow/panel/workflow-preview.tsx index ca1a8ba59aedc4..7fee3436b2f824 100644 --- a/web/app/components/workflow/panel/workflow-preview.tsx +++ b/web/app/components/workflow/panel/workflow-preview.tsx @@ -30,11 +30,7 @@ import cn from '@/utils/classnames' import Loading from '@/app/components/base/loading' import type { NodeTracing } from '@/types/workflow' -const WorkflowPreview = ({ - onShowIterationDetail, -}: { - onShowIterationDetail: (detail: NodeTracing[][]) => void -}) => { +const WorkflowPreview = () => { const { t } = useTranslation() const { handleCancelDebugAndPreviewPanel } = useWorkflowInteractions() const workflowRunningData = useStore(s => s.workflowRunningData) diff --git a/web/app/components/workflow/store.ts b/web/app/components/workflow/store.ts index 203aa828383263..9df22355dd6d38 100644 --- a/web/app/components/workflow/store.ts +++ b/web/app/components/workflow/store.ts @@ -12,6 +12,7 @@ import type { import type { VariableAssignerNodeType } from './nodes/variable-assigner/types' import type { Edge, + EnvironmentVariable, HistoryWorkflowData, Node, RunFile, @@ -59,6 +60,7 @@ type Shape = { edges: Edge[] viewport: Viewport features: Record + environmentVariables: EnvironmentVariable[] } setBackupDraft: (backupDraft?: Shape['backupDraft']) => void notInitialWorkflow: boolean @@ -82,6 +84,12 @@ type Shape = { setShortcutsDisabled: (shortcutsDisabled: boolean) => void showDebugAndPreviewPanel: boolean setShowDebugAndPreviewPanel: (showDebugAndPreviewPanel: boolean) => void + showEnvPanel: boolean + setShowEnvPanel: (showEnvPanel: boolean) => void + environmentVariables: EnvironmentVariable[] + setEnvironmentVariables: (environmentVariables: EnvironmentVariable[]) => void + envSecrets: Record + setEnvSecrets: (envSecrets: Record) => void selection: null | { x1: number; y1: number; x2: number; y2: number } setSelection: (selection: Shape['selection']) => void bundleNodeSize: { width: number; height: number } | null @@ -190,6 +198,12 @@ export const createWorkflowStore = () => { setShortcutsDisabled: shortcutsDisabled => set(() => ({ shortcutsDisabled })), showDebugAndPreviewPanel: false, setShowDebugAndPreviewPanel: showDebugAndPreviewPanel => set(() => ({ showDebugAndPreviewPanel })), + showEnvPanel: false, + setShowEnvPanel: showEnvPanel => set(() => ({ showEnvPanel })), + environmentVariables: [], + setEnvironmentVariables: environmentVariables => set(() => ({ environmentVariables })), + envSecrets: {}, + setEnvSecrets: envSecrets => set(() => ({ envSecrets })), selection: null, setSelection: selection => set(() => ({ selection })), bundleNodeSize: null, diff --git a/web/app/components/workflow/types.ts b/web/app/components/workflow/types.ts index b9605421678804..8b0e3113bc109e 100644 --- a/web/app/components/workflow/types.ts +++ b/web/app/components/workflow/types.ts @@ -102,6 +102,13 @@ export type Variable = { isParagraph?: boolean } +export type EnvironmentVariable = { + id: string + name: string + value: any + value_type: 'string' | 'number' | 'secret' +} + export type VariableWithValue = { key: string value: string @@ -183,6 +190,7 @@ export type Memory = { export enum VarType { string = 'string', number = 'number', + secret = 'secret', boolean = 'boolean', object = 'object', array = 'array', diff --git a/web/i18n/en-US/workflow.ts b/web/i18n/en-US/workflow.ts index 568823bb3a482d..233ba40450b034 100644 --- a/web/i18n/en-US/workflow.ts +++ b/web/i18n/en-US/workflow.ts @@ -19,7 +19,7 @@ const translation = { goBackToEdit: 'Go back to editor', conversationLog: 'Conversation Log', features: 'Features', - debugAndPreview: 'Debug and Preview', + debugAndPreview: 'Preview', restart: 'Restart', currentDraft: 'Current Draft', currentDraftUnpublished: 'Current Draft Unpublished', @@ -78,6 +78,27 @@ const translation = { importFailure: 'Import failure', importSuccess: 'Import success', }, + env: { + envPanelTitle: 'Environment Variables', + envDescription: 'Environment variables can be used to store private information and credentials. They are read-only and can be separated from the DSL file during export.', + envPanelButton: 'Add Variable', + modal: { + title: 'Add Environment Variable', + editTitle: 'Edit Environment Variable', + type: 'Type', + name: 'Name', + namePlaceholder: 'env name', + value: 'Value', + valuePlaceholder: 'env value', + secretTip: 'Used to define sensitive information or data, with DSL settings configured for leak prevention.', + }, + export: { + title: 'Export Secret environment variables?', + checkbox: 'Export secret values', + ignore: 'Export DSL', + export: 'Export DSL with secret values ', + }, + }, changeHistory: { title: 'Change History', placeholder: 'You haven\'t changed anything yet', diff --git a/web/i18n/zh-Hans/workflow.ts b/web/i18n/zh-Hans/workflow.ts index 2b9af83f6cbc50..648c7c6891b9e0 100644 --- a/web/i18n/zh-Hans/workflow.ts +++ b/web/i18n/zh-Hans/workflow.ts @@ -19,7 +19,7 @@ const translation = { goBackToEdit: '返回编辑模式', conversationLog: '对话记录', features: '功能', - debugAndPreview: '调试和预览', + debugAndPreview: '预览', restart: '重新开始', currentDraft: '当前草稿', currentDraftUnpublished: '当前草稿未发布', @@ -78,6 +78,27 @@ const translation = { importFailure: '导入失败', importSuccess: '导入成功', }, + env: { + envPanelTitle: '环境变量', + envDescription: '环境变量是一种存储敏感信息的方法，如 API 密钥、数据库密码等。它们被存储在工作流程中，而不是代码中，以便在不同环墋中共享。', + envPanelButton: '添加环境变量', + modal: { + title: '添加环境变量', + editTitle: '编辑环境变量', + type: '类型', + name: '名称', + namePlaceholder: '变量名', + value: '值', + valuePlaceholder: '变量值', + secretTip: '用于定义敏感信息或数据，导出 DSL 时设置了防泄露机制。', + }, + export: { + title: '导出 Secret 类型环境变量？', + checkbox: '导出 secret 值', + ignore: '导出 DSL', + export: '导出包含 Secret 值的 DSL', + }, + }, changeHistory: { title: '变更历史', placeholder: '尚未更改任何内容', diff --git a/web/service/apps.ts b/web/service/apps.ts index 5f2e9a3ffb49b2..3c12de5c6fbcdf 100644 --- a/web/service/apps.ts +++ b/web/service/apps.ts @@ -29,8 +29,8 @@ export const copyApp: Fetcher(`apps/${appID}/copy`, { body: { name, icon, icon_background, mode, description } }) } -export const exportAppConfig: Fetcher<{ data: string }, string> = (appID) => { - return get<{ data: string }>(`apps/${appID}/export`) +export const exportAppConfig: Fetcher<{ data: string }, { appID: string; include?: boolean }> = ({ appID, include = false }) => { + return get<{ data: string }>(`apps/${appID}/export?include_secret=${include}`) } export const importApp: Fetcher = ({ data, name, description, icon, icon_background }) => { diff --git a/web/service/workflow.ts b/web/service/workflow.ts index 4a47c999478023..1b805dff4f0b13 100644 --- a/web/service/workflow.ts +++ b/web/service/workflow.ts @@ -13,7 +13,7 @@ export const fetchWorkflowDraft = (url: string) => { return get(url, {}, { silent: true }) as Promise } -export const syncWorkflowDraft = ({ url, params }: { url: string; params: Pick }) => { +export const syncWorkflowDraft = ({ url, params }: { url: string; params: Pick }) => { return post(url, { body: params }, { silent: true }) } diff --git a/web/types/workflow.ts b/web/types/workflow.ts index a24ff8169ef659..35c1bd2791436e 100644 --- a/web/types/workflow.ts +++ b/web/types/workflow.ts @@ -2,6 +2,7 @@ import type { Viewport } from 'reactflow' import type { BlockEnum, Edge, + EnvironmentVariable, Node, } from '@/app/components/workflow/types' @@ -56,6 +57,7 @@ export type FetchWorkflowDraftResponse = { hash: string updated_at: number tool_published: boolean + environment_variables?: EnvironmentVariable[] } export type NodeTracingListResponse = {