diff --git a/wren-engine b/wren-engine index 47ca29ebba..5cbc99123c 160000 --- a/wren-engine +++ b/wren-engine @@ -1 +1 @@ -Subproject commit 47ca29ebba291100ba5d70ce1790f9887eaed7a0 +Subproject commit 5cbc99123ce949939cd328811a8a04f6a4635407 diff --git a/wren-launcher/commands/dbt/converter.go b/wren-launcher/commands/dbt/converter.go index 145adb4dee..e28207850b 100644 --- a/wren-launcher/commands/dbt/converter.go +++ b/wren-launcher/commands/dbt/converter.go @@ -212,6 +212,23 @@ func ConvertDbtProjectCore(opts ConvertOptions) (*ConvertResult, error) { "sslMode": typedDS.SslMode, }, } + case *WrenDorisDataSource: + var host string + if opts.UsedByContainer { + host = handleLocalhostForContainer(typedDS.Host) + } else { + host = typedDS.Host + } + wrenDataSource = map[string]interface{}{ + "type": "doris", + "properties": map[string]interface{}{ + "host": host, + "port": typedDS.Port, + "database": typedDS.Database, + "user": typedDS.User, + "password": typedDS.Password, + }, + } default: pterm.Warning.Printf("Warning: Unsupported data source type: %s\n", ds.GetType()) wrenDataSource = map[string]interface{}{ diff --git a/wren-launcher/commands/dbt/data_source.go b/wren-launcher/commands/dbt/data_source.go index 0903d232ce..8ac690ff1c 100644 --- a/wren-launcher/commands/dbt/data_source.go +++ b/wren-launcher/commands/dbt/data_source.go @@ -96,6 +96,8 @@ func convertConnectionToDataSource(conn DbtConnection, dbtHomePath, profileName, return convertToMSSQLDataSource(conn) case "mysql": return convertToMysqlDataSource(conn) + case "doris": + return convertToDorisDataSource(conn) case "bigquery": // Pass the dbtHomePath to the BigQuery converter return convertToBigQueryDataSource(conn, dbtHomePath) @@ -216,6 +218,25 @@ func convertToMysqlDataSource(conn DbtConnection) (*WrenMysqlDataSource, error) return ds, nil } +func convertToDorisDataSource(conn DbtConnection) (*WrenDorisDataSource, error) { + pterm.Info.Printf("Converting Doris data source: %s:%d/%s\n", conn.Host, conn.Port, conn.Database) + + port := strconv.Itoa(conn.Port) + if conn.Port == 0 { + port = "9030" + } + + ds := &WrenDorisDataSource{ + Host: conn.Host, + Port: port, + Database: conn.Database, + User: conn.User, + Password: conn.Password, + } + + return ds, nil +} + // convertToBigQueryDataSource converts to BigQuery data source func convertToBigQueryDataSource(conn DbtConnection, dbtHomePath string) (*WrenBigQueryDataSource, error) { method := strings.ToLower(strings.TrimSpace(conn.Method)) @@ -549,6 +570,86 @@ func (ds *WrenMysqlDataSource) MapType(sourceType string) string { } } +type WrenDorisDataSource struct { + Database string `json:"database"` + Host string `json:"host"` + Password string `json:"password"` + Port string `json:"port"` + User string `json:"user"` +} + +// GetType implements DataSource interface +func (ds *WrenDorisDataSource) GetType() string { + return "doris" +} + +// Validate implements DataSource interface +func (ds *WrenDorisDataSource) Validate() error { + if ds.Host == "" { + return fmt.Errorf("host cannot be empty") + } + if ds.Database == "" { + return fmt.Errorf("database cannot be empty") + } + if ds.User == "" { + return fmt.Errorf("user cannot be empty") + } + if ds.Port == "" { + return fmt.Errorf("port must be specified") + } + port, err := strconv.Atoi(ds.Port) + if err != nil { + return fmt.Errorf("port must be a valid number") + } + if port <= 0 || port > 65535 { + return fmt.Errorf("port must be between 1 and 65535") + } + return nil +} + +func (ds *WrenDorisDataSource) MapType(sourceType string) string { + sourceType = strings.ToUpper(sourceType) + switch sourceType { + case "CHAR": + return "char" + case "VARCHAR": + return varcharType + case "TEXT", "STRING": + return "text" + case "TINYINT": + return "TINYINT" + case "SMALLINT": + return "SMALLINT" + case "INT", integerSQL: + return "INTEGER" + case "BIGINT", "LARGEINT": + return "BIGINT" + case "FLOAT": + return "FLOAT" + case "DOUBLE": + return "DOUBLE" + case decimalSQL, "NUMERIC": + return decimalSQL + case dateSQL, "DATEV2": + return dateSQL + case datetimeSQL, "DATETIMEV2": + return datetimeSQL + case booleanSQL, "BOOL": + return booleanSQL + case jsonSQL, "JSONB": + return jsonSQL + case "HLL", "BITMAP", "QUANTILE_STATE", "AGG_STATE": + // Doris-specific aggregate types, map to varchar + return varcharType + case "ARRAY", "MAP", "STRUCT", "VARIANT": + // Doris complex types + return jsonSQL + default: + // Return the original type if no mapping is found + return strings.ToLower(sourceType) + } +} + type WrenBigQueryDataSource struct { Project string `json:"project_id"` Dataset string `json:"dataset_id"` diff --git a/wren-ui/public/images/dataSource/doris.svg b/wren-ui/public/images/dataSource/doris.svg new file mode 100644 index 0000000000..246954140f --- /dev/null +++ b/wren-ui/public/images/dataSource/doris.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/wren-ui/src/apollo/client/graphql/__types__.ts b/wren-ui/src/apollo/client/graphql/__types__.ts index a7f054a896..c1b8c782d5 100644 --- a/wren-ui/src/apollo/client/graphql/__types__.ts +++ b/wren-ui/src/apollo/client/graphql/__types__.ts @@ -331,6 +331,7 @@ export enum DataSourceName { BIG_QUERY = 'BIG_QUERY', CLICK_HOUSE = 'CLICK_HOUSE', DATABRICKS = 'DATABRICKS', + DORIS = 'DORIS', DUCKDB = 'DUCKDB', MSSQL = 'MSSQL', MYSQL = 'MYSQL', diff --git a/wren-ui/src/apollo/server/adaptors/ibisAdaptor.ts b/wren-ui/src/apollo/server/adaptors/ibisAdaptor.ts index 0d2d4a9c5e..f1006fba99 100644 --- a/wren-ui/src/apollo/server/adaptors/ibisAdaptor.ts +++ b/wren-ui/src/apollo/server/adaptors/ibisAdaptor.ts @@ -148,6 +148,7 @@ export enum SupportedDataSource { ATHENA = 'ATHENA', REDSHIFT = 'REDSHIFT', DATABRICKS = 'DATABRICKS', + DORIS = 'DORIS', } const dataSourceUrlMap: Record = { @@ -162,6 +163,7 @@ const dataSourceUrlMap: Record = { [SupportedDataSource.ATHENA]: 'athena', [SupportedDataSource.REDSHIFT]: 'redshift', [SupportedDataSource.DATABRICKS]: 'databricks', + [SupportedDataSource.DORIS]: 'doris', }; export interface TableResponse { diff --git a/wren-ui/src/apollo/server/backgrounds/dashboardCacheBackgroundTracker.ts b/wren-ui/src/apollo/server/backgrounds/dashboardCacheBackgroundTracker.ts index baa496b21f..0d29561703 100644 --- a/wren-ui/src/apollo/server/backgrounds/dashboardCacheBackgroundTracker.ts +++ b/wren-ui/src/apollo/server/backgrounds/dashboardCacheBackgroundTracker.ts @@ -104,6 +104,9 @@ export class DashboardCacheBackgroundTracker { // Get project and deployment info const project = await this.projectService.getCurrentProject(); const deployment = await this.deployService.getLastDeployment(project.id); + if (!deployment) { + throw new Error('No deployment found. Please deploy the model first.'); + } const mdl = deployment.manifest; const hash = uuidv4(); diff --git a/wren-ui/src/apollo/server/dataSource.ts b/wren-ui/src/apollo/server/dataSource.ts index b17c3c78a4..34e398bf66 100644 --- a/wren-ui/src/apollo/server/dataSource.ts +++ b/wren-ui/src/apollo/server/dataSource.ts @@ -16,6 +16,7 @@ import { BIG_QUERY_CONNECTION_INFO, DUCKDB_CONNECTION_INFO, MYSQL_CONNECTION_INFO, + DORIS_CONNECTION_INFO, POSTGRES_CONNECTION_INFO, MS_SQL_CONNECTION_INFO, WREN_AI_CONNECTION_INFO, @@ -197,6 +198,29 @@ const dataSource = { HostBasedConnectionInfo >, + // Doris + [DataSourceName.DORIS]: { + sensitiveProps: ['password'], + toIbisConnectionInfo(connectionInfo) { + const decryptedConnectionInfo = decryptConnectionInfo( + DataSourceName.DORIS, + connectionInfo, + ); + const { host, port, database, user, password } = + decryptedConnectionInfo as DORIS_CONNECTION_INFO; + return { + host, + port, + database, + user, + password, + }; + }, + } as IDataSourceConnectionInfo< + DORIS_CONNECTION_INFO, + HostBasedConnectionInfo + >, + // Oracle [DataSourceName.ORACLE]: { sensitiveProps: ['password', 'dsn'], diff --git a/wren-ui/src/apollo/server/mdl/mdlBuilder.ts b/wren-ui/src/apollo/server/mdl/mdlBuilder.ts index 4f7a87671a..f74e8716e3 100644 --- a/wren-ui/src/apollo/server/mdl/mdlBuilder.ts +++ b/wren-ui/src/apollo/server/mdl/mdlBuilder.ts @@ -516,6 +516,8 @@ export class MDLBuilder implements IMDLBuilder { return WrenEngineDataSourceType.REDSHIFT; case DataSourceName.DATABRICKS: return WrenEngineDataSourceType.DATABRICKS; + case DataSourceName.DORIS: + return WrenEngineDataSourceType.DORIS; default: throw new Error( `Unsupported data source type: ${type} found when building manifest`, diff --git a/wren-ui/src/apollo/server/mdl/type.ts b/wren-ui/src/apollo/server/mdl/type.ts index bb7c8be10f..b21ee21430 100644 --- a/wren-ui/src/apollo/server/mdl/type.ts +++ b/wren-ui/src/apollo/server/mdl/type.ts @@ -97,6 +97,7 @@ export enum WrenEngineDataSourceType { DUCKDB = 'DUCKDB', REDSHIFT = 'REDSHIFT', DATABRICKS = 'DATABRICKS', + DORIS = 'DORIS', // accepted by the wren engine, but not supported by the wren ui DATAFUSION = 'DATAFUSION', diff --git a/wren-ui/src/apollo/server/repositories/projectRepository.ts b/wren-ui/src/apollo/server/repositories/projectRepository.ts index 5dd054e273..5ef1c34ab0 100644 --- a/wren-ui/src/apollo/server/repositories/projectRepository.ts +++ b/wren-ui/src/apollo/server/repositories/projectRepository.ts @@ -37,6 +37,14 @@ export interface MYSQL_CONNECTION_INFO { ssl: boolean; } +export interface DORIS_CONNECTION_INFO { + host: string; + port: number; + user: string; + password: string; + database: string; +} + export interface ORACLE_CONNECTION_INFO { user: string; password: string; @@ -149,6 +157,7 @@ export type WREN_AI_CONNECTION_INFO = | BIG_QUERY_CONNECTION_INFO | POSTGRES_CONNECTION_INFO | MYSQL_CONNECTION_INFO + | DORIS_CONNECTION_INFO | ORACLE_CONNECTION_INFO | DUCKDB_CONNECTION_INFO | MS_SQL_CONNECTION_INFO diff --git a/wren-ui/src/apollo/server/resolvers/dashboardResolver.ts b/wren-ui/src/apollo/server/resolvers/dashboardResolver.ts index 8411b5409c..aa7def175f 100644 --- a/wren-ui/src/apollo/server/resolvers/dashboardResolver.ts +++ b/wren-ui/src/apollo/server/resolvers/dashboardResolver.ts @@ -96,6 +96,9 @@ export class DashboardResolver { // query with cache enabled const project = await ctx.projectService.getCurrentProject(); const deployment = await ctx.deployService.getLastDeployment(project.id); + if (!deployment) { + throw new Error('No deployment found. Please deploy the model first.'); + } const mdl = deployment.manifest; await ctx.queryService.preview(response.sql, { project, @@ -163,6 +166,9 @@ export class DashboardResolver { const { cacheEnabled } = await ctx.dashboardService.getCurrentDashboard(); const project = await ctx.projectService.getCurrentProject(); const deployment = await ctx.deployService.getLastDeployment(project.id); + if (!deployment) { + throw new Error('No deployment found. Please deploy the model first.'); + } const mdl = deployment.manifest; const data = (await ctx.queryService.preview(item.detail.sql, { project, diff --git a/wren-ui/src/apollo/server/resolvers/modelResolver.ts b/wren-ui/src/apollo/server/resolvers/modelResolver.ts index 6facf755ff..68091136cd 100644 --- a/wren-ui/src/apollo/server/resolvers/modelResolver.ts +++ b/wren-ui/src/apollo/server/resolvers/modelResolver.ts @@ -812,7 +812,11 @@ export class ModelResolver { // create view const project = await ctx.projectService.getCurrentProject(); - const { manifest } = await ctx.deployService.getLastDeployment(project.id); + const lastDeploy = await ctx.deployService.getLastDeployment(project.id); + if (!lastDeploy) { + throw new Error('No deployment found. Please deploy the model first.'); + } + const { manifest } = lastDeploy; // get sql statement of a response const response = await ctx.askingService.getResponse(responseId); @@ -941,7 +945,11 @@ export class ModelResolver { const project = projectId ? await ctx.projectService.getProjectById(parseInt(projectId)) : await ctx.projectService.getCurrentProject(); - const { manifest } = await ctx.deployService.getLastDeployment(project.id); + const lastDeploy = await ctx.deployService.getLastDeployment(project.id); + if (!lastDeploy) { + throw new Error('No deployment found. Please deploy the model first.'); + } + const { manifest } = lastDeploy; return await ctx.queryService.preview(sql, { project, limit: limit, diff --git a/wren-ui/src/apollo/server/resolvers/sqlPairResolver.ts b/wren-ui/src/apollo/server/resolvers/sqlPairResolver.ts index 4796b7aa9e..b1e6826b9e 100644 --- a/wren-ui/src/apollo/server/resolvers/sqlPairResolver.ts +++ b/wren-ui/src/apollo/server/resolvers/sqlPairResolver.ts @@ -102,6 +102,9 @@ export class SqlPairResolver { const lastDeployment = await ctx.deployService.getLastDeployment( project.id, ); + if (!lastDeployment) { + throw new Error('No deployment found. Please deploy the model first.'); + } const manifest = lastDeployment.manifest; const wrenSQL = await ctx.sqlPairService.modelSubstitute( @@ -114,11 +117,17 @@ export class SqlPairResolver { return safeFormatSQL(wrenSQL, { language: 'postgresql' }) as WrenSQL; } - private async validateSql(sql: string, ctx: IContext) { + private async validateSql(sql: string | undefined, ctx: IContext) { + if (sql == null) { + return; + } const project = await ctx.projectService.getCurrentProject(); const lastDeployment = await ctx.deployService.getLastDeployment( project.id, ); + if (!lastDeployment) { + throw new Error('No deployment found. Please deploy the model first.'); + } const manifest = lastDeployment.manifest; try { await ctx.queryService.preview(sql, { diff --git a/wren-ui/src/apollo/server/schema.ts b/wren-ui/src/apollo/server/schema.ts index 64698ad56d..6592748a98 100644 --- a/wren-ui/src/apollo/server/schema.ts +++ b/wren-ui/src/apollo/server/schema.ts @@ -70,6 +70,7 @@ export const typeDefs = gql` SNOWFLAKE REDSHIFT DATABRICKS + DORIS } enum RedshiftConnectionType { diff --git a/wren-ui/src/apollo/server/services/askingService.ts b/wren-ui/src/apollo/server/services/askingService.ts index 82c9375856..c4751af9c5 100644 --- a/wren-ui/src/apollo/server/services/askingService.ts +++ b/wren-ui/src/apollo/server/services/askingService.ts @@ -933,6 +933,9 @@ export class AskingService implements IAskingService { } const project = await this.projectService.getCurrentProject(); const deployment = await this.deployService.getLastDeployment(project.id); + if (!deployment) { + throw new Error('No deployment found. Please deploy the model first.'); + } const mdl = deployment.manifest; const eventName = TelemetryEvent.HOME_PREVIEW_ANSWER; try { @@ -973,6 +976,9 @@ export class AskingService implements IAskingService { } const project = await this.projectService.getCurrentProject(); const deployment = await this.deployService.getLastDeployment(project.id); + if (!deployment) { + throw new Error('No deployment found. Please deploy the model first.'); + } const mdl = deployment.manifest; const steps = response?.breakdownDetail?.steps; const sql = safeFormatSQL(constructCteSql(steps, stepIndex)); @@ -1000,7 +1006,11 @@ export class AskingService implements IAskingService { input: InstantRecommendedQuestionsInput, ): Promise { const project = await this.projectService.getCurrentProject(); - const { manifest } = await this.deployService.getLastDeployment(project.id); + const lastDeploy = await this.deployService.getLastDeployment(project.id); + if (!lastDeploy) { + throw new Error('No deployment found. Please deploy the model first.'); + } + const { manifest } = lastDeploy; const response = await this.wrenAIAdaptor.generateRecommendationQuestions({ manifest, @@ -1056,6 +1066,9 @@ export class AskingService implements IAskingService { private async getDeployId() { const { id } = await this.projectService.getCurrentProject(); const lastDeploy = await this.deployService.getLastDeployment(id); + if (!lastDeploy) { + throw new Error('No deployment found. Please deploy the model first.'); + } return lastDeploy.hash; } diff --git a/wren-ui/src/apollo/server/types/dataSource.ts b/wren-ui/src/apollo/server/types/dataSource.ts index dfe4489ab5..c8ab483120 100644 --- a/wren-ui/src/apollo/server/types/dataSource.ts +++ b/wren-ui/src/apollo/server/types/dataSource.ts @@ -11,6 +11,7 @@ export enum DataSourceName { ATHENA = 'ATHENA', REDSHIFT = 'REDSHIFT', DATABRICKS = 'DATABRICKS', + DORIS = 'DORIS', } export interface DataSource { diff --git a/wren-ui/src/components/pages/setup/dataSources/DorisProperties.tsx b/wren-ui/src/components/pages/setup/dataSources/DorisProperties.tsx new file mode 100644 index 0000000000..98b4c6f45b --- /dev/null +++ b/wren-ui/src/components/pages/setup/dataSources/DorisProperties.tsx @@ -0,0 +1,84 @@ +import { Form, Input } from 'antd'; +import { ERROR_TEXTS } from '@/utils/error'; +import { FORM_MODE } from '@/utils/enum'; +import { hostValidator } from '@/utils/validator'; + +interface Props { + mode?: FORM_MODE; +} + +export default function DorisProperties(props: Props) { + const { mode } = props; + const isEditMode = mode === FORM_MODE.EDIT; + return ( + <> + + + + + + + + + + + + + + + + + + + + ); +} diff --git a/wren-ui/src/components/pages/setup/utils.tsx b/wren-ui/src/components/pages/setup/utils.tsx index b2462d6d63..8fa7f69b3c 100644 --- a/wren-ui/src/components/pages/setup/utils.tsx +++ b/wren-ui/src/components/pages/setup/utils.tsx @@ -117,6 +117,11 @@ export const DATA_SOURCE_OPTIONS = { guide: 'https://docs.getwren.ai/oss/guide/connect/databricks', disabled: false, }, + [DATA_SOURCES.DORIS]: { + ...getDataSourceConfig(DATA_SOURCES.DORIS), + guide: 'https://docs.getwren.ai/oss/guide/connect/doris', + disabled: false, + }, } as { [key: string]: ButtonOption }; export const TEMPLATE_OPTIONS = { diff --git a/wren-ui/src/utils/dataSourceType.ts b/wren-ui/src/utils/dataSourceType.ts index 813dc1fb6e..66b9688542 100644 --- a/wren-ui/src/utils/dataSourceType.ts +++ b/wren-ui/src/utils/dataSourceType.ts @@ -11,6 +11,7 @@ import SnowflakeProperties from '@/components/pages/setup/dataSources/SnowflakeP import AthenaProperties from '@/components/pages/setup/dataSources/AthenaProperties'; import RedshiftProperties from '@/components/pages/setup/dataSources/RedshiftProperties'; import DatabricksProperties from '@/components/pages/setup/dataSources/DatabricksProperties'; +import DorisProperties from '@/components/pages/setup/dataSources/DorisProperties'; export const getDataSourceImage = (dataSource: DATA_SOURCES | string) => { switch (dataSource) { @@ -38,6 +39,8 @@ export const getDataSourceImage = (dataSource: DATA_SOURCES | string) => { return '/images/dataSource/redshift.svg'; case DATA_SOURCES.DATABRICKS: return '/images/dataSource/databricks.svg'; + case DATA_SOURCES.DORIS: + return '/images/dataSource/doris.svg'; default: return null; } @@ -69,6 +72,8 @@ export const getDataSourceName = (dataSource: DATA_SOURCES | string) => { return 'Redshift'; case DATA_SOURCES.DATABRICKS: return 'Databricks'; + case DATA_SOURCES.DORIS: + return 'Doris'; default: return ''; } @@ -100,6 +105,8 @@ export const getDataSourceProperties = (dataSource: DATA_SOURCES | string) => { return RedshiftProperties; case DATA_SOURCES.DATABRICKS: return DatabricksProperties; + case DATA_SOURCES.DORIS: + return DorisProperties; default: return null; } diff --git a/wren-ui/src/utils/enum/dataSources.ts b/wren-ui/src/utils/enum/dataSources.ts index ed8a608021..84158b8b4a 100644 --- a/wren-ui/src/utils/enum/dataSources.ts +++ b/wren-ui/src/utils/enum/dataSources.ts @@ -16,6 +16,7 @@ export enum DATA_SOURCES { ATHENA = 'ATHENA', REDSHIFT = 'REDSHIFT', DATABRICKS = 'DATABRICKS', + DORIS = 'DORIS', } export enum ATHENA_AUTH_METHOD {