From 61b0429f7f5a1de1ebe531b59d2805b84efac7d9 Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Fri, 1 Aug 2025 17:15:06 -0400 Subject: [PATCH 01/18] Update data_source.go --- wren-launcher/commands/dbt/data_source.go | 76 +++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/wren-launcher/commands/dbt/data_source.go b/wren-launcher/commands/dbt/data_source.go index a12aeac538..07046b4f28 100644 --- a/wren-launcher/commands/dbt/data_source.go +++ b/wren-launcher/commands/dbt/data_source.go @@ -81,6 +81,8 @@ func convertConnectionToDataSource(conn DbtConnection, dbtHomePath, profileName, return convertToLocalFileDataSource(conn, dbtHomePath) case "mysql": return convertToMysqlDataSource(conn) + case "bigquery": + return convertToBigQueryDataSource(conn) default: // For unsupported database types, we can choose to ignore or return error // Here we choose to return nil and log a warning @@ -178,6 +180,29 @@ func convertToMysqlDataSource(conn DbtConnection) (*WrenMysqlDataSource, error) return ds, nil } +// convertToBigQueryDataSource converts to BigQuery data source +func convertToBigQueryDataSource(conn DbtConnection) (*WrenBigQueryDataSource, error) { + // We need to extract the keyfile content, which might be in the 'Additional' map + var keyfileJSON string + if kfj, exists := conn.Additional["keyfile_json"]; exists { + if kfjStr, ok := kfj.(string); ok { + keyfileJSON = kfjStr + } + } + + ds := &WrenBigQueryDataSource{ + Project: conn.Project, + Dataset: conn.Dataset, + Location: conn.Location, + Threads: conn.Threads, + Method: conn.Method, + Keyfile: conn.Keyfile, + KeyfileJSON: keyfileJSON, + Priority: conn.Priority, + } + return ds, nil +} + type WrenLocalFileDataSource struct { Url string `json:"url"` Format string `json:"format"` @@ -341,6 +366,57 @@ func (ds *WrenMysqlDataSource) MapType(sourceType string) string { } } +type WrenBigQueryDataSource struct { + Project string `json:"project"` + Dataset string `json:"dataset"` + Location string `json:"location,omitempty"` + Threads int `json:"threads,omitempty"` + Method string `json:"method"` + Keyfile string `json:"keyfile,omitempty"` + KeyfileJSON string `json:"keyfile_json,omitempty"` + Priority string `json:"priority,omitempty"` +} + +// GetType implements DataSource interface +func (ds *WrenBigQueryDataSource) GetType() string { + return "bigquery" +} + +// Validate implements DataSource interface +func (ds *WrenBigQueryDataSource) Validate() error { + if ds.Project == "" { + return fmt.Errorf("project cannot be empty") + } + if ds.Dataset == "" { + return fmt.Errorf("dataset cannot be empty") + } + + // Validate based on the authentication method + switch ds.Method { + case "service-account": + if ds.Keyfile == "" { + return fmt.Errorf("keyfile cannot be empty for method 'service-account'") + } + case "service-account-json": + if ds.KeyfileJSON == "" { + return fmt.Errorf("keyfile_json cannot be empty for method 'service-account-json'") + } + case "oauth": + return fmt.Errorf("authentication method 'oauth' is not supported; please use a service account method") + default: + return fmt.Errorf("unsupported or missing authentication method: '%s'", ds.Method) + } + + return nil +} + +// MapType implements DataSource interface +func (ds *WrenBigQueryDataSource) MapType(sourceType string) string { + // Add BigQuery specific type mappings here if needed + // For now, we can use the default mapping logic + return sourceType +} + // GetActiveDataSources gets active data sources based on specified profile and target // If profileName is empty, it will use the first found profile // If targetName is empty, it will use the profile's default target From 31bcc4ecdf89d25536159859a82885a3d68674eb Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Fri, 1 Aug 2025 17:18:32 -0400 Subject: [PATCH 02/18] Update data_source.go --- wren-launcher/commands/dbt/data_source.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wren-launcher/commands/dbt/data_source.go b/wren-launcher/commands/dbt/data_source.go index 07046b4f28..715705f299 100644 --- a/wren-launcher/commands/dbt/data_source.go +++ b/wren-launcher/commands/dbt/data_source.go @@ -6,7 +6,7 @@ import ( "strconv" "strings" - "github.com/pterm/pterm" + "github.com.com/pterm/pterm" ) // Constants for data types @@ -401,8 +401,8 @@ func (ds *WrenBigQueryDataSource) Validate() error { if ds.KeyfileJSON == "" { return fmt.Errorf("keyfile_json cannot be empty for method 'service-account-json'") } - case "oauth": - return fmt.Errorf("authentication method 'oauth' is not supported; please use a service account method") + case "oauth", "oauth-secrets": + return fmt.Errorf("authentication method '%s' is not supported; please use a service account method", ds.Method) default: return fmt.Errorf("unsupported or missing authentication method: '%s'", ds.Method) } From 1912f49d624f45c7a685f1b173ec3d878c343d3d Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Fri, 1 Aug 2025 17:22:18 -0400 Subject: [PATCH 03/18] Update profiles.go --- wren-launcher/commands/dbt/profiles.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wren-launcher/commands/dbt/profiles.go b/wren-launcher/commands/dbt/profiles.go index e87822e70b..b8f68edb32 100644 --- a/wren-launcher/commands/dbt/profiles.go +++ b/wren-launcher/commands/dbt/profiles.go @@ -8,7 +8,7 @@ type DbtProfiles struct { // DbtProfile represents a single profile in profiles.yml type DbtProfile struct { - Target string `yaml:"target" json:"target"` + Target string `yaml:"target" json:"target"` Outputs map[string]DbtConnection `yaml:"outputs" json:"outputs"` } @@ -26,13 +26,14 @@ type DbtConnection struct { Project string `yaml:"project,omitempty" json:"project,omitempty"` // BigQuery Dataset string `yaml:"dataset,omitempty" json:"dataset,omitempty"` // BigQuery Keyfile string `yaml:"keyfile,omitempty" json:"keyfile,omitempty"` // BigQuery + Method string `yaml:"method,omitempty" json:"method,omitempty"` // BigQuery Account string `yaml:"account,omitempty" json:"account,omitempty"` // Snowflake Warehouse string `yaml:"warehouse,omitempty" json:"warehouse,omitempty"` // Snowflake Role string `yaml:"role,omitempty" json:"role,omitempty"` // Snowflake KeepAlive bool `yaml:"keepalive,omitempty" json:"keepalive,omitempty"` // Postgres SearchPath string `yaml:"search_path,omitempty" json:"search_path,omitempty"` // Postgres - SSLMode string `yaml:"sslmode,omitempty" json:"sslmode,omitempty"` // Postgres + SSLMode string `yaml:"sslmode,omitempty" json:"sslmode,omitempty"` // Postgres SslDisable bool `yaml:"ssl_disable,omitempty" json:"ssl_disable,omitempty"` // MySQL From 4fc4f962c37871080e345d80f27191ad1a61a6c0 Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Fri, 1 Aug 2025 17:23:04 -0400 Subject: [PATCH 04/18] Update data_source.go --- wren-launcher/commands/dbt/data_source.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/wren-launcher/commands/dbt/data_source.go b/wren-launcher/commands/dbt/data_source.go index 715705f299..06903fb9f8 100644 --- a/wren-launcher/commands/dbt/data_source.go +++ b/wren-launcher/commands/dbt/data_source.go @@ -6,7 +6,7 @@ import ( "strconv" "strings" - "github.com.com/pterm/pterm" + "github.com/pterm/pterm" ) // Constants for data types @@ -193,12 +193,9 @@ func convertToBigQueryDataSource(conn DbtConnection) (*WrenBigQueryDataSource, e ds := &WrenBigQueryDataSource{ Project: conn.Project, Dataset: conn.Dataset, - Location: conn.Location, - Threads: conn.Threads, Method: conn.Method, Keyfile: conn.Keyfile, KeyfileJSON: keyfileJSON, - Priority: conn.Priority, } return ds, nil } @@ -369,12 +366,9 @@ func (ds *WrenMysqlDataSource) MapType(sourceType string) string { type WrenBigQueryDataSource struct { Project string `json:"project"` Dataset string `json:"dataset"` - Location string `json:"location,omitempty"` - Threads int `json:"threads,omitempty"` Method string `json:"method"` Keyfile string `json:"keyfile,omitempty"` KeyfileJSON string `json:"keyfile_json,omitempty"` - Priority string `json:"priority,omitempty"` } // GetType implements DataSource interface From 737c50ccc7d65d9c0c65a3b60264cf52517552e3 Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Fri, 1 Aug 2025 17:23:35 -0400 Subject: [PATCH 05/18] Update converter.go --- wren-launcher/commands/dbt/converter.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/wren-launcher/commands/dbt/converter.go b/wren-launcher/commands/dbt/converter.go index afb10d03d7..783c9d28b4 100644 --- a/wren-launcher/commands/dbt/converter.go +++ b/wren-launcher/commands/dbt/converter.go @@ -154,6 +154,17 @@ func ConvertDbtProjectCore(opts ConvertOptions) (*ConvertResult, error) { "format": typedDS.Format, }, } + case *WrenBigQueryDataSource: + wrenDataSource = map[string]interface{}{ + "type": "bigquery", + "properties": map[string]interface{}{ + "project": typedDS.Project, + "dataset": typedDS.Dataset, + "method": typedDS.Method, + "keyfile": typedDS.Keyfile, + "keyfile_json": typedDS.KeyfileJSON, + }, + } case *WrenMysqlDataSource: wrenDataSource = map[string]interface{}{ "type": "mysql", From 37135982e2f111865bb1cfcc892858e9fc16ecfa Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Fri, 1 Aug 2025 17:28:02 -0400 Subject: [PATCH 06/18] Update data_source_test.go --- .../commands/dbt/data_source_test.go | 169 ++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/wren-launcher/commands/dbt/data_source_test.go b/wren-launcher/commands/dbt/data_source_test.go index 4bdd3e7c1d..2ca12e32c6 100644 --- a/wren-launcher/commands/dbt/data_source_test.go +++ b/wren-launcher/commands/dbt/data_source_test.go @@ -519,3 +519,172 @@ func TestValidateAllDataSources(t *testing.T) { t.Error("ValidateAllDataSources should fail for invalid profiles") } } + +func TestFromDbtProfiles_BigQuery(t *testing.T) { + t.Run("service-account", func(t *testing.T) { + profiles := &DbtProfiles{ + Profiles: map[string]DbtProfile{ + "test_profile": { + Target: "dev", + Outputs: map[string]DbtConnection{ + "dev": { + Type: "bigquery", + Method: "service-account", + Project: "test-project", + Dataset: "test-dataset", + Keyfile: "/path/to/key.json", + }, + }, + }, + }, + } + + dataSources, err := FromDbtProfiles(profiles) + if err != nil { + t.Fatalf("FromDbtProfiles failed: %v", err) + } + + if len(dataSources) != 1 { + t.Fatalf("Expected 1 data source, got %d", len(dataSources)) + } + + ds, ok := dataSources[0].(*WrenBigQueryDataSource) + if !ok { + t.Fatalf("Expected WrenBigQueryDataSource, got %T", dataSources[0]) + } + + if ds.Project != "test-project" { + t.Errorf("Expected project 'test-project', got '%s'", ds.Project) + } + + if ds.Method != "service-account" { + t.Errorf("Expected method 'service-account', got '%s'", ds.Method) + } + + if ds.Keyfile != "/path/to/key.json" { + t.Errorf("Expected keyfile '/path/to/key.json', got '%s'", ds.Keyfile) + } + }) + + t.Run("service-account-json", func(t *testing.T) { + keyfileContent := `{"type": "service_account"}` + profiles := &DbtProfiles{ + Profiles: map[string]DbtProfile{ + "test_profile": { + Target: "dev", + Outputs: map[string]DbtConnection{ + "dev": { + Type: "bigquery", + Method: "service-account-json", + Project: "test-project", + Dataset: "test-dataset", + Additional: map[string]interface{}{ + "keyfile_json": keyfileContent, + }, + }, + }, + }, + }, + } + + dataSources, err := FromDbtProfiles(profiles) + if err != nil { + t.Fatalf("FromDbtProfiles failed: %v", err) + } + + if len(dataSources) != 1 { + t.Fatalf("Expected 1 data source, got %d", len(dataSources)) + } + + ds, ok := dataSources[0].(*WrenBigQueryDataSource) + if !ok { + t.Fatalf("Expected WrenBigQueryDataSource, got %T", dataSources[0]) + } + + if ds.Project != "test-project" { + t.Errorf("Expected project 'test-project', got '%s'", ds.Project) + } + + if ds.Method != "service-account-json" { + t.Errorf("Expected method 'service-account-json', got '%s'", ds.Method) + } + + if ds.KeyfileJSON != keyfileContent { + t.Errorf("Expected keyfile_json content, got '%s'", ds.KeyfileJSON) + } + }) +} + +func TestBigQueryDataSourceValidation(t *testing.T) { + tests := []struct { + name string + ds *WrenBigQueryDataSource + wantErr bool + }{ + { + name: "valid service-account", + ds: &WrenBigQueryDataSource{ + Project: "test-project", + Dataset: "test-dataset", + Method: "service-account", + Keyfile: "/path/to/key.json", + }, + wantErr: false, + }, + { + name: "valid service-account-json", + ds: &WrenBigQueryDataSource{ + Project: "test-project", + Dataset: "test-dataset", + Method: "service-account-json", + KeyfileJSON: `{"type": "service_account"}`, + }, + wantErr: false, + }, + { + name: "invalid - missing project", + ds: &WrenBigQueryDataSource{ + Dataset: "test-dataset", + Method: "service-account", + Keyfile: "/path/to/key.json", + }, + wantErr: true, + }, + { + name: "invalid - missing dataset", + ds: &WrenBigQueryDataSource{ + Project: "test-project", + Method: "service-account", + Keyfile: "/path/to/key.json", + }, + wantErr: true, + }, + { + name: "invalid - unsupported oauth", + ds: &WrenBigQueryDataSource{ + Project: "test-project", + Dataset: "test-dataset", + Method: "oauth", + }, + wantErr: true, + }, + { + name: "invalid - missing keyfile for service-account", + ds: &WrenBigQueryDataSource{ + Project: "test-project", + Dataset: "test-dataset", + Method: "service-account", + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.ds.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} From 5c7616362558dea182ed47a72910525bd825c806 Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Fri, 1 Aug 2025 17:30:10 -0400 Subject: [PATCH 07/18] Update profiles_analyzer.go --- wren-launcher/commands/dbt/profiles_analyzer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wren-launcher/commands/dbt/profiles_analyzer.go b/wren-launcher/commands/dbt/profiles_analyzer.go index 1a0698ba83..f120f4dadf 100644 --- a/wren-launcher/commands/dbt/profiles_analyzer.go +++ b/wren-launcher/commands/dbt/profiles_analyzer.go @@ -134,6 +134,7 @@ func parseConnection(connectionMap map[string]interface{}) (*DbtConnection, erro connection.Project = getString("project") connection.Dataset = getString("dataset") connection.Keyfile = getString("keyfile") + connection.Method = getString("method") connection.Account = getString("account") connection.Warehouse = getString("warehouse") connection.Role = getString("role") @@ -147,7 +148,7 @@ func parseConnection(connectionMap map[string]interface{}) (*DbtConnection, erro knownFields := map[string]bool{ "type": true, "host": true, "port": true, "user": true, "password": true, "database": true, "dbname": true, "schema": true, "project": true, "dataset": true, - "keyfile": true, "account": true, "warehouse": true, "role": true, + "keyfile": true, "method": true, "account": true, "warehouse": true, "role": true, "keepalive": true, "search_path": true, "sslmode": true, "path": true, "ssl_disable": true, } From 3b24f58ae7047db85baf5efd3f9019e178e8ff9a Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Sun, 3 Aug 2025 17:35:30 -0400 Subject: [PATCH 08/18] More WIP --- wren-launcher/commands/dbt.go | 38 +- wren-launcher/commands/dbt/converter.go | 532 ++++++++++++++++++++++-- wren-launcher/commands/dbt/wren_mdl.go | 31 +- wren-launcher/commands/launch.go | 4 +- 4 files changed, 544 insertions(+), 61 deletions(-) diff --git a/wren-launcher/commands/dbt.go b/wren-launcher/commands/dbt.go index f23643cefe..09f96d0d6f 100644 --- a/wren-launcher/commands/dbt.go +++ b/wren-launcher/commands/dbt.go @@ -12,10 +12,11 @@ import ( // then converts them to WrenDataSource and Wren MDL format func DbtAutoConvert() { var opts struct { - ProjectPath string - OutputDir string - ProfileName string - Target string + ProjectPath string + OutputDir string + ProfileName string + Target string + IncludeStagingModels bool } // Define command line flags @@ -23,6 +24,7 @@ func DbtAutoConvert() { flag.StringVar(&opts.OutputDir, "output", "", "Output directory for generated JSON files") flag.StringVar(&opts.ProfileName, "profile", "", "Specific profile name to use (optional, uses first found if not provided)") flag.StringVar(&opts.Target, "target", "", "Specific target to use (optional, uses profile default if not provided)") + flag.BoolVar(&opts.IncludeStagingModels, "include-staging-models", false, "If set, staging models will be included during conversion") flag.Parse() // Validate required parameters @@ -40,11 +42,12 @@ func DbtAutoConvert() { // ConvertOptions struct for core conversion logic convertOpts := dbt.ConvertOptions{ - ProjectPath: opts.ProjectPath, - OutputDir: opts.OutputDir, - ProfileName: opts.ProfileName, - Target: opts.Target, - RequireCatalog: true, // DbtAutoConvert requires catalog.json to exist + ProjectPath: opts.ProjectPath, + OutputDir: opts.OutputDir, + ProfileName: opts.ProfileName, + Target: opts.Target, + RequireCatalog: true, // DbtAutoConvert requires catalog.json to exist + IncludeStagingModels: opts.IncludeStagingModels, } // Call the core conversion logic @@ -57,15 +60,16 @@ func DbtAutoConvert() { // DbtConvertProject is a public wrapper function for processDbtProject to use // It converts a dbt project without requiring catalog.json to exist -func DbtConvertProject(projectPath, outputDir, profileName, target string, usedByContainer bool) (*dbt.ConvertResult, error) { +func DbtConvertProject(projectPath, outputDir, profileName, target string, usedByContainer bool, IncludeStagingModels bool) (*dbt.ConvertResult, error) { convertOpts := dbt.ConvertOptions{ - ProjectPath: projectPath, - OutputDir: outputDir, - ProfileName: profileName, - Target: target, - RequireCatalog: false, // Allow processDbtProject to continue without catalog.json - UsedByContainer: usedByContainer, + ProjectPath: projectPath, + OutputDir: outputDir, + ProfileName: profileName, + Target: target, + RequireCatalog: false, // Allow processDbtProject to continue without catalog.json + UsedByContainer: usedByContainer, + IncludeStagingModels: IncludeStagingModels, } return dbt.ConvertDbtProjectCore(convertOpts) -} +} \ No newline at end of file diff --git a/wren-launcher/commands/dbt/converter.go b/wren-launcher/commands/dbt/converter.go index 783c9d28b4..86494777ec 100644 --- a/wren-launcher/commands/dbt/converter.go +++ b/wren-launcher/commands/dbt/converter.go @@ -5,20 +5,25 @@ import ( "fmt" "os" "path/filepath" + "regexp" "sort" "strings" "github.com/pterm/pterm" ) +// Note: All struct definitions (WrenMDLManifest, WrenModel, etc.) are defined +// in wren_mdl.go to prevent "redeclared in this block" compilation errors. + // ConvertOptions holds the options for dbt project conversion type ConvertOptions struct { - ProjectPath string - OutputDir string - ProfileName string - Target string - RequireCatalog bool // if true, missing catalog.json is an error; if false, it's a warning - UsedByContainer bool // if true, used by container, no need to print usage info + ProjectPath string + OutputDir string + ProfileName string + Target string + RequireCatalog bool // if true, missing catalog.json is an error; if false, it's a warning + UsedByContainer bool // if true, used by container, no need to print usage info + IncludeStagingModels bool // if true, staging models will be included in the conversion } // ConvertResult holds the result of dbt project conversion @@ -51,10 +56,11 @@ func ConvertDbtProjectCore(opts ConvertOptions) (*ConvertResult, error) { pterm.Info.Println("Skipping data source conversion...") } - // Search for catalog.json and manifest.json in target directory + // Search for catalog.json, manifest.json, and semantic_manifest.json in target directory targetDir := filepath.Join(opts.ProjectPath, "target") catalogPath := filepath.Join(targetDir, "catalog.json") manifestPath := filepath.Join(targetDir, "manifest.json") + semanticManifestPath := filepath.Join(targetDir, "semantic_manifest.json") if !FileExists(catalogPath) { if opts.RequireCatalog { @@ -66,14 +72,23 @@ func ConvertDbtProjectCore(opts ConvertOptions) (*ConvertResult, error) { } } - // Check for manifest.json (optional but recommended for descriptions) + // Check for manifest.json (optional but recommended for descriptions and relationships) var manifestPathForConversion string if FileExists(manifestPath) { pterm.Info.Printf("Found manifest.json at: %s\n", manifestPath) manifestPathForConversion = manifestPath } else { pterm.Warning.Printf("Warning: manifest.json not found at: %s\n", manifestPath) - pterm.Info.Println("Model and column descriptions will not be included") + pterm.Info.Println("Model descriptions, column descriptions, and relationships will not be included") + } + + // Check for semantic_manifest.json (optional) + var semanticManifestPathForConversion string + if FileExists(semanticManifestPath) { + pterm.Info.Printf("Found semantic_manifest.json at: %s\n", semanticManifestPath) + semanticManifestPathForConversion = semanticManifestPath + } else { + pterm.Info.Println("semantic_manifest.json not found, skipping metric and primary key conversion.") } // Convert profiles.yml to WrenDataSource (if profiles found) @@ -209,7 +224,7 @@ func ConvertDbtProjectCore(opts ConvertOptions) (*ConvertResult, error) { ds = &DefaultDataSource{} } - manifest, err := ConvertDbtCatalogToWrenMDL(catalogPath, ds, manifestPathForConversion) + manifest, err := ConvertDbtCatalogToWrenMDL(catalogPath, ds, manifestPathForConversion, semanticManifestPathForConversion, opts.IncludeStagingModels) if err != nil { return nil, fmt.Errorf("failed to convert catalog: %w", err) } @@ -230,6 +245,9 @@ func ConvertDbtProjectCore(opts ConvertOptions) (*ConvertResult, error) { // Summary pterm.Success.Println("\nšŸŽ‰ Conversion completed successfully!") pterm.Info.Printf("Models converted: %d\n", len(manifest.Models)) + pterm.Info.Printf("Relationships generated: %d\n", len(manifest.Relationships)) + pterm.Info.Printf("Metrics generated: %d\n", len(manifest.Metrics)) + pterm.Info.Printf("Enums generated: %d\n", len(manifest.EnumDefinitions)) if dataSourceGenerated { pterm.Info.Println("Generated files:") @@ -265,7 +283,7 @@ func handleLocalhostForContainer(host string) string { } // ConvertDbtCatalogToWrenMDL converts dbt catalog.json to Wren MDL format -func ConvertDbtCatalogToWrenMDL(catalogPath string, data_source DataSource, manifestPath string) (*WrenMDLManifest, error) { +func ConvertDbtCatalogToWrenMDL(catalogPath string, dataSource DataSource, manifestPath string, semanticManifestPath string, includeStagingModels bool) (*WrenMDLManifest, error) { // Read and parse the catalog.json file data, err := os.ReadFile(catalogPath) // #nosec G304 -- catalogPath is controlled by application if err != nil { @@ -277,10 +295,10 @@ func ConvertDbtCatalogToWrenMDL(catalogPath string, data_source DataSource, mani return nil, fmt.Errorf("failed to parse catalog JSON: %w", err) } - // Parse manifest.json for descriptions (optional) + // Parse manifest.json for descriptions and relationships (optional) var manifestData map[string]interface{} if manifestPath != "" { - pterm.Info.Printf("Reading manifest.json for descriptions from: %s\n", manifestPath) + pterm.Info.Printf("Reading manifest.json for descriptions and relationships from: %s\n", manifestPath) manifestBytes, err := os.ReadFile(manifestPath) // #nosec G304 -- manifestPath is controlled by application if err != nil { pterm.Warning.Printf("Warning: Failed to read manifest file %s: %v\n", manifestPath, err) @@ -291,7 +309,24 @@ func ConvertDbtCatalogToWrenMDL(catalogPath string, data_source DataSource, mani } } - // Extract nodes + // Parse semantic_manifest.json for metrics and primary keys (optional and robust) + var semanticManifestData map[string]interface{} + if semanticManifestPath != "" { + pterm.Info.Printf("Reading semantic_manifest.json for metrics and primary keys from: %s\n", semanticManifestPath) + semanticBytes, err := os.ReadFile(semanticManifestPath) + if err != nil { + pterm.Warning.Printf("Warning: Could not read semantic_manifest.json: %v\n", err) + pterm.Warning.Println("Skipping metric and primary key conversion.") + } else { + if err := json.Unmarshal(semanticBytes, &semanticManifestData); err != nil { + pterm.Warning.Printf("Warning: Failed to parse semantic_manifest.json: %v\n", err) + pterm.Warning.Println("Skipping metric and primary key conversion.") + semanticManifestData = nil + } + } + } + + // Extract nodes from catalog nodesValue, exists := catalogData["nodes"] if !exists { return nil, fmt.Errorf("no 'nodes' section found in catalog") @@ -304,12 +339,96 @@ func ConvertDbtCatalogToWrenMDL(catalogPath string, data_source DataSource, mani // Initialize Wren MDL manifest manifest := &WrenMDLManifest{ - Catalog: "wren", // Default catalog name - Schema: "public", // Default schema name - Models: []WrenModel{}, - Relationships: []Relationship{}, - Views: []View{}, - DataSources: data_source.GetType(), // Default data source name + Catalog: "wren", + Schema: "public", + EnumDefinitions: []EnumDefinition{}, + Models: []WrenModel{}, + Relationships: []Relationship{}, + Metrics: []Metric{}, + Views: []View{}, + DataSources: dataSource.GetType(), + } + + // Maps to store pre-processed information + enumValueToNameMap := make(map[string]string) + columnToEnumNameMap := make(map[string]string) + columnToNotNullMap := make(map[string]bool) + modelToPrimaryKeyMap := make(map[string]string) + + // Pre-process manifest to find all tests (enums, not_null) + if manifestData != nil { + if nodes, ok := manifestData["nodes"].(map[string]interface{}); ok { + for nodeKey, nodeValue := range nodes { + nodeMap, ok := nodeValue.(map[string]interface{}) + if !ok { + continue + } + + // Handle tests on model columns (including structs) + if strings.HasPrefix(nodeKey, "model.") { + modelName := getModelNameFromNodeKey(nodeKey) + if modelName == "" { + continue + } + if columns, ok := nodeMap["columns"].(map[string]interface{}); ok { + for columnName, colData := range columns { + if colMap, ok := colData.(map[string]interface{}); ok { + processColumnForTests(nodeKey, modelName, columnName, colMap, &manifest.EnumDefinitions, enumValueToNameMap, columnToEnumNameMap, columnToNotNullMap) + } + } + } + } + + // Handle compiled test nodes for simple columns + if strings.HasPrefix(nodeKey, "test.") { + if testMeta, ok := nodeMap["test_metadata"].(map[string]interface{}); ok { + testName := getStringFromMap(testMeta, "name", "") + attachedNodeID := getStringFromMap(nodeMap, "attached_node", "") + columnName := getStringFromMap(nodeMap, "column_name", "") + modelName := getModelNameFromNodeKey(attachedNodeID) + + if attachedNodeID != "" && columnName != "" && modelName != "" { + columnKey := fmt.Sprintf("%s.%s", attachedNodeID, columnName) + + if testName == "not_null" { + columnToNotNullMap[columnKey] = true + } + + if testName == "accepted_values" { + if kwargs, ok := testMeta["kwargs"].(map[string]interface{}); ok { + if values, ok := kwargs["values"].([]interface{}); ok && len(values) > 0 { + createOrLinkEnum(modelName, columnName, columnKey, values, &manifest.EnumDefinitions, enumValueToNameMap, columnToEnumNameMap) + } + } + } + } + } + } + } + } + } + + // Pre-process semantic manifest for primary keys + if semanticManifestData != nil { + if semanticModels, ok := semanticManifestData["semantic_models"].([]interface{}); ok { + for _, sm := range semanticModels { + if smMap, ok := sm.(map[string]interface{}); ok { + modelName := getStringFromMap(smMap, "name", "") + if entities, ok := smMap["entities"].([]interface{}); ok { + for _, entity := range entities { + if entityMap, ok := entity.(map[string]interface{}); ok { + if getStringFromMap(entityMap, "type", "") == "primary" { + pk := getStringFromMap(entityMap, "expr", "") + if modelName != "" && pk != "" { + modelToPrimaryKeyMap[modelName] = pk + } + } + } + } + } + } + } + } } // Convert each dbt model to Wren model @@ -318,37 +437,344 @@ func ConvertDbtCatalogToWrenMDL(catalogPath string, data_source DataSource, mani if !ok { continue } - - // Only process models (skip seeds, tests, etc.) if !strings.HasPrefix(nodeKey, "model.") { continue } - - // Skip staging models - if strings.Contains(nodeKey, ".stg_") || strings.Contains(nodeKey, ".staging_") { + if !includeStagingModels && (strings.Contains(nodeKey, ".stg_") || strings.Contains(nodeKey, ".staging_")) { continue } - - model, err := convertDbtNodeToWrenModel(nodeKey, nodeMap, data_source, manifestData) + model, err := convertDbtNodeToWrenModel(nodeKey, nodeMap, dataSource, manifestData, columnToEnumNameMap, columnToNotNullMap, modelToPrimaryKeyMap) if err != nil { pterm.Warning.Printf("Warning: Failed to convert model %s: %v\n", nodeKey, err) continue } - manifest.Models = append(manifest.Models, *model) } + // Generate relationships from manifest.json + if manifestData != nil { + manifest.Relationships = generateRelationships(manifestData) + } + + // Generate metrics from semantic_manifest.json, only if data is available + if semanticManifestData != nil { + manifest.Metrics = convertDbtMetricsToWrenMetrics(semanticManifestData) + } + return manifest, nil } +// generateRelationships iterates through the manifest and creates relationship definitions. +func generateRelationships(manifestData map[string]interface{}) []Relationship { + var relationships []Relationship + if nodes, ok := manifestData["nodes"].(map[string]interface{}); ok { + for nodeKey, nodeValue := range nodes { + nodeMap, ok := nodeValue.(map[string]interface{}) + if !ok { + continue + } + + // Case 1: Handle tests on model columns (including structs) + if strings.HasPrefix(nodeKey, "model.") { + fromModelName := getModelNameFromNodeKey(nodeKey) + if fromModelName == "" { + continue + } + if columns, ok := nodeMap["columns"].(map[string]interface{}); ok { + for columnName, colData := range columns { + if colMap, ok := colData.(map[string]interface{}); ok { + relationships = append(relationships, parseTestsForRelationships(fromModelName, columnName, colMap)...) + } + } + } + } + + // Case 2: Handle compiled test nodes for simple columns + if strings.HasPrefix(nodeKey, "test.") { + if testMeta, ok := nodeMap["test_metadata"].(map[string]interface{}); ok { + if getStringFromMap(testMeta, "name", "") == "relationships" { + if kwargs, ok := testMeta["kwargs"].(map[string]interface{}); ok { + toRef := getStringFromMap(kwargs, "to", "") + toField := getStringFromMap(kwargs, "field", "") + toModelName := parseRef(toRef) + fromColumnName := getStringFromMap(nodeMap, "column_name", "") + attachedNodeID := getStringFromMap(nodeMap, "attached_node", "") + fromModelName := getModelNameFromNodeKey(attachedNodeID) + + if toModelName != "" && toField != "" && fromModelName != "" && fromColumnName != "" { + rel := Relationship{ + Name: fmt.Sprintf("%s_to_%s_by_%s", fromModelName, toModelName, fromColumnName), + Models: []string{fromModelName, toModelName}, + JoinType: "MANY_TO_ONE", + Condition: fmt.Sprintf("%s.%s = %s.%s", fromModelName, fromColumnName, toModelName, toField), + } + relationships = append(relationships, rel) + } + } + } + } + } + } + } + return relationships +} + +// parseTestsForRelationships is a helper function to extract relationship tests from a column or its fields. +func parseTestsForRelationships(fromModelName, columnName string, colMap map[string]interface{}) []Relationship { + var relationships []Relationship + // Case 1: Tests are directly on the column. + if tests, ok := colMap["tests"].([]interface{}); ok { + relationships = append(relationships, extractRelationshipsFromTests(fromModelName, columnName, tests)...) + } + // Case 2: Tests are on fields within a struct column. + if fields, ok := colMap["fields"].([]interface{}); ok { + for _, fieldData := range fields { + if fieldMap, ok := fieldData.(map[string]interface{}); ok { + fieldName := getStringFromMap(fieldMap, "name", "") + if fieldName == "" { + continue + } + if tests, ok := fieldMap["tests"].([]interface{}); ok { + relationships = append(relationships, extractRelationshipsFromTests(fromModelName, fieldName, tests)...) + } + } + } + } + return relationships +} + +// extractRelationshipsFromTests extracts relationship info from a 'tests' array. +func extractRelationshipsFromTests(fromModelName, fromColumnName string, tests []interface{}) []Relationship { + var relationships []Relationship + for _, test := range tests { + if relTest, ok := test.(map[string]interface{}); ok { + if relData, ok := relTest["relationships"].(map[string]interface{}); ok { + toRef := getStringFromMap(relData, "to", "") + toField := getStringFromMap(relData, "field", "") + toModelName := parseRef(toRef) + + if toModelName != "" && toField != "" { + rel := Relationship{ + Name: fmt.Sprintf("%s_to_%s_by_%s", fromModelName, toModelName, fromColumnName), + Models: []string{fromModelName, toModelName}, + JoinType: "MANY_TO_ONE", + Condition: fmt.Sprintf("%s.%s = %s.%s", fromModelName, fromColumnName, toModelName, toField), + } + relationships = append(relationships, rel) + } + } + } + } + return relationships +} + +// createOrLinkEnum is a helper to de-duplicate and manage enum creation. +func createOrLinkEnum(modelName, columnName, columnKey string, values []interface{}, + allEnums *[]EnumDefinition, enumValueToNameMap, columnToEnumNameMap map[string]string) { + + var strValues []string + for _, v := range values { + if s, ok := v.(string); ok { + strValues = append(strValues, s) + } + } + if len(strValues) == 0 { + return + } + sort.Strings(strValues) + valueKey := strings.Join(strValues, ",") + + enumName, exists := enumValueToNameMap[valueKey] + if !exists { + enumName = fmt.Sprintf("%s_%s_Enum", modelName, columnName) + // Sanitize enum name + enumName = strings.ReplaceAll(enumName, ".", "_") + *allEnums = append(*allEnums, EnumDefinition{ + Name: enumName, + Values: strValues, + }) + enumValueToNameMap[valueKey] = enumName + } + columnToEnumNameMap[columnKey] = enumName +} + +// processColumnForTests recursively finds tests in embedded column definitions. +func processColumnForTests(nodeKey, modelName, columnName string, colMap map[string]interface{}, + allEnums *[]EnumDefinition, enumValueToNameMap, columnToEnumNameMap map[string]string, columnToNotNullMap map[string]bool) { + + // Helper to handle the actual test processing + processTests := func(currentColumnKey, currentColumnName string, tests []interface{}) { + for _, test := range tests { + // Handle not_null test (string) + if testStr, ok := test.(string); ok && testStr == "not_null" { + columnToNotNullMap[currentColumnKey] = true + } + + // Handle accepted_values test (map) + if testMap, ok := test.(map[string]interface{}); ok { + if accepted, ok := testMap["accepted_values"].(map[string]interface{}); ok { + if values, ok := accepted["values"].([]interface{}); ok && len(values) > 0 { + createOrLinkEnum(modelName, currentColumnName, currentColumnKey, values, allEnums, enumValueToNameMap, columnToEnumNameMap) + } + } + } + } + } + + // Case 1: Tests are directly on the column (for structs). + if tests, ok := colMap["tests"].([]interface{}); ok { + columnKey := fmt.Sprintf("%s.%s", nodeKey, columnName) + processTests(columnKey, columnName, tests) + } + + // Case 2: Column is a struct with tests on its fields. + if fields, ok := colMap["fields"].([]interface{}); ok { + for _, fieldData := range fields { + if fieldMap, ok := fieldData.(map[string]interface{}); ok { + fieldName := getStringFromMap(fieldMap, "name", "") + if fieldName == "" { + continue + } + if tests, ok := fieldMap["tests"].([]interface{}); ok { + columnKey := fmt.Sprintf("%s.%s", nodeKey, fieldName) // The key is based on the field name + processTests(columnKey, fieldName, tests) + } + } + } + } +} + +// convertDbtMetricsToWrenMetrics converts dbt metrics from semantic manifest to Wren MDL format +func convertDbtMetricsToWrenMetrics(semanticData map[string]interface{}) []Metric { + var wrenMetrics []Metric + measureLookup := make(map[string]map[string]interface{}) // modelName -> measureName -> measureData + + // First, build a lookup table for all measures + if semanticModels, ok := semanticData["semantic_models"].([]interface{}); ok { + for _, sm := range semanticModels { + if smMap, ok := sm.(map[string]interface{}); ok { + modelName := getStringFromMap(smMap, "name", "") + if modelName == "" { + continue + } + measureLookup[modelName] = make(map[string]interface{}) + if measures, ok := smMap["measures"].([]interface{}); ok { + for _, m := range measures { + if measureMap, ok := m.(map[string]interface{}); ok { + measureName := getStringFromMap(measureMap, "name", "") + if measureName != "" { + measureLookup[modelName][measureName] = measureMap + } + } + } + } + } + } + } + + // Now, iterate through the metrics and build Wren metrics + if metrics, ok := semanticData["metrics"].([]interface{}); ok { + for _, m := range metrics { + if metricMap, ok := m.(map[string]interface{}); ok { + metricName := getStringFromMap(metricMap, "name", "") + metricLabel := getStringFromMap(metricMap, "label", metricName) + metricDesc := getStringFromMap(metricMap, "description", "") + metricType := getStringFromMap(metricMap, "type", "") + + wrenMetric := Metric{ + Name: metricName, + DisplayName: metricLabel, + Description: metricDesc, + } + + typeParams, _ := metricMap["type_params"].(map[string]interface{}) + + // Find the underlying model and dimensions + var baseModel string + var timeDimensions []string + if inputMeasuresValue, ok := typeParams["input_measures"]; ok { + if inputMeasuresList, ok := inputMeasuresValue.([]interface{}); ok && len(inputMeasuresList) > 0 { + // Find the model this metric is based on + for model, measures := range measureLookup { + for _, inputMeasure := range inputMeasuresList { + if imMap, ok := inputMeasure.(map[string]interface{}); ok { + imName := getStringFromMap(imMap, "name", "") + if _, exists := measures[imName]; exists { + baseModel = model + break + } + } + } + if baseModel != "" { + break + } + } + } + } + + // Find time dimensions from the semantic model + if baseModel != "" { + wrenMetric.Models = []string{baseModel} + if semanticModels, ok := semanticData["semantic_models"].([]interface{}); ok { + for _, sm := range semanticModels { + if smMap, ok := sm.(map[string]interface{}); ok { + if getStringFromMap(smMap, "name", "") == baseModel { + if dims, ok := smMap["dimensions"].([]interface{}); ok { + for _, d := range dims { + if dimMap, ok := d.(map[string]interface{}); ok { + if getStringFromMap(dimMap, "type", "") == "time" { + timeDimensions = append(timeDimensions, getStringFromMap(dimMap, "name", "")) + } + } + } + } + } + } + } + } + wrenMetric.Dimensions = timeDimensions + } + + // Build the aggregation expression + switch metricType { + case "simple": + if measure, ok := typeParams["measure"].(map[string]interface{}); ok { + measureName := getStringFromMap(measure, "name", "") + if measureData, ok := measureLookup[baseModel][measureName].(map[string]interface{}); ok { + agg := getStringFromMap(measureData, "agg", "sum") + expr := getStringFromMap(measureData, "expr", measureName) + wrenMetric.Aggregation = fmt.Sprintf("%s(%s)", strings.ToUpper(agg), expr) + } + } + case "ratio": + if num, ok := typeParams["numerator"].(map[string]interface{}); ok { + if den, ok := typeParams["denominator"].(map[string]interface{}); ok { + numName := getStringFromMap(num, "name", "") + denName := getStringFromMap(den, "name", "") + wrenMetric.Aggregation = fmt.Sprintf("%s / %s", numName, denName) + } + } + case "derived": + wrenMetric.Aggregation = getStringFromMap(typeParams, "expr", "") + } + + if wrenMetric.Aggregation != "" && len(wrenMetric.Models) > 0 { + wrenMetrics = append(wrenMetrics, wrenMetric) + } + } + } + } + + return wrenMetrics +} + // convertDbtNodeToWrenModel converts a single dbt node to Wren model -func convertDbtNodeToWrenModel(nodeKey string, nodeData map[string]interface{}, data_source DataSource, manifestData map[string]interface{}) (*WrenModel, error) { - // Extract model name from node key (e.g., "model.jaffle_shop.customers" -> "customers") - parts := strings.Split(nodeKey, ".") - if len(parts) < 3 { +func convertDbtNodeToWrenModel(nodeKey string, nodeData map[string]interface{}, dataSource DataSource, manifestData map[string]interface{}, columnToEnumNameMap map[string]string, columnToNotNullMap map[string]bool, modelToPrimaryKeyMap map[string]string) (*WrenModel, error) { + // Extract model name from node key + modelName := getModelNameFromNodeKey(nodeKey) + if modelName == "" { return nil, fmt.Errorf("invalid node key format: %s", nodeKey) } - modelName := parts[len(parts)-1] // Extract metadata metadataValue, exists := nodeData["metadata"] @@ -417,9 +843,18 @@ func convertDbtNodeToWrenModel(nodeKey string, nodeData map[string]interface{}, continue } + columnName := getStringFromMap(colMap, "name", "") + columnKey := fmt.Sprintf("%s.%s", nodeKey, columnName) + column := WrenColumn{ - Name: getStringFromMap(colMap, "name", ""), - Type: data_source.MapType(getStringFromMap(colMap, "type", "")), + Name: columnName, + Type: dataSource.MapType(getStringFromMap(colMap, "type", "")), + NotNull: columnToNotNullMap[columnKey], // Will be false if not found + } + + // Check for and assign enum + if enumName, ok := columnToEnumNameMap[columnKey]; ok { + column.Enum = enumName } // Initialize properties map if needed @@ -435,7 +870,6 @@ func convertDbtNodeToWrenModel(nodeKey string, nodeData map[string]interface{}, } // Set notNull based on comment or other indicators - // This is a basic implementation - you might need more sophisticated logic if comment := getStringFromMap(colMap, "comment", ""); comment != "" { column.Properties["comment"] = comment } @@ -455,6 +889,11 @@ func convertDbtNodeToWrenModel(nodeKey string, nodeData map[string]interface{}, Columns: wrenColumns, } + // Set primary key from semantic manifest if available + if pk, ok := modelToPrimaryKeyMap[modelName]; ok { + model.PrimaryKey = pk + } + // Set model description from manifest if available if modelDescription != "" { if model.Properties == nil { @@ -475,3 +914,24 @@ func getStringFromMap(m map[string]interface{}, key, defaultValue string) string } return defaultValue } + +// getModelNameFromNodeKey extracts the model name from a dbt node key. +// e.g., "model.jaffle_shop.customers" -> "customers" +func getModelNameFromNodeKey(nodeKey string) string { + parts := strings.Split(nodeKey, ".") + if len(parts) > 0 { + return parts[len(parts)-1] + } + return "" +} + +// parseRef extracts the model name from a dbt ref string. +// e.g., "ref('stg_orders')" +func parseRef(refStr string) string { + re := regexp.MustCompile(`ref\(['"]([^'"]+)['"]\)`) + matches := re.FindStringSubmatch(refStr) + if len(matches) > 1 { + return matches[1] + } + return "" +} diff --git a/wren-launcher/commands/dbt/wren_mdl.go b/wren-launcher/commands/dbt/wren_mdl.go index cc0b44021c..d0c1d628ee 100644 --- a/wren-launcher/commands/dbt/wren_mdl.go +++ b/wren-launcher/commands/dbt/wren_mdl.go @@ -2,12 +2,20 @@ package dbt // WrenMDLManifest represents the complete Wren MDL structure type WrenMDLManifest struct { - Catalog string `json:"catalog"` - Schema string `json:"schema"` - Models []WrenModel `json:"models"` - Relationships []Relationship `json:"relationships"` - Views []View `json:"views"` - DataSources string `json:"dataSources,omitempty"` + Catalog string `json:"catalog"` + Schema string `json:"schema"` + EnumDefinitions []EnumDefinition `json:"enumDefinitions,omitempty"` // Added EnumDefinitions + Models []WrenModel `json:"models"` + Relationships []Relationship `json:"relationships"` + Metrics []Metric `json:"metrics,omitempty"` + Views []View `json:"views"` + DataSources string `json:"dataSources,omitempty"` +} + +// EnumDefinition represents a named list of values that can be used by columns. +type EnumDefinition struct { + Name string `json:"name"` + Values []string `json:"values"` } // WrenModel represents a model in the Wren MDL format @@ -32,6 +40,7 @@ type TableReference struct { type WrenColumn struct { Name string `json:"name"` Type string `json:"type"` + Enum string `json:"enum,omitempty"` // Added Enum field Relationship string `json:"relationship,omitempty"` IsCalculated bool `json:"isCalculated,omitempty"` NotNull bool `json:"notNull,omitempty"` @@ -48,6 +57,16 @@ type Relationship struct { Properties map[string]string `json:"properties,omitempty"` } +// Metric defines a business-level calculation in Wren MDL. +type Metric struct { + Name string `json:"name"` + Models []string `json:"models"` + Dimensions []string `json:"dimensions"` + Aggregation string `json:"aggregation"` + DisplayName string `json:"displayName"` + Description string `json:"description,omitempty"` +} + // View represents a view in the Wren MDL format type View struct { Name string `json:"name"` diff --git a/wren-launcher/commands/launch.go b/wren-launcher/commands/launch.go index 4edb0f6d4c..ad7be2bad3 100644 --- a/wren-launcher/commands/launch.go +++ b/wren-launcher/commands/launch.go @@ -522,7 +522,7 @@ func processDbtProject(projectDir string) (string, error) { } // Use the core conversion function from dbt package - result, err := DbtConvertProject(dbtProjectPath, targetDir, profileName, target, true) + result, err := DbtConvertProject(dbtProjectPath, targetDir, profileName, target, true, false) if err != nil { return "", fmt.Errorf("failed to convert dbt project: %w", err) } @@ -530,4 +530,4 @@ func processDbtProject(projectDir string) (string, error) { pterm.Info.Printf("Successfully processed dbt project to target directory: %s\n", targetDir) return result.LocalStoragePath, nil -} +} \ No newline at end of file From 5b29a3829e67bfaeb084572ba86e53ff919ceb10 Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Sun, 3 Aug 2025 18:33:04 -0400 Subject: [PATCH 09/18] feat(dbt): implement review feedback for robustness and security This commit incorporates a series of changes based on code review feedback to improve the dbt to Wren MDL converter. The updates focus on enhancing security, adding more robust validation and parsing, and increasing the completeness of the generated MDL file. In `wren_mdl.go`: - feat: Adds a `DisplayName` field to the `WrenColumn` struct to support user-friendly labels for columns. In `data_source.go`: - feat: Adds JSON validation for the `keyfile_json` field in BigQuery data sources to provide earlier feedback on malformed credentials. - refactor: Implements BigQuery-specific data type mappings in the `MapType` method to correctly convert types like `INT64`, `TIMESTAMP`, and `NUMERIC`. In `converter.go`: - fix(security): Omits the `keyfile_json` content from the generated `wren-datasource.json` to prevent exposing sensitive credentials in the output file. - feat: Adds mapping for a dbt column's `meta.label` to the `DisplayName` property in the corresponding Wren `WrenColumn`, improving the usability of the output. - refactor: Enhances enum name generation with more robust sanitization, replacing all non-alphanumeric characters to ensure valid identifiers. - refactor: Improves the `ref()` parsing regex to handle optional whitespace, making it more resilient to formatting variations. - refactor: Adds validation for metric `input_measures` to log a warning if a referenced measure cannot be found, improving debuggability. - fix: Resolves a persistent compilation error related to an incorrect type assertion on `inputMeasures` by refactoring the logic to correctly identify the base model for a metric. --- wren-launcher/commands/dbt/converter.go | 78 +++++++++++++++-------- wren-launcher/commands/dbt/data_source.go | 31 ++++++++- wren-launcher/commands/dbt/wren_mdl.go | 5 +- 3 files changed, 81 insertions(+), 33 deletions(-) diff --git a/wren-launcher/commands/dbt/converter.go b/wren-launcher/commands/dbt/converter.go index 86494777ec..634ef6caad 100644 --- a/wren-launcher/commands/dbt/converter.go +++ b/wren-launcher/commands/dbt/converter.go @@ -170,14 +170,14 @@ func ConvertDbtProjectCore(opts ConvertOptions) (*ConvertResult, error) { }, } case *WrenBigQueryDataSource: + // SECURITY FIX: Omit keyfile_json from output for security. wrenDataSource = map[string]interface{}{ "type": "bigquery", "properties": map[string]interface{}{ - "project": typedDS.Project, - "dataset": typedDS.Dataset, - "method": typedDS.Method, - "keyfile": typedDS.Keyfile, - "keyfile_json": typedDS.KeyfileJSON, + "project": typedDS.Project, + "dataset": typedDS.Dataset, + "method": typedDS.Method, + "keyfile": typedDS.Keyfile, // Path reference is safe }, } case *WrenMysqlDataSource: @@ -413,7 +413,11 @@ func ConvertDbtCatalogToWrenMDL(catalogPath string, dataSource DataSource, manif if semanticModels, ok := semanticManifestData["semantic_models"].([]interface{}); ok { for _, sm := range semanticModels { if smMap, ok := sm.(map[string]interface{}); ok { - modelName := getStringFromMap(smMap, "name", "") + var modelName string + if nr, ok := smMap["node_relation"].(map[string]interface{}); ok { + modelName = getStringFromMap(nr, "alias", "") + } + if entities, ok := smMap["entities"].([]interface{}); ok { for _, entity := range entities { if entityMap, ok := entity.(map[string]interface{}); ok { @@ -587,8 +591,12 @@ func createOrLinkEnum(modelName, columnName, columnKey string, values []interfac enumName, exists := enumValueToNameMap[valueKey] if !exists { enumName = fmt.Sprintf("%s_%s_Enum", modelName, columnName) - // Sanitize enum name - enumName = strings.ReplaceAll(enumName, ".", "_") + // Sanitize enum name to be a valid identifier + re := regexp.MustCompile(`[^a-zA-Z0-9_]`) + enumName = re.ReplaceAllString(enumName, "_") + if len(enumName) > 0 && enumName[0] >= '0' && enumName[0] <= '9' { + enumName = "_" + enumName + } *allEnums = append(*allEnums, EnumDefinition{ Name: enumName, Values: strValues, @@ -647,9 +655,10 @@ func processColumnForTests(nodeKey, modelName, columnName string, colMap map[str // convertDbtMetricsToWrenMetrics converts dbt metrics from semantic manifest to Wren MDL format func convertDbtMetricsToWrenMetrics(semanticData map[string]interface{}) []Metric { var wrenMetrics []Metric - measureLookup := make(map[string]map[string]interface{}) // modelName -> measureName -> measureData + measureToModelMap := make(map[string]string) + measureDataLookup := make(map[string]map[string]interface{}) // measureName -> measureData - // First, build a lookup table for all measures + // First, build lookup tables for all measures if semanticModels, ok := semanticData["semantic_models"].([]interface{}); ok { for _, sm := range semanticModels { if smMap, ok := sm.(map[string]interface{}); ok { @@ -657,13 +666,13 @@ func convertDbtMetricsToWrenMetrics(semanticData map[string]interface{}) []Metri if modelName == "" { continue } - measureLookup[modelName] = make(map[string]interface{}) if measures, ok := smMap["measures"].([]interface{}); ok { for _, m := range measures { if measureMap, ok := m.(map[string]interface{}); ok { measureName := getStringFromMap(measureMap, "name", "") if measureName != "" { - measureLookup[modelName][measureName] = measureMap + measureToModelMap[measureName] = modelName + measureDataLookup[measureName] = measureMap } } } @@ -695,19 +704,17 @@ func convertDbtMetricsToWrenMetrics(semanticData map[string]interface{}) []Metri if inputMeasuresValue, ok := typeParams["input_measures"]; ok { if inputMeasuresList, ok := inputMeasuresValue.([]interface{}); ok && len(inputMeasuresList) > 0 { // Find the model this metric is based on - for model, measures := range measureLookup { - for _, inputMeasure := range inputMeasuresList { - if imMap, ok := inputMeasure.(map[string]interface{}); ok { - imName := getStringFromMap(imMap, "name", "") - if _, exists := measures[imName]; exists { - baseModel = model - break - } + for _, inputMeasure := range inputMeasuresList { + if imMap, ok := inputMeasure.(map[string]interface{}); ok { + imName := getStringFromMap(imMap, "name", "") + if model, exists := measureToModelMap[imName]; exists { + baseModel = model + break // Assume all measures for a metric come from the same model } } - if baseModel != "" { - break - } + } + if baseModel == "" { + pterm.Warning.Printf("Could not find a parent model for metric '%s'\n", metricName) } } } @@ -740,7 +747,7 @@ func convertDbtMetricsToWrenMetrics(semanticData map[string]interface{}) []Metri case "simple": if measure, ok := typeParams["measure"].(map[string]interface{}); ok { measureName := getStringFromMap(measure, "name", "") - if measureData, ok := measureLookup[baseModel][measureName].(map[string]interface{}); ok { + if measureData, ok := measureDataLookup[measureName]; ok { agg := getStringFromMap(measureData, "agg", "sum") expr := getStringFromMap(measureData, "expr", measureName) wrenMetric.Aggregation = fmt.Sprintf("%s(%s)", strings.ToUpper(agg), expr) @@ -847,9 +854,10 @@ func convertDbtNodeToWrenModel(nodeKey string, nodeData map[string]interface{}, columnKey := fmt.Sprintf("%s.%s", nodeKey, columnName) column := WrenColumn{ - Name: columnName, - Type: dataSource.MapType(getStringFromMap(colMap, "type", "")), - NotNull: columnToNotNullMap[columnKey], // Will be false if not found + Name: columnName, + DisplayName: getStringFromMap(getMapFromMap(colMap, "meta", nil), "label", ""), + Type: dataSource.MapType(getStringFromMap(colMap, "type", "")), + NotNull: columnToNotNullMap[columnKey], // Will be false if not found } // Check for and assign enum @@ -907,6 +915,9 @@ func convertDbtNodeToWrenModel(nodeKey string, nodeData map[string]interface{}, // getStringFromMap safely extracts a string value from a map func getStringFromMap(m map[string]interface{}, key, defaultValue string) string { + if m == nil { + return defaultValue + } if value, exists := m[key]; exists { if str, ok := value.(string); ok { return str @@ -915,6 +926,16 @@ func getStringFromMap(m map[string]interface{}, key, defaultValue string) string return defaultValue } +// getMapFromMap safely extracts a map value from a map +func getMapFromMap(m map[string]interface{}, key string, defaultValue map[string]interface{}) map[string]interface{} { + if value, exists := m[key]; exists { + if str, ok := value.(map[string]interface{}); ok { + return str + } + } + return defaultValue +} + // getModelNameFromNodeKey extracts the model name from a dbt node key. // e.g., "model.jaffle_shop.customers" -> "customers" func getModelNameFromNodeKey(nodeKey string) string { @@ -928,7 +949,8 @@ func getModelNameFromNodeKey(nodeKey string) string { // parseRef extracts the model name from a dbt ref string. // e.g., "ref('stg_orders')" func parseRef(refStr string) string { - re := regexp.MustCompile(`ref\(['"]([^'"]+)['"]\)`) + // Handle ref() with optional spaces and both quote types + re := regexp.MustCompile(`ref\s*\(\s*['"]([^'"]+)['"]\s*\)`) matches := re.FindStringSubmatch(refStr) if len(matches) > 1 { return matches[1] diff --git a/wren-launcher/commands/dbt/data_source.go b/wren-launcher/commands/dbt/data_source.go index 06903fb9f8..5eb50e5780 100644 --- a/wren-launcher/commands/dbt/data_source.go +++ b/wren-launcher/commands/dbt/data_source.go @@ -1,6 +1,7 @@ package dbt import ( + "encoding/json" "fmt" "path/filepath" "strconv" @@ -395,6 +396,11 @@ func (ds *WrenBigQueryDataSource) Validate() error { if ds.KeyfileJSON == "" { return fmt.Errorf("keyfile_json cannot be empty for method 'service-account-json'") } + // Validate that keyfile_json is valid JSON + var js map[string]interface{} + if json.Unmarshal([]byte(ds.KeyfileJSON), &js) != nil { + return fmt.Errorf("keyfile_json is not valid JSON") + } case "oauth", "oauth-secrets": return fmt.Errorf("authentication method '%s' is not supported; please use a service account method", ds.Method) default: @@ -406,9 +412,28 @@ func (ds *WrenBigQueryDataSource) Validate() error { // MapType implements DataSource interface func (ds *WrenBigQueryDataSource) MapType(sourceType string) string { - // Add BigQuery specific type mappings here if needed - // For now, we can use the default mapping logic - return sourceType + switch strings.ToUpper(sourceType) { + case "INT64", "INTEGER": + return "integer" + case "FLOAT64", "FLOAT": + return "double" + case "STRING": + return "varchar" + case "BOOL", "BOOLEAN": + return "boolean" + case "DATE": + return "date" + case "TIMESTAMP", "DATETIME": + return "timestamp" + case "NUMERIC", "DECIMAL", "BIGNUMERIC": + return "double" + case "BYTES": + return "varchar" + case "JSON": + return "varchar" + default: + return strings.ToLower(sourceType) + } } // GetActiveDataSources gets active data sources based on specified profile and target diff --git a/wren-launcher/commands/dbt/wren_mdl.go b/wren-launcher/commands/dbt/wren_mdl.go index d0c1d628ee..ca77ad0524 100644 --- a/wren-launcher/commands/dbt/wren_mdl.go +++ b/wren-launcher/commands/dbt/wren_mdl.go @@ -4,7 +4,7 @@ package dbt type WrenMDLManifest struct { Catalog string `json:"catalog"` Schema string `json:"schema"` - EnumDefinitions []EnumDefinition `json:"enumDefinitions,omitempty"` // Added EnumDefinitions + EnumDefinitions []EnumDefinition `json:"enumDefinitions,omitempty"` Models []WrenModel `json:"models"` Relationships []Relationship `json:"relationships"` Metrics []Metric `json:"metrics,omitempty"` @@ -39,8 +39,9 @@ type TableReference struct { // WrenColumn represents a column in the Wren MDL format type WrenColumn struct { Name string `json:"name"` + DisplayName string `json:"displayName,omitempty"` Type string `json:"type"` - Enum string `json:"enum,omitempty"` // Added Enum field + Enum string `json:"enum,omitempty"` Relationship string `json:"relationship,omitempty"` IsCalculated bool `json:"isCalculated,omitempty"` NotNull bool `json:"notNull,omitempty"` From 518d3e165f1b06f92a988d51041ca79c04d7e992 Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Mon, 18 Aug 2025 17:33:55 -0400 Subject: [PATCH 10/18] Data connection and key adjustments --- wren-launcher/commands/dbt/converter.go | 8 ++-- wren-launcher/commands/dbt/data_source.go | 57 ++++++++++------------- 2 files changed, 27 insertions(+), 38 deletions(-) diff --git a/wren-launcher/commands/dbt/converter.go b/wren-launcher/commands/dbt/converter.go index 634ef6caad..19e4b5bd82 100644 --- a/wren-launcher/commands/dbt/converter.go +++ b/wren-launcher/commands/dbt/converter.go @@ -170,14 +170,12 @@ func ConvertDbtProjectCore(opts ConvertOptions) (*ConvertResult, error) { }, } case *WrenBigQueryDataSource: - // SECURITY FIX: Omit keyfile_json from output for security. wrenDataSource = map[string]interface{}{ "type": "bigquery", "properties": map[string]interface{}{ - "project": typedDS.Project, - "dataset": typedDS.Dataset, - "method": typedDS.Method, - "keyfile": typedDS.Keyfile, // Path reference is safe + "project_id": typedDS.Project, + "dataset_id": typedDS.Dataset, + "credentials": typedDS.Credentials, }, } case *WrenMysqlDataSource: diff --git a/wren-launcher/commands/dbt/data_source.go b/wren-launcher/commands/dbt/data_source.go index 5eb50e5780..36416fc824 100644 --- a/wren-launcher/commands/dbt/data_source.go +++ b/wren-launcher/commands/dbt/data_source.go @@ -1,6 +1,7 @@ package dbt import ( + "encoding/base64" "encoding/json" "fmt" "path/filepath" @@ -183,7 +184,7 @@ func convertToMysqlDataSource(conn DbtConnection) (*WrenMysqlDataSource, error) // convertToBigQueryDataSource converts to BigQuery data source func convertToBigQueryDataSource(conn DbtConnection) (*WrenBigQueryDataSource, error) { - // We need to extract the keyfile content, which might be in the 'Additional' map + // Extract the keyfile content from the 'Additional' map var keyfileJSON string if kfj, exists := conn.Additional["keyfile_json"]; exists { if kfjStr, ok := kfj.(string); ok { @@ -191,12 +192,23 @@ func convertToBigQueryDataSource(conn DbtConnection) (*WrenBigQueryDataSource, e } } + if keyfileJSON == "" { + return nil, fmt.Errorf("keyfile_json not found or is empty in BigQuery connection details") + } + + // Validate that keyfile_json is valid JSON + var js map[string]interface{} + if err := json.Unmarshal([]byte(keyfileJSON), &js); err != nil { + return nil, fmt.Errorf("keyfile_json is not valid JSON: %w", err) + } + + // Base64 encode the keyfile JSON string for the credentials field + credentials := base64.StdEncoding.EncodeToString([]byte(keyfileJSON)) + ds := &WrenBigQueryDataSource{ Project: conn.Project, Dataset: conn.Dataset, - Method: conn.Method, - Keyfile: conn.Keyfile, - KeyfileJSON: keyfileJSON, + Credentials: credentials, } return ds, nil } @@ -365,11 +377,9 @@ func (ds *WrenMysqlDataSource) MapType(sourceType string) string { } type WrenBigQueryDataSource struct { - Project string `json:"project"` - Dataset string `json:"dataset"` - Method string `json:"method"` - Keyfile string `json:"keyfile,omitempty"` - KeyfileJSON string `json:"keyfile_json,omitempty"` + Project string `json:"project_id"` + Dataset string `json:"dataset_id"` + Credentials string `json:"credentials"` } // GetType implements DataSource interface @@ -380,33 +390,14 @@ func (ds *WrenBigQueryDataSource) GetType() string { // Validate implements DataSource interface func (ds *WrenBigQueryDataSource) Validate() error { if ds.Project == "" { - return fmt.Errorf("project cannot be empty") + return fmt.Errorf("project_id cannot be empty") } if ds.Dataset == "" { - return fmt.Errorf("dataset cannot be empty") + return fmt.Errorf("dataset_id cannot be empty") } - - // Validate based on the authentication method - switch ds.Method { - case "service-account": - if ds.Keyfile == "" { - return fmt.Errorf("keyfile cannot be empty for method 'service-account'") - } - case "service-account-json": - if ds.KeyfileJSON == "" { - return fmt.Errorf("keyfile_json cannot be empty for method 'service-account-json'") - } - // Validate that keyfile_json is valid JSON - var js map[string]interface{} - if json.Unmarshal([]byte(ds.KeyfileJSON), &js) != nil { - return fmt.Errorf("keyfile_json is not valid JSON") - } - case "oauth", "oauth-secrets": - return fmt.Errorf("authentication method '%s' is not supported; please use a service account method", ds.Method) - default: - return fmt.Errorf("unsupported or missing authentication method: '%s'", ds.Method) + if ds.Credentials == "" { + return fmt.Errorf("credentials cannot be empty") } - return nil } @@ -550,4 +541,4 @@ func (d *DefaultDataSource) MapType(sourceType string) string { default: return strings.ToLower(sourceType) } -} +} \ No newline at end of file From e4f41c1c1629a79d2b42c205a1b345099c3fe430 Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Mon, 18 Aug 2025 18:32:32 -0400 Subject: [PATCH 11/18] Small fixes * fix: Adjusted some behaviors around keyfiles vs. inlined JSON * fix: Corrected a logical flaw where ratio metrics in dbt were calculated at a row level (numerator / denominator) instead of after aggregation. The code now correctly generates expressions like SUM(numerator) / SUM(denominator). * perf: Improved performance by pre-compiling a regular expression for parsing ref() functions at the package level, preventing it from being re-compiled on every function call. --- wren-launcher/commands/dbt/converter.go | 39 ++++++++-- wren-launcher/commands/dbt/data_source.go | 90 ++++++++++++++++++----- wren-launcher/commands/launch.go | 26 ++++++- 3 files changed, 126 insertions(+), 29 deletions(-) diff --git a/wren-launcher/commands/dbt/converter.go b/wren-launcher/commands/dbt/converter.go index 19e4b5bd82..eedea934e6 100644 --- a/wren-launcher/commands/dbt/converter.go +++ b/wren-launcher/commands/dbt/converter.go @@ -442,8 +442,11 @@ func ConvertDbtCatalogToWrenMDL(catalogPath string, dataSource DataSource, manif if !strings.HasPrefix(nodeKey, "model.") { continue } - if !includeStagingModels && (strings.Contains(nodeKey, ".stg_") || strings.Contains(nodeKey, ".staging_")) { - continue + if !includeStagingModels { + mn := getModelNameFromNodeKey(nodeKey) + if strings.HasPrefix(mn, "stg_") || strings.HasPrefix(mn, "staging_") { + continue + } } model, err := convertDbtNodeToWrenModel(nodeKey, nodeMap, dataSource, manifestData, columnToEnumNameMap, columnToNotNullMap, modelToPrimaryKeyMap) if err != nil { @@ -518,7 +521,18 @@ func generateRelationships(manifestData map[string]interface{}) []Relationship { } } } - return relationships + seen := make(map[string]struct{}, len(relationships)) + var unique []Relationship + for _, r := range relationships { + key := r.Name "|" r.JoinType "|" r.Condition + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + unique = append(unique, r) + } + return unique + } } // parseTestsForRelationships is a helper function to extract relationship tests from a column or its fields. @@ -756,7 +770,15 @@ func convertDbtMetricsToWrenMetrics(semanticData map[string]interface{}) []Metri if den, ok := typeParams["denominator"].(map[string]interface{}); ok { numName := getStringFromMap(num, "name", "") denName := getStringFromMap(den, "name", "") - wrenMetric.Aggregation = fmt.Sprintf("%s / %s", numName, denName) + if numData, ok := measureDataLookup[numName]; ok { + if denData, ok := measureDataLookup[denName]; ok { + numAgg := strings.ToUpper(getStringFromMap(numData, "agg", "sum")) + denAgg := strings.ToUpper(getStringFromMap(denData, "agg", "sum")) + numExpr := getStringFromMap(numData, "expr", numName) + denExpr := getStringFromMap(denData, "expr", denName) + wrenMetric.Aggregation = fmt.Sprintf("(%s(%s)) / (%s(%s))", numAgg, numExpr, denAgg, denExpr) + } + } } } case "derived": @@ -944,13 +966,16 @@ func getModelNameFromNodeKey(nodeKey string) string { return "" } +var refRegex = regexp.MustCompile(`ref\s*\(\s*['"]([^'"]+)['"]\s*\)`) + // parseRef extracts the model name from a dbt ref string. // e.g., "ref('stg_orders')" func parseRef(refStr string) string { - // Handle ref() with optional spaces and both quote types - re := regexp.MustCompile(`ref\s*\(\s*['"]([^'"]+)['"]\s*\)`) - matches := re.FindStringSubmatch(refStr) + // Use the precompiled regex to find matches. + matches := refRegex.FindStringSubmatch(refStr) if len(matches) > 1 { + // The first submatch (index 1) is the captured group, + // which is the model name we want. return matches[1] } return "" diff --git a/wren-launcher/commands/dbt/data_source.go b/wren-launcher/commands/dbt/data_source.go index 36416fc824..80943d4ac6 100644 --- a/wren-launcher/commands/dbt/data_source.go +++ b/wren-launcher/commands/dbt/data_source.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "os" "path/filepath" "strconv" "strings" @@ -184,27 +185,78 @@ func convertToMysqlDataSource(conn DbtConnection) (*WrenMysqlDataSource, error) // convertToBigQueryDataSource converts to BigQuery data source func convertToBigQueryDataSource(conn DbtConnection) (*WrenBigQueryDataSource, error) { - // Extract the keyfile content from the 'Additional' map - var keyfileJSON string - if kfj, exists := conn.Additional["keyfile_json"]; exists { - if kfjStr, ok := kfj.(string); ok { - keyfileJSON = kfjStr + method := strings.ToLower(strings.TrimSpace(conn.Method)) + var credentials string + + // Helper: validate JSON and base64 encode + encodeJSON := func(b []byte) (string, error) { + var js map[string]interface{} + if err := json.Unmarshal(b, &js); err != nil { + return "", fmt.Errorf("service account JSON is invalid: %w", err) } + return base64.StdEncoding.EncodeToString(b), nil } - if keyfileJSON == "" { - return nil, fmt.Errorf("keyfile_json not found or is empty in BigQuery connection details") - } - - // Validate that keyfile_json is valid JSON - var js map[string]interface{} - if err := json.Unmarshal([]byte(keyfileJSON), &js); err != nil { - return nil, fmt.Errorf("keyfile_json is not valid JSON: %w", err) + switch method { + case "service-account-json": + // Extract inline JSON from Additional["keyfile_json"] + var keyfileJSON string + if kfj, exists := conn.Additional["keyfile_json"]; exists { + if kfjStr, ok := kfj.(string); ok { + keyfileJSON = kfjStr + } + } + if keyfileJSON == "" { + return nil, fmt.Errorf("bigquery: method 'service-account-json' requires 'keyfile_json'") + } + enc, err := encodeJSON([]byte(keyfileJSON)) + if err != nil { + return nil, err + } + credentials = enc + case "service-account", "": + // Prefer structured field; fall back to Additional["keyfile"] + keyfilePath := strings.TrimSpace(conn.Keyfile) + if keyfilePath == "" { + if kf, ok := conn.Additional["keyfile"]; ok { + if kfStr, ok := kf.(string); ok { + keyfilePath = strings.TrimSpace(kfStr) + } + } + } + if keyfilePath == "" { + // If method was omitted (""), try as a fallback to inline json + if kfj, ok := conn.Additional["keyfile_json"]; ok { + if kfjStr, ok := kfj.(string); ok && kfjStr != "" { + enc, err := encodeJSON([]byte(kfjStr)) + if err != nil { + return nil, err + } + credentials = enc + } + } + if credentials == "" { + return nil, fmt.Errorf("bigquery: method 'service-account' requires 'keyfile' path") + } + } else { + b, err := os.ReadFile(keyfilePath) + if err != nil { + return nil, fmt.Errorf("failed to read keyfile '%s': %w", keyfilePath, err) + } + enc, err := encodeJSON(b) + if err != nil { + return nil, err + } + credentials = enc + } + case "oauth": + pterm.Warning.Println("bigquery: oauth auth method is not supported; skipping data source") + return nil, nil + default: + pterm.Warning.Printf("bigquery: unsupported auth method '%s'; supported: service-account, service-account-json\n", method) + return nil, nil } - // Base64 encode the keyfile JSON string for the credentials field - credentials := base64.StdEncoding.EncodeToString([]byte(keyfileJSON)) - ds := &WrenBigQueryDataSource{ Project: conn.Project, Dataset: conn.Dataset, @@ -389,13 +441,13 @@ func (ds *WrenBigQueryDataSource) GetType() string { // Validate implements DataSource interface func (ds *WrenBigQueryDataSource) Validate() error { - if ds.Project == "" { + if strings.TrimSpace(ds.Project) == "" { return fmt.Errorf("project_id cannot be empty") } - if ds.Dataset == "" { + if strings.TrimSpace(ds.Dataset) == "" { return fmt.Errorf("dataset_id cannot be empty") } - if ds.Credentials == "" { + if strings.TrimSpace(ds.Credentials) == "" { return fmt.Errorf("credentials cannot be empty") } return nil diff --git a/wren-launcher/commands/launch.go b/wren-launcher/commands/launch.go index ad7be2bad3..2abe9d56e7 100644 --- a/wren-launcher/commands/launch.go +++ b/wren-launcher/commands/launch.go @@ -11,7 +11,7 @@ import ( "strings" "time" - "github.com/Canner/WrenAI/wren-launcher/config" + "github.comcom/Canner/WrenAI/wren-launcher/config" utils "github.com/Canner/WrenAI/wren-launcher/utils" "github.com/common-nighthawk/go-figure" "github.com/manifoldco/promptui" @@ -187,6 +187,19 @@ func askForDbtTarget() (string, error) { return result, nil } +// *** NEW FUNCTION ADDED HERE *** +func askForIncludeStagingModels() (bool, error) { + prompt := promptui.Select{ + Label: "Include staging models (stg_*, staging_*)?", + Items: []string{"No", "Yes"}, + } + _, result, err := prompt.Run() + if err != nil { + return false, err + } + return result == "Yes", nil +} + func Launch() { // recover from panic defer func() { @@ -521,8 +534,15 @@ func processDbtProject(projectDir string) (string, error) { return "", err } - // Use the core conversion function from dbt package - result, err := DbtConvertProject(dbtProjectPath, targetDir, profileName, target, true, false) + // *** MODIFIED SECTION HERE *** + // Ask the user whether to include staging models + includeStagingModels, err := askForIncludeStagingModels() + if err != nil { + pterm.Warning.Println("Could not get staging model preference, defaulting to 'No'.") + } + + // Use the core conversion function from dbt package, passing the user's choice + result, err := DbtConvertProject(dbtProjectPath, targetDir, profileName, target, true, includeStagingModels) if err != nil { return "", fmt.Errorf("failed to convert dbt project: %w", err) } From 3d4de02df4ecbc6671e1e4a49c1c388336d92cf0 Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Mon, 18 Aug 2025 18:43:55 -0400 Subject: [PATCH 12/18] Upstream cleanup --- wren-launcher/commands/launch.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/wren-launcher/commands/launch.go b/wren-launcher/commands/launch.go index 2abe9d56e7..0cd2de7cbd 100644 --- a/wren-launcher/commands/launch.go +++ b/wren-launcher/commands/launch.go @@ -187,7 +187,6 @@ func askForDbtTarget() (string, error) { return result, nil } -// *** NEW FUNCTION ADDED HERE *** func askForIncludeStagingModels() (bool, error) { prompt := promptui.Select{ Label: "Include staging models (stg_*, staging_*)?", @@ -534,7 +533,6 @@ func processDbtProject(projectDir string) (string, error) { return "", err } - // *** MODIFIED SECTION HERE *** // Ask the user whether to include staging models includeStagingModels, err := askForIncludeStagingModels() if err != nil { From 5590bb8c5aad8800a5fd8793d2fd21a2b800dd08 Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Mon, 18 Aug 2025 18:48:05 -0400 Subject: [PATCH 13/18] More trivial updates --- wren-launcher/commands/dbt/converter.go | 3 +-- wren-launcher/commands/launch.go | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/wren-launcher/commands/dbt/converter.go b/wren-launcher/commands/dbt/converter.go index eedea934e6..2e60a6f746 100644 --- a/wren-launcher/commands/dbt/converter.go +++ b/wren-launcher/commands/dbt/converter.go @@ -524,7 +524,7 @@ func generateRelationships(manifestData map[string]interface{}) []Relationship { seen := make(map[string]struct{}, len(relationships)) var unique []Relationship for _, r := range relationships { - key := r.Name "|" r.JoinType "|" r.Condition + key := r.Name + "|" + r.JoinType + "|" + r.Condition if _, ok := seen[key]; ok { continue } @@ -533,7 +533,6 @@ func generateRelationships(manifestData map[string]interface{}) []Relationship { } return unique } -} // parseTestsForRelationships is a helper function to extract relationship tests from a column or its fields. func parseTestsForRelationships(fromModelName, columnName string, colMap map[string]interface{}) []Relationship { diff --git a/wren-launcher/commands/launch.go b/wren-launcher/commands/launch.go index 0cd2de7cbd..1f8ff2ede3 100644 --- a/wren-launcher/commands/launch.go +++ b/wren-launcher/commands/launch.go @@ -11,7 +11,7 @@ import ( "strings" "time" - "github.comcom/Canner/WrenAI/wren-launcher/config" + "github.com/Canner/WrenAI/wren-launcher/config" utils "github.com/Canner/WrenAI/wren-launcher/utils" "github.com/common-nighthawk/go-figure" "github.com/manifoldco/promptui" @@ -537,6 +537,7 @@ func processDbtProject(projectDir string) (string, error) { includeStagingModels, err := askForIncludeStagingModels() if err != nil { pterm.Warning.Println("Could not get staging model preference, defaulting to 'No'.") + includeStagingModels = false } // Use the core conversion function from dbt package, passing the user's choice From 97a7294182c57fe025ee07d5ccb5e54bd53beb4e Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Mon, 18 Aug 2025 21:28:39 -0400 Subject: [PATCH 14/18] Added support for relative pathing on keyfiles --- wren-launcher/commands/dbt/data_source.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/wren-launcher/commands/dbt/data_source.go b/wren-launcher/commands/dbt/data_source.go index 80943d4ac6..3742ac2fb0 100644 --- a/wren-launcher/commands/dbt/data_source.go +++ b/wren-launcher/commands/dbt/data_source.go @@ -4,7 +4,7 @@ import ( "encoding/base64" "encoding/json" "fmt" - "os" + "os" "path/filepath" "strconv" "strings" @@ -85,7 +85,8 @@ func convertConnectionToDataSource(conn DbtConnection, dbtHomePath, profileName, case "mysql": return convertToMysqlDataSource(conn) case "bigquery": - return convertToBigQueryDataSource(conn) + // Pass the dbtHomePath to the BigQuery converter + return convertToBigQueryDataSource(conn, dbtHomePath) default: // For unsupported database types, we can choose to ignore or return error // Here we choose to return nil and log a warning @@ -184,7 +185,7 @@ func convertToMysqlDataSource(conn DbtConnection) (*WrenMysqlDataSource, error) } // convertToBigQueryDataSource converts to BigQuery data source -func convertToBigQueryDataSource(conn DbtConnection) (*WrenBigQueryDataSource, error) { +func convertToBigQueryDataSource(conn DbtConnection, dbtHomePath string) (*WrenBigQueryDataSource, error) { method := strings.ToLower(strings.TrimSpace(conn.Method)) var credentials string @@ -239,9 +240,17 @@ func convertToBigQueryDataSource(conn DbtConnection) (*WrenBigQueryDataSource, e return nil, fmt.Errorf("bigquery: method 'service-account' requires 'keyfile' path") } } else { - b, err := os.ReadFile(keyfilePath) + // If keyfile path is not absolute, join it + // with the dbt project's home directory path. + resolvedKeyfilePath := keyfilePath + if !filepath.IsAbs(keyfilePath) && dbtHomePath != "" { + resolvedKeyfilePath = filepath.Join(dbtHomePath, keyfilePath) + } + + b, err := os.ReadFile(resolvedKeyfilePath) if err != nil { - return nil, fmt.Errorf("failed to read keyfile '%s': %w", keyfilePath, err) + // Update the error message to show the path that was attempted + return nil, fmt.Errorf("failed to read keyfile '%s': %w", resolvedKeyfilePath, err) } enc, err := encodeJSON(b) if err != nil { @@ -593,4 +602,4 @@ func (d *DefaultDataSource) MapType(sourceType string) string { default: return strings.ToLower(sourceType) } -} \ No newline at end of file +} From ef1f4fb0da069b6627e90cd3e74836dd36bebb72 Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Wed, 20 Aug 2025 09:54:51 -0400 Subject: [PATCH 15/18] Test refactor + go fmt test: Adjusted data source testing fix: Go fmt linting --- wren-launcher/commands/dbt/converter.go | 2 +- wren-launcher/commands/dbt/data_source.go | 2 +- .../commands/dbt/data_source_test.go | 487 ++++++++++-------- wren-launcher/commands/dbt/profiles.go | 4 +- wren-launcher/commands/dbt/wren_mdl.go | 16 +- 5 files changed, 289 insertions(+), 222 deletions(-) diff --git a/wren-launcher/commands/dbt/converter.go b/wren-launcher/commands/dbt/converter.go index 2e60a6f746..5f0c96e3ba 100644 --- a/wren-launcher/commands/dbt/converter.go +++ b/wren-launcher/commands/dbt/converter.go @@ -532,7 +532,7 @@ func generateRelationships(manifestData map[string]interface{}) []Relationship { unique = append(unique, r) } return unique - } +} // parseTestsForRelationships is a helper function to extract relationship tests from a column or its fields. func parseTestsForRelationships(fromModelName, columnName string, colMap map[string]interface{}) []Relationship { diff --git a/wren-launcher/commands/dbt/data_source.go b/wren-launcher/commands/dbt/data_source.go index 3742ac2fb0..4acfa26d41 100644 --- a/wren-launcher/commands/dbt/data_source.go +++ b/wren-launcher/commands/dbt/data_source.go @@ -240,7 +240,7 @@ func convertToBigQueryDataSource(conn DbtConnection, dbtHomePath string) (*WrenB return nil, fmt.Errorf("bigquery: method 'service-account' requires 'keyfile' path") } } else { - // If keyfile path is not absolute, join it + // If keyfile path is not absolute, join it // with the dbt project's home directory path. resolvedKeyfilePath := keyfilePath if !filepath.IsAbs(keyfilePath) && dbtHomePath != "" { diff --git a/wren-launcher/commands/dbt/data_source_test.go b/wren-launcher/commands/dbt/data_source_test.go index 2ca12e32c6..557ad8d41e 100644 --- a/wren-launcher/commands/dbt/data_source_test.go +++ b/wren-launcher/commands/dbt/data_source_test.go @@ -1,6 +1,9 @@ package dbt import ( + "encoding/base64" + "os" + "path/filepath" "testing" ) @@ -81,8 +84,8 @@ func TestFromDbtProfiles_Postgres(t *testing.T) { validatePostgresDataSource(t, ds, "test_db") } -func TestFromDbtProfiles_PostgresWithDbName(t *testing.T) { - // Test PostgreSQL connection conversion with dbname field (PostgreSQL specific) +func TestFromDbtProfiles_PostgresWithDefaultPort(t *testing.T) { + // Test PostgreSQL connection conversion when port is not specified profiles := &DbtProfiles{ Profiles: map[string]DbtProfile{ "test_profile": { @@ -225,28 +228,252 @@ func TestFromDbtProfiles_NilProfiles(t *testing.T) { } } -// Validator interface for data sources -type Validator interface { - Validate() error +func TestValidateAllDataSources(t *testing.T) { + // Test valid profiles + validProfiles := &DbtProfiles{ + Profiles: map[string]DbtProfile{ + "valid_project": { + Target: "dev", + Outputs: map[string]DbtConnection{ + "dev": { + Type: "postgres", + Host: "localhost", + Port: 5432, + Database: "test_db", + User: "user", + }, + }, + }, + }, + } + + err := ValidateAllDataSources(validProfiles) + if err != nil { + t.Errorf("ValidateAllDataSources failed for valid profiles: %v", err) + } + + // Test invalid profiles + invalidProfiles := &DbtProfiles{ + Profiles: map[string]DbtProfile{ + "invalid_project": { + Target: "dev", + Outputs: map[string]DbtConnection{ + "dev": { + Type: "postgres", + Host: "localhost", + // Missing required fields + }, + }, + }, + }, + } + + err = ValidateAllDataSources(invalidProfiles) + if err == nil { + t.Error("ValidateAllDataSources should fail for invalid profiles") + } } -// Helper function to test data source validation -func testDataSourceValidation(t *testing.T, testName string, validDS Validator, invalidDSCases []struct { - name string - ds Validator -}) { - t.Helper() +func TestFromDbtProfiles_BigQuery(t *testing.T) { + tempDir, err := os.MkdirTemp("", "test-dbt-home") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + t.Run("service-account-json", func(t *testing.T) { + keyfileContent := `{"type": "service_account", "project_id": "test-project", "private_key_id": "test-key-id", "private_key": "test-private-key", "client_email": "test-client-email", "client_id": "test-client-id", "auth_uri": "test-auth-uri", "token_uri": "test-token-uri", "auth_provider_x509_cert_url": "test-cert-url", "client_x509_cert_url": "test-client-cert-url"}` + profiles := &DbtProfiles{ + Profiles: map[string]DbtProfile{ + "test_profile": { + Target: "dev", + Outputs: map[string]DbtConnection{ + "dev": { + Type: "bigquery", + Method: "service-account-json", + Project: "test-project", + Dataset: "test-dataset", + Additional: map[string]interface{}{ + "keyfile_json": keyfileContent, + }, + }, + }, + }, + }, + } + + dataSources, err := GetActiveDataSources(profiles, "", "test_profile", "dev") + if err != nil { + t.Fatalf("GetActiveDataSources failed: %v", err) + } + + if len(dataSources) != 1 { + t.Fatalf("Expected 1 data source, got %d", len(dataSources)) + } + + ds, ok := dataSources[0].(*WrenBigQueryDataSource) + if !ok { + t.Fatalf("Expected WrenBigQueryDataSource, got %T", dataSources[0]) + } + + if ds.Project != "test-project" { + t.Errorf("Expected project 'test-project', got '%s'", ds.Project) + } + + if ds.Dataset != "test-dataset" { + t.Errorf("Expected dataset 'test-dataset', got '%s'", ds.Dataset) + } + + encodedContent, _ := base64.StdEncoding.DecodeString(ds.Credentials) + if string(encodedContent) != keyfileContent { + t.Errorf("Expected base64-encoded keyfile JSON content, got different content") + } + }) - t.Run(testName+" valid", func(t *testing.T) { - if err := validDS.Validate(); err != nil { - t.Errorf("Valid data source validation failed: %v", err) + t.Run("service-account-with-absolute-keyfile-path", func(t *testing.T) { + keyfileContent := `{"type": "service_account"}` + keyfilePath := filepath.Join(tempDir, "keyfile.json") + if err := os.WriteFile(keyfilePath, []byte(keyfileContent), 0644); err != nil { + t.Fatal(err) + } + + profiles := &DbtProfiles{ + Profiles: map[string]DbtProfile{ + "test_profile": { + Target: "dev", + Outputs: map[string]DbtConnection{ + "dev": { + Type: "bigquery", + Method: "service-account", + Project: "test-project", + Dataset: "test-dataset", + Keyfile: keyfilePath, + }, + }, + }, + }, + } + + dataSources, err := GetActiveDataSources(profiles, "", "test_profile", "dev") + if err != nil { + t.Fatalf("GetActiveDataSources failed: %v", err) + } + + if len(dataSources) != 1 { + t.Fatalf("Expected 1 data source, got %d", len(dataSources)) + } + + ds, ok := dataSources[0].(*WrenBigQueryDataSource) + if !ok { + t.Fatalf("Expected WrenBigQueryDataSource, got %T", dataSources[0]) + } + + encodedContent, _ := base64.StdEncoding.DecodeString(ds.Credentials) + if string(encodedContent) != keyfileContent { + t.Errorf("Expected base64-encoded keyfile content, got different content") } }) - for _, tt := range invalidDSCases { - t.Run(testName+" "+tt.name, func(t *testing.T) { - if err := tt.ds.Validate(); err == nil { - t.Errorf("Expected validation error for %s, but got none", tt.name) + t.Run("service-account-with-relative-keyfile-path", func(t *testing.T) { + dbtHomePath := tempDir + keyfileContent := `{"type": "service_account"}` + keyfilePath := "keys/keyfile.json" + fullKeyfilePath := filepath.Join(dbtHomePath, keyfilePath) + + if err := os.MkdirAll(filepath.Dir(fullKeyfilePath), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(fullKeyfilePath, []byte(keyfileContent), 0644); err != nil { + t.Fatal(err) + } + + profiles := &DbtProfiles{ + Profiles: map[string]DbtProfile{ + "test_profile": { + Target: "dev", + Outputs: map[string]DbtConnection{ + "dev": { + Type: "bigquery", + Method: "service-account", + Project: "test-project", + Dataset: "test-dataset", + Keyfile: keyfilePath, + }, + }, + }, + }, + } + + dataSources, err := GetActiveDataSources(profiles, dbtHomePath, "test_profile", "dev") + if err != nil { + t.Fatalf("GetActiveDataSources failed: %v", err) + } + + if len(dataSources) != 1 { + t.Fatalf("Expected 1 data source, got %d", len(dataSources)) + } + + ds, ok := dataSources[0].(*WrenBigQueryDataSource) + if !ok { + t.Fatalf("Expected WrenBigQueryDataSource, got %T", dataSources[0]) + } + + encodedContent, _ := base64.StdEncoding.DecodeString(ds.Credentials) + if string(encodedContent) != keyfileContent { + t.Errorf("Expected base64-encoded keyfile content, got different content") + } + }) +} + +func TestBigQueryDataSourceValidation(t *testing.T) { + tests := []struct { + name string + ds *WrenBigQueryDataSource + wantErr bool + }{ + { + name: "valid", + ds: &WrenBigQueryDataSource{ + Project: "test-project", + Dataset: "test-dataset", + Credentials: "dGVzdC1jcmVkZW50aWFscw==", // "test-credentials" + }, + wantErr: false, + }, + { + name: "invalid - missing project", + ds: &WrenBigQueryDataSource{ + Project: "", + Dataset: "test-dataset", + Credentials: "dGVzdC1jcmVkZW50aWFscw==", + }, + wantErr: true, + }, + { + name: "invalid - missing dataset", + ds: &WrenBigQueryDataSource{ + Project: "test-project", + Dataset: "", + Credentials: "dGVzdC1jcmVkZW50aWFscw==", + }, + wantErr: true, + }, + { + name: "invalid - missing credentials", + ds: &WrenBigQueryDataSource{ + Project: "test-project", + Dataset: "test-dataset", + Credentials: "", + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.ds.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) } }) } @@ -474,216 +701,56 @@ func TestGetDataSourceByType(t *testing.T) { } } -func TestValidateAllDataSources(t *testing.T) { - // Test valid profiles - validProfiles := &DbtProfiles{ - Profiles: map[string]DbtProfile{ - "valid_project": { - Target: "dev", - Outputs: map[string]DbtConnection{ - "dev": { - Type: "postgres", - Host: "localhost", - Port: 5432, - Database: "test_db", - User: "user", - }, - }, - }, - }, - } - - err := ValidateAllDataSources(validProfiles) - if err != nil { - t.Errorf("ValidateAllDataSources failed for valid profiles: %v", err) - } - - // Test invalid profiles - invalidProfiles := &DbtProfiles{ - Profiles: map[string]DbtProfile{ - "invalid_project": { - Target: "dev", - Outputs: map[string]DbtConnection{ - "dev": { - Type: "postgres", - Host: "localhost", - // Missing required fields - }, - }, - }, - }, - } - - err = ValidateAllDataSources(invalidProfiles) - if err == nil { - t.Error("ValidateAllDataSources should fail for invalid profiles") - } -} - -func TestFromDbtProfiles_BigQuery(t *testing.T) { - t.Run("service-account", func(t *testing.T) { - profiles := &DbtProfiles{ - Profiles: map[string]DbtProfile{ - "test_profile": { - Target: "dev", - Outputs: map[string]DbtConnection{ - "dev": { - Type: "bigquery", - Method: "service-account", - Project: "test-project", - Dataset: "test-dataset", - Keyfile: "/path/to/key.json", - }, - }, - }, - }, - } - - dataSources, err := FromDbtProfiles(profiles) - if err != nil { - t.Fatalf("FromDbtProfiles failed: %v", err) - } - - if len(dataSources) != 1 { - t.Fatalf("Expected 1 data source, got %d", len(dataSources)) - } - - ds, ok := dataSources[0].(*WrenBigQueryDataSource) - if !ok { - t.Fatalf("Expected WrenBigQueryDataSource, got %T", dataSources[0]) - } - - if ds.Project != "test-project" { - t.Errorf("Expected project 'test-project', got '%s'", ds.Project) - } - - if ds.Method != "service-account" { - t.Errorf("Expected method 'service-account', got '%s'", ds.Method) - } - - if ds.Keyfile != "/path/to/key.json" { - t.Errorf("Expected keyfile '/path/to/key.json', got '%s'", ds.Keyfile) - } - }) - - t.Run("service-account-json", func(t *testing.T) { - keyfileContent := `{"type": "service_account"}` - profiles := &DbtProfiles{ - Profiles: map[string]DbtProfile{ - "test_profile": { - Target: "dev", - Outputs: map[string]DbtConnection{ - "dev": { - Type: "bigquery", - Method: "service-account-json", - Project: "test-project", - Dataset: "test-dataset", - Additional: map[string]interface{}{ - "keyfile_json": keyfileContent, - }, - }, - }, - }, - }, - } - - dataSources, err := FromDbtProfiles(profiles) - if err != nil { - t.Fatalf("FromDbtProfiles failed: %v", err) - } - - if len(dataSources) != 1 { - t.Fatalf("Expected 1 data source, got %d", len(dataSources)) - } - - ds, ok := dataSources[0].(*WrenBigQueryDataSource) - if !ok { - t.Fatalf("Expected WrenBigQueryDataSource, got %T", dataSources[0]) - } - - if ds.Project != "test-project" { - t.Errorf("Expected project 'test-project', got '%s'", ds.Project) - } - - if ds.Method != "service-account-json" { - t.Errorf("Expected method 'service-account-json', got '%s'", ds.Method) - } - - if ds.KeyfileJSON != keyfileContent { - t.Errorf("Expected keyfile_json content, got '%s'", ds.KeyfileJSON) - } - }) -} - -func TestBigQueryDataSourceValidation(t *testing.T) { +func TestMapType(t *testing.T) { tests := []struct { - name string - ds *WrenBigQueryDataSource - wantErr bool + name string + dataSource DataSource + sourceType string + want string }{ { - name: "valid service-account", - ds: &WrenBigQueryDataSource{ - Project: "test-project", - Dataset: "test-dataset", - Method: "service-account", - Keyfile: "/path/to/key.json", - }, - wantErr: false, + name: "BigQuery INT64 to integer", + dataSource: &WrenBigQueryDataSource{}, + sourceType: "INT64", + want: "integer", }, { - name: "valid service-account-json", - ds: &WrenBigQueryDataSource{ - Project: "test-project", - Dataset: "test-dataset", - Method: "service-account-json", - KeyfileJSON: `{"type": "service_account"}`, - }, - wantErr: false, + name: "BigQuery STRING to varchar", + dataSource: &WrenBigQueryDataSource{}, + sourceType: "STRING", + want: "varchar", }, { - name: "invalid - missing project", - ds: &WrenBigQueryDataSource{ - Dataset: "test-dataset", - Method: "service-account", - Keyfile: "/path/to/key.json", - }, - wantErr: true, + name: "LocalFile INTEGER to integer", + dataSource: &WrenLocalFileDataSource{}, + sourceType: "INTEGER", + want: "integer", }, { - name: "invalid - missing dataset", - ds: &WrenBigQueryDataSource{ - Project: "test-project", - Method: "service-account", - Keyfile: "/path/to/key.json", - }, - wantErr: true, + name: "LocalFile VARCHAR to varchar", + dataSource: &WrenLocalFileDataSource{}, + sourceType: "VARCHAR", + want: "varchar", }, { - name: "invalid - unsupported oauth", - ds: &WrenBigQueryDataSource{ - Project: "test-project", - Dataset: "test-dataset", - Method: "oauth", - }, - wantErr: true, + name: "DefaultDataSource int to integer", + dataSource: &DefaultDataSource{}, + sourceType: "int", + want: "integer", }, { - name: "invalid - missing keyfile for service-account", - ds: &WrenBigQueryDataSource{ - Project: "test-project", - Dataset: "test-dataset", - Method: "service-account", - }, - wantErr: true, + name: "PostgresDataSource (no mapping)", + dataSource: &WrenPostgresDataSource{}, + sourceType: "unknown_type", + want: "unknown_type", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := tt.ds.Validate() - if (err != nil) != tt.wantErr { - t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + got := tt.dataSource.MapType(tt.sourceType) + if got != tt.want { + t.Errorf("MapType(%s) = %s; want %s", tt.sourceType, got, tt.want) } }) } diff --git a/wren-launcher/commands/dbt/profiles.go b/wren-launcher/commands/dbt/profiles.go index b8f68edb32..62ca16746a 100644 --- a/wren-launcher/commands/dbt/profiles.go +++ b/wren-launcher/commands/dbt/profiles.go @@ -8,7 +8,7 @@ type DbtProfiles struct { // DbtProfile represents a single profile in profiles.yml type DbtProfile struct { - Target string `yaml:"target" json:"target"` + Target string `yaml:"target" json:"target"` Outputs map[string]DbtConnection `yaml:"outputs" json:"outputs"` } @@ -33,7 +33,7 @@ type DbtConnection struct { KeepAlive bool `yaml:"keepalive,omitempty" json:"keepalive,omitempty"` // Postgres SearchPath string `yaml:"search_path,omitempty" json:"search_path,omitempty"` // Postgres - SSLMode string `yaml:"sslmode,omitempty" json:"sslmode,omitempty"` // Postgres + SSLMode string `yaml:"sslmode,omitempty" json:"sslmode,omitempty"` // Postgres SslDisable bool `yaml:"ssl_disable,omitempty" json:"ssl_disable,omitempty"` // MySQL diff --git a/wren-launcher/commands/dbt/wren_mdl.go b/wren-launcher/commands/dbt/wren_mdl.go index ca77ad0524..60c084f9be 100644 --- a/wren-launcher/commands/dbt/wren_mdl.go +++ b/wren-launcher/commands/dbt/wren_mdl.go @@ -2,14 +2,14 @@ package dbt // WrenMDLManifest represents the complete Wren MDL structure type WrenMDLManifest struct { - Catalog string `json:"catalog"` - Schema string `json:"schema"` - EnumDefinitions []EnumDefinition `json:"enumDefinitions,omitempty"` - Models []WrenModel `json:"models"` - Relationships []Relationship `json:"relationships"` - Metrics []Metric `json:"metrics,omitempty"` - Views []View `json:"views"` - DataSources string `json:"dataSources,omitempty"` + Catalog string `json:"catalog"` + Schema string `json:"schema"` + EnumDefinitions []EnumDefinition `json:"enumDefinitions,omitempty"` + Models []WrenModel `json:"models"` + Relationships []Relationship `json:"relationships"` + Metrics []Metric `json:"metrics,omitempty"` + Views []View `json:"views"` + DataSources string `json:"dataSources,omitempty"` } // EnumDefinition represents a named list of values that can be used by columns. From d5826f6f60e9e9562a2137fe74c03ce5f9b96e4e Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Sun, 14 Sep 2025 20:30:19 -0400 Subject: [PATCH 16/18] chore: Lint updates chore: Lint updates --- wren-launcher/commands/dbt/converter.go | 769 ++++++++++-------- wren-launcher/commands/dbt/data_source.go | 25 +- .../commands/dbt/data_source_test.go | 18 +- 3 files changed, 468 insertions(+), 344 deletions(-) diff --git a/wren-launcher/commands/dbt/converter.go b/wren-launcher/commands/dbt/converter.go index 5f0c96e3ba..acc1b3aa16 100644 --- a/wren-launcher/commands/dbt/converter.go +++ b/wren-launcher/commands/dbt/converter.go @@ -280,62 +280,46 @@ func handleLocalhostForContainer(host string) string { return host } -// ConvertDbtCatalogToWrenMDL converts dbt catalog.json to Wren MDL format +// ConvertDbtCatalogToWrenMDL is the main function to convert a dbt catalog into a Wren MDL manifest. +// It orchestrates the reading of dbt artifacts and processes each dbt node to convert it into a Wren model. func ConvertDbtCatalogToWrenMDL(catalogPath string, dataSource DataSource, manifestPath string, semanticManifestPath string, includeStagingModels bool) (*WrenMDLManifest, error) { - // Read and parse the catalog.json file - data, err := os.ReadFile(catalogPath) // #nosec G304 -- catalogPath is controlled by application + // --- 1. Read and Parse All Necessary DBT Artifact Files --- + + // Read and unmarshal the primary catalog.json file. + catalogBytes, err := os.ReadFile(filepath.Clean(catalogPath)) if err != nil { return nil, fmt.Errorf("failed to read catalog file %s: %w", catalogPath, err) } - var catalogData map[string]interface{} - if err := json.Unmarshal(data, &catalogData); err != nil { + if err := json.Unmarshal(catalogBytes, &catalogData); err != nil { return nil, fmt.Errorf("failed to parse catalog JSON: %w", err) } - // Parse manifest.json for descriptions and relationships (optional) + // Read and unmarshal the manifest.json file, which contains rich metadata. var manifestData map[string]interface{} if manifestPath != "" { pterm.Info.Printf("Reading manifest.json for descriptions and relationships from: %s\n", manifestPath) - manifestBytes, err := os.ReadFile(manifestPath) // #nosec G304 -- manifestPath is controlled by application + manifestBytes, err := os.ReadFile(filepath.Clean(manifestPath)) // #nosec G304 -- manifestPath is controlled by application if err != nil { - pterm.Warning.Printf("Warning: Failed to read manifest file %s: %v\n", manifestPath, err) - } else { - if err := json.Unmarshal(manifestBytes, &manifestData); err != nil { - pterm.Warning.Printf("Warning: Failed to parse manifest JSON: %v\n", err) - } + pterm.Warning.Printf("Could not read manifest file %s: %v. Descriptions and relationships will be missing.\n", manifestPath, err) + } else if err := json.Unmarshal(manifestBytes, &manifestData); err != nil { + pterm.Warning.Printf("Could not parse manifest file %s: %v. Descriptions and relationships will be missing.\n", manifestPath, err) } } - // Parse semantic_manifest.json for metrics and primary keys (optional and robust) + // Read and unmarshal the semantic_manifest.json file for metrics and primary keys. var semanticManifestData map[string]interface{} if semanticManifestPath != "" { - pterm.Info.Printf("Reading semantic_manifest.json for metrics and primary keys from: %s\n", semanticManifestPath) - semanticBytes, err := os.ReadFile(semanticManifestPath) + semanticBytes, err := os.ReadFile(filepath.Clean(semanticManifestPath)) if err != nil { - pterm.Warning.Printf("Warning: Could not read semantic_manifest.json: %v\n", err) - pterm.Warning.Println("Skipping metric and primary key conversion.") - } else { - if err := json.Unmarshal(semanticBytes, &semanticManifestData); err != nil { - pterm.Warning.Printf("Warning: Failed to parse semantic_manifest.json: %v\n", err) - pterm.Warning.Println("Skipping metric and primary key conversion.") - semanticManifestData = nil - } + pterm.Warning.Printf("Could not read semantic_manifest.json: %v. Metrics and primary keys will be missing.\n", err) + } else if err := json.Unmarshal(semanticBytes, &semanticManifestData); err != nil { + pterm.Warning.Printf("Could not parse semantic_manifest.json: %v. Metrics and primary keys will be missing.\n", err) } } - // Extract nodes from catalog - nodesValue, exists := catalogData["nodes"] - if !exists { - return nil, fmt.Errorf("no 'nodes' section found in catalog") - } - - nodesMap, ok := nodesValue.(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("invalid 'nodes' format in catalog") - } + // --- 2. Initialize Wren Manifest and Pre-process Metadata --- - // Initialize Wren MDL manifest manifest := &WrenMDLManifest{ Catalog: "wren", Schema: "public", @@ -347,121 +331,67 @@ func ConvertDbtCatalogToWrenMDL(catalogPath string, dataSource DataSource, manif DataSources: dataSource.GetType(), } - // Maps to store pre-processed information + // Create lookup maps to store pre-processed information for quick access. enumValueToNameMap := make(map[string]string) columnToEnumNameMap := make(map[string]string) columnToNotNullMap := make(map[string]bool) modelToPrimaryKeyMap := make(map[string]string) - // Pre-process manifest to find all tests (enums, not_null) + // Pre-process the manifest to extract test data (enums, not-null constraints). if manifestData != nil { - if nodes, ok := manifestData["nodes"].(map[string]interface{}); ok { - for nodeKey, nodeValue := range nodes { - nodeMap, ok := nodeValue.(map[string]interface{}) - if !ok { - continue - } - - // Handle tests on model columns (including structs) - if strings.HasPrefix(nodeKey, "model.") { - modelName := getModelNameFromNodeKey(nodeKey) - if modelName == "" { - continue - } - if columns, ok := nodeMap["columns"].(map[string]interface{}); ok { - for columnName, colData := range columns { - if colMap, ok := colData.(map[string]interface{}); ok { - processColumnForTests(nodeKey, modelName, columnName, colMap, &manifest.EnumDefinitions, enumValueToNameMap, columnToEnumNameMap, columnToNotNullMap) - } - } - } - } - - // Handle compiled test nodes for simple columns - if strings.HasPrefix(nodeKey, "test.") { - if testMeta, ok := nodeMap["test_metadata"].(map[string]interface{}); ok { - testName := getStringFromMap(testMeta, "name", "") - attachedNodeID := getStringFromMap(nodeMap, "attached_node", "") - columnName := getStringFromMap(nodeMap, "column_name", "") - modelName := getModelNameFromNodeKey(attachedNodeID) - - if attachedNodeID != "" && columnName != "" && modelName != "" { - columnKey := fmt.Sprintf("%s.%s", attachedNodeID, columnName) - - if testName == "not_null" { - columnToNotNullMap[columnKey] = true - } - - if testName == "accepted_values" { - if kwargs, ok := testMeta["kwargs"].(map[string]interface{}); ok { - if values, ok := kwargs["values"].([]interface{}); ok && len(values) > 0 { - createOrLinkEnum(modelName, columnName, columnKey, values, &manifest.EnumDefinitions, enumValueToNameMap, columnToEnumNameMap) - } - } - } - } - } - } - } - } + preprocessManifestForTests(manifestData, &manifest.EnumDefinitions, enumValueToNameMap, columnToEnumNameMap, columnToNotNullMap) } - // Pre-process semantic manifest for primary keys + // Pre-process the semantic manifest to extract primary key information. if semanticManifestData != nil { - if semanticModels, ok := semanticManifestData["semantic_models"].([]interface{}); ok { - for _, sm := range semanticModels { - if smMap, ok := sm.(map[string]interface{}); ok { - var modelName string - if nr, ok := smMap["node_relation"].(map[string]interface{}); ok { - modelName = getStringFromMap(nr, "alias", "") - } + preprocessSemanticManifestForPrimaryKeys(semanticManifestData, modelToPrimaryKeyMap) + } - if entities, ok := smMap["entities"].([]interface{}); ok { - for _, entity := range entities { - if entityMap, ok := entity.(map[string]interface{}); ok { - if getStringFromMap(entityMap, "type", "") == "primary" { - pk := getStringFromMap(entityMap, "expr", "") - if modelName != "" && pk != "" { - modelToPrimaryKeyMap[modelName] = pk - } - } - } - } - } - } - } - } + // --- 3. Convert dbt Nodes to Wren Models --- + + nodesValue, exists := catalogData["nodes"] + if !exists { + return nil, fmt.Errorf("no 'nodes' section found in catalog") + } + nodesMap, ok := nodesValue.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid 'nodes' format in catalog") } - // Convert each dbt model to Wren model + // Iterate through each node in the catalog and convert it to a Wren model. for nodeKey, nodeValue := range nodesMap { nodeMap, ok := nodeValue.(map[string]interface{}) if !ok { continue } + // We are only interested in nodes that represent dbt models. if !strings.HasPrefix(nodeKey, "model.") { continue } - if !includeStagingModels { - mn := getModelNameFromNodeKey(nodeKey) - if strings.HasPrefix(mn, "stg_") || strings.HasPrefix(mn, "staging_") { - continue - } + + // Skip staging models if the user has opted to exclude them. + modelName := getModelNameFromNodeKey(nodeKey) + if !includeStagingModels && (strings.HasPrefix(modelName, "stg_") || strings.HasPrefix(modelName, "staging_")) { + continue } + + // Perform the conversion for the single node. model, err := convertDbtNodeToWrenModel(nodeKey, nodeMap, dataSource, manifestData, columnToEnumNameMap, columnToNotNullMap, modelToPrimaryKeyMap) if err != nil { - pterm.Warning.Printf("Warning: Failed to convert model %s: %v\n", nodeKey, err) + pterm.Warning.Printf("Failed to convert model %s: %v\n", nodeKey, err) continue } manifest.Models = append(manifest.Models, *model) } - // Generate relationships from manifest.json + // --- 4. Generate Relationships and Metrics --- + + // Generate relationships between models based on the dbt manifest. if manifestData != nil { manifest.Relationships = generateRelationships(manifestData) } - // Generate metrics from semantic_manifest.json, only if data is available + // Generate metrics from the semantic manifest. if semanticManifestData != nil { manifest.Metrics = convertDbtMetricsToWrenMetrics(semanticManifestData) } @@ -469,6 +399,92 @@ func ConvertDbtCatalogToWrenMDL(catalogPath string, dataSource DataSource, manif return manifest, nil } +// preprocessManifestForTests extracts information from dbt tests (like 'not_null' and 'accepted_values') +// and populates maps that will be used later during model conversion. +func preprocessManifestForTests(manifestData map[string]interface{}, enums *[]EnumDefinition, enumValueToNameMap, columnToEnumNameMap map[string]string, columnToNotNullMap map[string]bool) { + nodes, ok := manifestData["nodes"].(map[string]interface{}) + if !ok { + return + } + + for nodeKey, nodeValue := range nodes { + nodeMap, ok := nodeValue.(map[string]interface{}) + if !ok { + continue + } + + // Process tests defined directly on model columns. + if strings.HasPrefix(nodeKey, "model.") { + modelName := getModelNameFromNodeKey(nodeKey) + if columns, ok := nodeMap["columns"].(map[string]interface{}); ok { + for columnName, colData := range columns { + if colMap, ok := colData.(map[string]interface{}); ok { + processColumnForTests(nodeKey, modelName, columnName, colMap, enums, enumValueToNameMap, columnToEnumNameMap, columnToNotNullMap) + } + } + } + } + + // Process compiled test nodes which are separate entries in the manifest. + if strings.HasPrefix(nodeKey, "test.") { + testMeta, _ := nodeMap["test_metadata"].(map[string]interface{}) + testName := getStringFromMap(testMeta, "name", "") + attachedNodeID := getStringFromMap(nodeMap, "attached_node", "") + columnName := getStringFromMap(nodeMap, "column_name", "") + + if attachedNodeID != "" && columnName != "" { + columnKey := fmt.Sprintf("%s.%s", attachedNodeID, columnName) + modelName := getModelNameFromNodeKey(attachedNodeID) + + if testName == "not_null" { + columnToNotNullMap[columnKey] = true + } + + if testName == "accepted_values" { + if kwargs, ok := testMeta["kwargs"].(map[string]interface{}); ok { + if values, ok := kwargs["values"].([]interface{}); ok && len(values) > 0 { + createOrLinkEnum(modelName, columnName, columnKey, values, enums, enumValueToNameMap, columnToEnumNameMap) + } + } + } + } + } + } +} + +// preprocessSemanticManifestForPrimaryKeys extracts primary key information from the semantic manifest. +func preprocessSemanticManifestForPrimaryKeys(semanticData map[string]interface{}, modelToPrimaryKeyMap map[string]string) { + semanticModels, ok := semanticData["semantic_models"].([]interface{}) + if !ok { + return + } + + for _, sm := range semanticModels { + smMap, ok := sm.(map[string]interface{}) + if !ok { + continue + } + + var modelName string + if nr, ok := smMap["node_relation"].(map[string]interface{}); ok { + modelName = getStringFromMap(nr, "alias", "") + } + + if entities, ok := smMap["entities"].([]interface{}); ok { + for _, entity := range entities { + if entityMap, ok := entity.(map[string]interface{}); ok { + if getStringFromMap(entityMap, "type", "") == "primary" { + pk := getStringFromMap(entityMap, "expr", "") + if modelName != "" && pk != "" { + modelToPrimaryKeyMap[modelName] = pk + } + } + } + } + } + } +} + // generateRelationships iterates through the manifest and creates relationship definitions. func generateRelationships(manifestData map[string]interface{}) []Relationship { var relationships []Relationship @@ -583,10 +599,8 @@ func extractRelationshipsFromTests(fromModelName, fromColumnName string, tests [ return relationships } -// createOrLinkEnum is a helper to de-duplicate and manage enum creation. -func createOrLinkEnum(modelName, columnName, columnKey string, values []interface{}, - allEnums *[]EnumDefinition, enumValueToNameMap, columnToEnumNameMap map[string]string) { - +// createOrLinkEnum is a helper to de-duplicate and manage enum creation based on 'accepted_values' tests. +func createOrLinkEnum(modelName, columnName, columnKey string, values []interface{}, allEnums *[]EnumDefinition, enumValueToNameMap, columnToEnumNameMap map[string]string) { var strValues []string for _, v := range values { if s, ok := v.(string); ok { @@ -617,19 +631,17 @@ func createOrLinkEnum(modelName, columnName, columnKey string, values []interfac columnToEnumNameMap[columnKey] = enumName } -// processColumnForTests recursively finds tests in embedded column definitions. -func processColumnForTests(nodeKey, modelName, columnName string, colMap map[string]interface{}, - allEnums *[]EnumDefinition, enumValueToNameMap, columnToEnumNameMap map[string]string, columnToNotNullMap map[string]bool) { - - // Helper to handle the actual test processing +// processColumnForTests finds tests in a column definition (including nested fields) and processes them. +func processColumnForTests(nodeKey, modelName, columnName string, colMap map[string]interface{}, allEnums *[]EnumDefinition, enumValueToNameMap, columnToEnumNameMap map[string]string, columnToNotNullMap map[string]bool) { + // Helper to handle the actual test processing for a given column/field processTests := func(currentColumnKey, currentColumnName string, tests []interface{}) { for _, test := range tests { - // Handle not_null test (string) + // Handle not_null test (string format) if testStr, ok := test.(string); ok && testStr == "not_null" { columnToNotNullMap[currentColumnKey] = true } - // Handle accepted_values test (map) + // Handle tests in map format (e.g., accepted_values) if testMap, ok := test.(map[string]interface{}); ok { if accepted, ok := testMap["accepted_values"].(map[string]interface{}); ok { if values, ok := accepted["values"].([]interface{}); ok && len(values) > 0 { @@ -640,13 +652,13 @@ func processColumnForTests(nodeKey, modelName, columnName string, colMap map[str } } - // Case 1: Tests are directly on the column (for structs). + // Case 1: Tests are directly on the column itself. if tests, ok := colMap["tests"].([]interface{}); ok { columnKey := fmt.Sprintf("%s.%s", nodeKey, columnName) processTests(columnKey, columnName, tests) } - // Case 2: Column is a struct with tests on its fields. + // Case 2: The column is a struct, and tests are on its fields. if fields, ok := colMap["fields"].([]interface{}); ok { for _, fieldData := range fields { if fieldMap, ok := fieldData.(map[string]interface{}); ok { @@ -655,7 +667,8 @@ func processColumnForTests(nodeKey, modelName, columnName string, colMap map[str continue } if tests, ok := fieldMap["tests"].([]interface{}); ok { - columnKey := fmt.Sprintf("%s.%s", nodeKey, fieldName) // The key is based on the field name + // The unique key for a field is based on the field name. + columnKey := fmt.Sprintf("%s.%s", nodeKey, fieldName) processTests(columnKey, fieldName, tests) } } @@ -663,195 +676,288 @@ func processColumnForTests(nodeKey, modelName, columnName string, colMap map[str } } -// convertDbtMetricsToWrenMetrics converts dbt metrics from semantic manifest to Wren MDL format +// convertDbtMetricsToWrenMetrics converts dbt metrics from the semantic manifest into the Wren MDL format. +// It serves as the main entry point for metric conversion, orchestrating the creation of lookup tables +// and processing each metric definition. func convertDbtMetricsToWrenMetrics(semanticData map[string]interface{}) []Metric { var wrenMetrics []Metric + + // --- 1. Pre-process semantic models to build fast lookup maps --- + // These maps are essential for quickly finding the model a measure belongs to and its details. + measureToModelMap, measureDataLookup := buildMeasureLookups(semanticData) + + // --- 2. Iterate through each metric and convert it --- + metrics, ok := semanticData["metrics"].([]interface{}) + if !ok { + // If there's no 'metrics' array, there's nothing to do. + return wrenMetrics + } + + for _, m := range metrics { + metricMap, ok := m.(map[string]interface{}) + if !ok { + continue // Skip if the item is not a valid map. + } + + // --- 3. Extract basic metric information --- + metricName := getStringFromMap(metricMap, "name", "") + if metricName == "" { + continue // A metric must have a name. + } + + wrenMetric := Metric{ + Name: metricName, + DisplayName: getStringFromMap(metricMap, "label", metricName), + Description: getStringFromMap(metricMap, "description", ""), + } + + typeParams, _ := metricMap["type_params"].(map[string]interface{}) + + // --- 4. Determine the base model and time dimensions for the metric --- + baseModel := findBaseModelForMetric(typeParams, measureToModelMap) + if baseModel == "" { + pterm.Warning.Printf("Could not find a parent model for metric '%s'\n", metricName) + continue // Skip metric if we can't associate it with a model. + } + + wrenMetric.Models = []string{baseModel} + wrenMetric.Dimensions = findTimeDimensionsForModel(semanticData, baseModel) + + // --- 5. Build the specific aggregation expression based on the metric type --- + metricType := getStringFromMap(metricMap, "type", "") + wrenMetric.Aggregation = buildAggregationExpression(metricType, typeParams, measureDataLookup) + + // --- 6. Final validation before adding to the list --- + // A metric is only valid if it has a base model and a valid aggregation expression. + if wrenMetric.Aggregation != "" && len(wrenMetric.Models) > 0 { + wrenMetrics = append(wrenMetrics, wrenMetric) + } + } + + return wrenMetrics +} + +// buildMeasureLookups preprocesses the semantic models to create two essential maps: +// 1. measureToModelMap: Maps a measure's name to the name of the model it belongs to. +// 2. measureDataLookup: Maps a measure's name to its full data map for easy access to properties like `agg` and `expr`. +func buildMeasureLookups(semanticData map[string]interface{}) (map[string]string, map[string]map[string]interface{}) { measureToModelMap := make(map[string]string) - measureDataLookup := make(map[string]map[string]interface{}) // measureName -> measureData - - // First, build lookup tables for all measures - if semanticModels, ok := semanticData["semantic_models"].([]interface{}); ok { - for _, sm := range semanticModels { - if smMap, ok := sm.(map[string]interface{}); ok { - modelName := getStringFromMap(smMap, "name", "") - if modelName == "" { - continue - } - if measures, ok := smMap["measures"].([]interface{}); ok { - for _, m := range measures { - if measureMap, ok := m.(map[string]interface{}); ok { - measureName := getStringFromMap(measureMap, "name", "") - if measureName != "" { - measureToModelMap[measureName] = modelName - measureDataLookup[measureName] = measureMap - } - } + measureDataLookup := make(map[string]map[string]interface{}) + + semanticModels, ok := semanticData["semantic_models"].([]interface{}) + if !ok { + return measureToModelMap, measureDataLookup + } + + for _, sm := range semanticModels { + smMap, ok := sm.(map[string]interface{}) + if !ok { + continue + } + + modelName := getStringFromMap(smMap, "name", "") + if modelName == "" { + continue + } + + if measures, ok := smMap["measures"].([]interface{}); ok { + for _, m := range measures { + if measureMap, ok := m.(map[string]interface{}); ok { + measureName := getStringFromMap(measureMap, "name", "") + if measureName != "" { + measureToModelMap[measureName] = modelName + measureDataLookup[measureName] = measureMap } } } } } + return measureToModelMap, measureDataLookup +} - // Now, iterate through the metrics and build Wren metrics - if metrics, ok := semanticData["metrics"].([]interface{}); ok { - for _, m := range metrics { - if metricMap, ok := m.(map[string]interface{}); ok { - metricName := getStringFromMap(metricMap, "name", "") - metricLabel := getStringFromMap(metricMap, "label", metricName) - metricDesc := getStringFromMap(metricMap, "description", "") - metricType := getStringFromMap(metricMap, "type", "") - - wrenMetric := Metric{ - Name: metricName, - DisplayName: metricLabel, - Description: metricDesc, +// findBaseModelForMetric identifies the underlying base model for a given metric +// by looking at its "input_measures". +func findBaseModelForMetric(typeParams map[string]interface{}, measureToModelMap map[string]string) string { + inputMeasuresValue, ok := typeParams["input_measures"] + if !ok { + // Fallback for simple metrics that use "measure" instead of "input_measures" + if measureValue, ok := typeParams["measure"]; ok { + if measureMap, ok := measureValue.(map[string]interface{}); ok { + measureName := getStringFromMap(measureMap, "name", "") + if model, exists := measureToModelMap[measureName]; exists { + return model } + } + } + return "" + } - typeParams, _ := metricMap["type_params"].(map[string]interface{}) - - // Find the underlying model and dimensions - var baseModel string - var timeDimensions []string - if inputMeasuresValue, ok := typeParams["input_measures"]; ok { - if inputMeasuresList, ok := inputMeasuresValue.([]interface{}); ok && len(inputMeasuresList) > 0 { - // Find the model this metric is based on - for _, inputMeasure := range inputMeasuresList { - if imMap, ok := inputMeasure.(map[string]interface{}); ok { - imName := getStringFromMap(imMap, "name", "") - if model, exists := measureToModelMap[imName]; exists { - baseModel = model - break // Assume all measures for a metric come from the same model - } - } - } - if baseModel == "" { - pterm.Warning.Printf("Could not find a parent model for metric '%s'\n", metricName) - } - } - } + inputMeasuresList, ok := inputMeasuresValue.([]interface{}) + if !ok || len(inputMeasuresList) == 0 { + return "" + } - // Find time dimensions from the semantic model - if baseModel != "" { - wrenMetric.Models = []string{baseModel} - if semanticModels, ok := semanticData["semantic_models"].([]interface{}); ok { - for _, sm := range semanticModels { - if smMap, ok := sm.(map[string]interface{}); ok { - if getStringFromMap(smMap, "name", "") == baseModel { - if dims, ok := smMap["dimensions"].([]interface{}); ok { - for _, d := range dims { - if dimMap, ok := d.(map[string]interface{}); ok { - if getStringFromMap(dimMap, "type", "") == "time" { - timeDimensions = append(timeDimensions, getStringFromMap(dimMap, "name", "")) - } - } - } - } - } - } - } - } - wrenMetric.Dimensions = timeDimensions - } + // Assume all measures for a given metric come from the same base model. + // We only need to find the first valid one. + for _, inputMeasure := range inputMeasuresList { + if imMap, ok := inputMeasure.(map[string]interface{}); ok { + imName := getStringFromMap(imMap, "name", "") + if model, exists := measureToModelMap[imName]; exists { + return model // Return the first model we find. + } + } + } + return "" +} - // Build the aggregation expression - switch metricType { - case "simple": - if measure, ok := typeParams["measure"].(map[string]interface{}); ok { - measureName := getStringFromMap(measure, "name", "") - if measureData, ok := measureDataLookup[measureName]; ok { - agg := getStringFromMap(measureData, "agg", "sum") - expr := getStringFromMap(measureData, "expr", measureName) - wrenMetric.Aggregation = fmt.Sprintf("%s(%s)", strings.ToUpper(agg), expr) - } - } - case "ratio": - if num, ok := typeParams["numerator"].(map[string]interface{}); ok { - if den, ok := typeParams["denominator"].(map[string]interface{}); ok { - numName := getStringFromMap(num, "name", "") - denName := getStringFromMap(den, "name", "") - if numData, ok := measureDataLookup[numName]; ok { - if denData, ok := measureDataLookup[denName]; ok { - numAgg := strings.ToUpper(getStringFromMap(numData, "agg", "sum")) - denAgg := strings.ToUpper(getStringFromMap(denData, "agg", "sum")) - numExpr := getStringFromMap(numData, "expr", numName) - denExpr := getStringFromMap(denData, "expr", denName) - wrenMetric.Aggregation = fmt.Sprintf("(%s(%s)) / (%s(%s))", numAgg, numExpr, denAgg, denExpr) - } - } +// findTimeDimensionsForModel scans the semantic models to find all columns +// marked with type "time" for a specific model name. +func findTimeDimensionsForModel(semanticData map[string]interface{}, baseModelName string) []string { + var timeDimensions []string + semanticModels, ok := semanticData["semantic_models"].([]interface{}) + if !ok { + return timeDimensions + } + + for _, sm := range semanticModels { + smMap, ok := sm.(map[string]interface{}) + if !ok { + continue + } + + if getStringFromMap(smMap, "name", "") == baseModelName { + if dims, ok := smMap["dimensions"].([]interface{}); ok { + for _, d := range dims { + if dimMap, ok := d.(map[string]interface{}); ok { + if getStringFromMap(dimMap, "type", "") == "time" { + timeDimensions = append(timeDimensions, getStringFromMap(dimMap, "name", "")) } } - case "derived": - wrenMetric.Aggregation = getStringFromMap(typeParams, "expr", "") - } - - if wrenMetric.Aggregation != "" && len(wrenMetric.Models) > 0 { - wrenMetrics = append(wrenMetrics, wrenMetric) } } + break // Found the model, no need to continue looping. } } + return timeDimensions +} - return wrenMetrics +// buildAggregationExpression constructs the SQL aggregation string for a Wren metric +// based on its dbt type ('simple', 'ratio', or 'derived'). +func buildAggregationExpression(metricType string, typeParams map[string]interface{}, measureDataLookup map[string]map[string]interface{}) string { + switch metricType { + case "simple": + // A simple metric is a direct aggregation of one measure (e.g., SUM(revenue)). + if measure, ok := typeParams["measure"].(map[string]interface{}); ok { + measureName := getStringFromMap(measure, "name", "") + if measureData, ok := measureDataLookup[measureName]; ok { + agg := getStringFromMap(measureData, "agg", "sum") // Default to SUM + expr := getStringFromMap(measureData, "expr", measureName) // Fallback to measure name + return fmt.Sprintf("%s(%s)", strings.ToUpper(agg), expr) + } + } + case "ratio": + // A ratio metric is a division of two measures (e.g., SUM(profit) / SUM(revenue)). + num, numOK := typeParams["numerator"].(map[string]interface{}) + den, denOK := typeParams["denominator"].(map[string]interface{}) + if !numOK || !denOK { + return "" + } + + numName := getStringFromMap(num, "name", "") + denName := getStringFromMap(den, "name", "") + numData, numDataOK := measureDataLookup[numName] + denData, denDataOK := measureDataLookup[denName] + + if numDataOK && denDataOK { + numAgg := strings.ToUpper(getStringFromMap(numData, "agg", "sum")) + denAgg := strings.ToUpper(getStringFromMap(denData, "agg", "sum")) + numExpr := getStringFromMap(numData, "expr", numName) + denExpr := getStringFromMap(denData, "expr", denName) + return fmt.Sprintf("(%s(%s)) / (%s(%s))", numAgg, numExpr, denAgg, denExpr) + } + case "derived": + // A derived metric uses a freeform SQL expression. + return getStringFromMap(typeParams, "expr", "") + } + return "" // Return empty string if no valid aggregation could be built. } -// convertDbtNodeToWrenModel converts a single dbt node to Wren model -func convertDbtNodeToWrenModel(nodeKey string, nodeData map[string]interface{}, dataSource DataSource, manifestData map[string]interface{}, columnToEnumNameMap map[string]string, columnToNotNullMap map[string]bool, modelToPrimaryKeyMap map[string]string) (*WrenModel, error) { - // Extract model name from node key - modelName := getModelNameFromNodeKey(nodeKey) - if modelName == "" { - return nil, fmt.Errorf("invalid node key format: %s", nodeKey) +// extractDescriptionsFromManifest parses the manifest.json data to find the +// model-level description and a map of all column-level descriptions. +func extractDescriptionsFromManifest(manifestData map[string]interface{}, nodeKey string) (string, map[string]string) { + if manifestData == nil { + return "", nil } - // Extract metadata - metadataValue, exists := nodeData["metadata"] - if !exists { - return nil, fmt.Errorf("no metadata found for model %s", nodeKey) + nodes, ok := manifestData["nodes"].(map[string]interface{}) + if !ok { + return "", nil } - metadata, ok := metadataValue.(map[string]interface{}) + manifestNode, ok := nodes[nodeKey].(map[string]interface{}) if !ok { - return nil, fmt.Errorf("invalid metadata format for model %s", nodeKey) + return "", nil } - // Create table reference - tableRef := TableReference{ - Table: getStringFromMap(metadata, "name", modelName), + // Extract the top-level model description + modelDescription := getStringFromMap(manifestNode, "description", "") + columnDescriptions := make(map[string]string) + + manifestColumns, ok := manifestNode["columns"].(map[string]interface{}) + if !ok { + // Return the model description even if columns aren't found + return modelDescription, nil } - if catalog := getStringFromMap(metadata, "database", ""); catalog != "" { - tableRef.Catalog = catalog + // Iterate through columns to extract their descriptions + for colName, colData := range manifestColumns { + if colMap, ok := colData.(map[string]interface{}); ok { + if description := getStringFromMap(colMap, "description", ""); description != "" { + columnDescriptions[colName] = description + } + } } - if schema := getStringFromMap(metadata, "schema", ""); schema != "" { - tableRef.Schema = schema + + return modelDescription, columnDescriptions +} + +// buildWrenColumn creates a single WrenColumn from its corresponding dbt column data map. +// It populates the name, type, and properties like enums, descriptions, and comments. +func buildWrenColumn(colMap map[string]interface{}, nodeKey string, dataSource DataSource, columnDescriptions map[string]string, columnToEnumNameMap map[string]string, columnToNotNullMap map[string]bool) WrenColumn { + columnName := getStringFromMap(colMap, "name", "") + columnKey := fmt.Sprintf("%s.%s", nodeKey, columnName) + + column := WrenColumn{ + Name: columnName, + DisplayName: getStringFromMap(getMapFromMap(colMap, "meta", nil), "label", ""), + Type: dataSource.MapType(getStringFromMap(colMap, "type", "")), + NotNull: columnToNotNullMap[columnKey], // Defaults to false if not found } - // Extract descriptions from manifest.json if available - var modelDescription string - var columnDescriptions map[string]string + // Assign an enum if one was derived from dbt tests + if enumName, ok := columnToEnumNameMap[columnKey]; ok { + column.Enum = enumName + } - if manifestData != nil { - if nodes, ok := manifestData["nodes"].(map[string]interface{}); ok { - if manifestNode, ok := nodes[nodeKey].(map[string]interface{}); ok { - // Extract model description - modelDescription = getStringFromMap(manifestNode, "description", "") - - // Extract column descriptions - if manifestColumns, ok := manifestNode["columns"].(map[string]interface{}); ok { - columnDescriptions = make(map[string]string) - for colName, colData := range manifestColumns { - if colMap, ok := colData.(map[string]interface{}); ok { - description := getStringFromMap(colMap, "description", "") - if description != "" { - columnDescriptions[colName] = description - } - } - } - } - } - } + // Use a temporary map to build the properties + properties := make(map[string]string) + if description, exists := columnDescriptions[column.Name]; exists && description != "" { + properties["description"] = description + } + if comment := getStringFromMap(colMap, "comment", ""); comment != "" { + properties["comment"] = comment + } + + // Assign the properties map only if it's not empty + if len(properties) > 0 { + column.Properties = properties } - // Convert columns + return column +} + +// convertAndSortColumns extracts, sorts, and converts dbt columns to the WrenColumn format. +func convertAndSortColumns(nodeData map[string]interface{}, nodeKey string, dataSource DataSource, columnDescriptions map[string]string, columnToEnumNameMap map[string]string, columnToNotNullMap map[string]bool) ([]WrenColumn, error) { columnsValue, exists := nodeData["columns"] if !exists { return nil, fmt.Errorf("no columns found for model %s", nodeKey) @@ -862,71 +968,84 @@ func convertDbtNodeToWrenModel(nodeKey string, nodeData map[string]interface{}, return nil, fmt.Errorf("invalid columns format for model %s", nodeKey) } - var wrenColumns []WrenColumn + // Convert map to a slice for sorting + var columnsData []map[string]interface{} for _, colValue := range columnsMap { - colMap, ok := colValue.(map[string]interface{}) - if !ok { - continue - } - - columnName := getStringFromMap(colMap, "name", "") - columnKey := fmt.Sprintf("%s.%s", nodeKey, columnName) - - column := WrenColumn{ - Name: columnName, - DisplayName: getStringFromMap(getMapFromMap(colMap, "meta", nil), "label", ""), - Type: dataSource.MapType(getStringFromMap(colMap, "type", "")), - NotNull: columnToNotNullMap[columnKey], // Will be false if not found + if colMap, ok := colValue.(map[string]interface{}); ok { + columnsData = append(columnsData, colMap) } + } - // Check for and assign enum - if enumName, ok := columnToEnumNameMap[columnKey]; ok { - column.Enum = enumName + // Sort columns by the 'index' field, falling back to name + sort.Slice(columnsData, func(i, j int) bool { + indexI, okI := columnsData[i]["index"].(float64) + indexJ, okJ := columnsData[j]["index"].(float64) + if okI && okJ { + return indexI < indexJ } + return getStringFromMap(columnsData[i], "name", "") < getStringFromMap(columnsData[j], "name", "") + }) - // Initialize properties map if needed - if column.Properties == nil { - column.Properties = make(map[string]string) + // Build the final slice of WrenColumns + var wrenColumns []WrenColumn + for _, colMap := range columnsData { + if getStringFromMap(colMap, "name", "") == "" { + continue } + column := buildWrenColumn(colMap, nodeKey, dataSource, columnDescriptions, columnToEnumNameMap, columnToNotNullMap) + wrenColumns = append(wrenColumns, column) + } - // Set description from manifest if available - if columnDescriptions != nil { - if description, exists := columnDescriptions[column.Name]; exists && description != "" { - column.Properties["description"] = description - } - } + return wrenColumns, nil +} - // Set notNull based on comment or other indicators - if comment := getStringFromMap(colMap, "comment", ""); comment != "" { - column.Properties["comment"] = comment - } +// convertDbtNodeToWrenModel converts a single dbt node to a Wren model. +// This function now orchestrates calls to helpers to perform the conversion. +func convertDbtNodeToWrenModel(nodeKey string, nodeData map[string]interface{}, dataSource DataSource, manifestData map[string]interface{}, columnToEnumNameMap map[string]string, columnToNotNullMap map[string]bool, modelToPrimaryKeyMap map[string]string) (*WrenModel, error) { + modelName := getModelNameFromNodeKey(nodeKey) + if modelName == "" { + return nil, fmt.Errorf("invalid node key format: %s", nodeKey) + } - wrenColumns = append(wrenColumns, column) + // --- 1. Extract Metadata and Table Reference --- + metadataValue, exists := nodeData["metadata"] + if !exists { + return nil, fmt.Errorf("no metadata found for model %s", nodeKey) + } + metadata, ok := metadataValue.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid metadata format for model %s", nodeKey) + } + tableRef := TableReference{ + Table: getStringFromMap(metadata, "name", modelName), + Catalog: getStringFromMap(metadata, "database", ""), + Schema: getStringFromMap(metadata, "schema", ""), } - // Sort columns by index if available - sort.Slice(wrenColumns, func(i, j int) bool { - // This is a simplified sort - you might want to use the index from dbt - return wrenColumns[i].Name < wrenColumns[j].Name - }) + // --- 2. Extract Descriptions from Manifest --- + modelDescription, columnDescriptions := extractDescriptionsFromManifest(manifestData, nodeKey) + // --- 3. Convert and Sort Columns --- + wrenColumns, err := convertAndSortColumns(nodeData, nodeKey, dataSource, columnDescriptions, columnToEnumNameMap, columnToNotNullMap) + if err != nil { + return nil, err + } + + // --- 4. Assemble the Final WrenModel --- model := &WrenModel{ Name: modelName, TableReference: tableRef, Columns: wrenColumns, } - // Set primary key from semantic manifest if available + // Set primary key if available if pk, ok := modelToPrimaryKeyMap[modelName]; ok { model.PrimaryKey = pk } - // Set model description from manifest if available + // Set model description if available if modelDescription != "" { - if model.Properties == nil { - model.Properties = make(map[string]string) - } - model.Properties["description"] = modelDescription + model.Properties = map[string]string{"description": modelDescription} } return model, nil diff --git a/wren-launcher/commands/dbt/data_source.go b/wren-launcher/commands/dbt/data_source.go index 4acfa26d41..5afc37accc 100644 --- a/wren-launcher/commands/dbt/data_source.go +++ b/wren-launcher/commands/dbt/data_source.go @@ -247,10 +247,11 @@ func convertToBigQueryDataSource(conn DbtConnection, dbtHomePath string) (*WrenB resolvedKeyfilePath = filepath.Join(dbtHomePath, keyfilePath) } - b, err := os.ReadFile(resolvedKeyfilePath) + cleanPath := filepath.Clean(resolvedKeyfilePath) + b, err := os.ReadFile(cleanPath) if err != nil { // Update the error message to show the path that was attempted - return nil, fmt.Errorf("failed to read keyfile '%s': %w", resolvedKeyfilePath, err) + return nil, fmt.Errorf("failed to read keyfile '%s': %w", cleanPath, err) } enc, err := encodeJSON(b) if err != nil { @@ -466,23 +467,23 @@ func (ds *WrenBigQueryDataSource) Validate() error { func (ds *WrenBigQueryDataSource) MapType(sourceType string) string { switch strings.ToUpper(sourceType) { case "INT64", "INTEGER": - return "integer" + return integerType case "FLOAT64", "FLOAT": - return "double" + return doubleType case "STRING": - return "varchar" + return varcharType case "BOOL", "BOOLEAN": - return "boolean" + return booleanType case "DATE": - return "date" + return dateType case "TIMESTAMP", "DATETIME": - return "timestamp" + return timestampType case "NUMERIC", "DECIMAL", "BIGNUMERIC": - return "double" + return doubleType case "BYTES": - return "varchar" + return varcharType case "JSON": - return "varchar" + return varcharType default: return strings.ToLower(sourceType) } @@ -602,4 +603,4 @@ func (d *DefaultDataSource) MapType(sourceType string) string { default: return strings.ToLower(sourceType) } -} +} \ No newline at end of file diff --git a/wren-launcher/commands/dbt/data_source_test.go b/wren-launcher/commands/dbt/data_source_test.go index 557ad8d41e..e53c35efcb 100644 --- a/wren-launcher/commands/dbt/data_source_test.go +++ b/wren-launcher/commands/dbt/data_source_test.go @@ -279,10 +279,14 @@ func TestFromDbtProfiles_BigQuery(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.RemoveAll(tempDir) +defer func() { + if err := os.RemoveAll(tempDir); err != nil { + t.Logf("Failed to remove temporary directory %s: %v", tempDir, err) + } +}() t.Run("service-account-json", func(t *testing.T) { - keyfileContent := `{"type": "service_account", "project_id": "test-project", "private_key_id": "test-key-id", "private_key": "test-private-key", "client_email": "test-client-email", "client_id": "test-client-id", "auth_uri": "test-auth-uri", "token_uri": "test-token-uri", "auth_provider_x509_cert_url": "test-cert-url", "client_x509_cert_url": "test-client-cert-url"}` + keyfileContent := `{"type": "service_account", "project_id": "test-project", "private_key_id": "test-key-id", "private_key": "test-private-key", "client_email": "test-client-email", "client_id": "test-client-id", "auth_uri": "test-auth-uri", "token_uri": "test-token-uri", "auth_provider_x509_cert_url": "test-cert-url", "client_x509_cert_url": "test-client-cert-url"}` // #nosec G101 profiles := &DbtProfiles{ Profiles: map[string]DbtProfile{ "test_profile": { @@ -331,9 +335,9 @@ func TestFromDbtProfiles_BigQuery(t *testing.T) { }) t.Run("service-account-with-absolute-keyfile-path", func(t *testing.T) { - keyfileContent := `{"type": "service_account"}` + keyfileContent := `{"type": "service_account"}` // #nosec G101 keyfilePath := filepath.Join(tempDir, "keyfile.json") - if err := os.WriteFile(keyfilePath, []byte(keyfileContent), 0644); err != nil { + if err := os.WriteFile(keyfilePath, []byte(keyfileContent), 0600); err != nil { t.Fatal(err) } @@ -376,14 +380,14 @@ func TestFromDbtProfiles_BigQuery(t *testing.T) { t.Run("service-account-with-relative-keyfile-path", func(t *testing.T) { dbtHomePath := tempDir - keyfileContent := `{"type": "service_account"}` + keyfileContent := `{"type": "service_account"}` // #nosec G101 keyfilePath := "keys/keyfile.json" fullKeyfilePath := filepath.Join(dbtHomePath, keyfilePath) - if err := os.MkdirAll(filepath.Dir(fullKeyfilePath), 0755); err != nil { + if err := os.MkdirAll(filepath.Dir(fullKeyfilePath), 0750); err != nil { t.Fatal(err) } - if err := os.WriteFile(fullKeyfilePath, []byte(keyfileContent), 0644); err != nil { + if err := os.WriteFile(fullKeyfilePath, []byte(keyfileContent), 0600); err != nil { t.Fatal(err) } From bdf4f221f575523394b26094a29eb83e02230e4f Mon Sep 17 00:00:00 2001 From: Casey Grimes Date: Sun, 14 Sep 2025 20:42:34 -0400 Subject: [PATCH 17/18] Some small updates chore: Ran go fmt one more time for good measure chore(test): Adjusted test on abs_path to work cross-platform --- wren-launcher/commands/dbt/data_source.go | 2 +- wren-launcher/commands/dbt/data_source_test.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/wren-launcher/commands/dbt/data_source.go b/wren-launcher/commands/dbt/data_source.go index 5afc37accc..f274b4224a 100644 --- a/wren-launcher/commands/dbt/data_source.go +++ b/wren-launcher/commands/dbt/data_source.go @@ -603,4 +603,4 @@ func (d *DefaultDataSource) MapType(sourceType string) string { default: return strings.ToLower(sourceType) } -} \ No newline at end of file +} diff --git a/wren-launcher/commands/dbt/data_source_test.go b/wren-launcher/commands/dbt/data_source_test.go index e53c35efcb..09fdd5d7ce 100644 --- a/wren-launcher/commands/dbt/data_source_test.go +++ b/wren-launcher/commands/dbt/data_source_test.go @@ -175,7 +175,7 @@ func TestFromDbtProfiles_LocalFile(t *testing.T) { t.Fatalf("Expected WrenLocalFileDataSource, got %T", dataSources[0]) } - if ds.Url != "/abs_path" { + if filepath.ToSlash(ds.Url) != "/abs_path" { t.Errorf("Expected url '/abs_path', got '%s'", ds.Url) } if ds.Format != duckdbType { @@ -279,11 +279,11 @@ func TestFromDbtProfiles_BigQuery(t *testing.T) { if err != nil { t.Fatal(err) } -defer func() { - if err := os.RemoveAll(tempDir); err != nil { - t.Logf("Failed to remove temporary directory %s: %v", tempDir, err) - } -}() + defer func() { + if err := os.RemoveAll(tempDir); err != nil { + t.Logf("Failed to remove temporary directory %s: %v", tempDir, err) + } + }() t.Run("service-account-json", func(t *testing.T) { keyfileContent := `{"type": "service_account", "project_id": "test-project", "private_key_id": "test-key-id", "private_key": "test-private-key", "client_email": "test-client-email", "client_id": "test-client-id", "auth_uri": "test-auth-uri", "token_uri": "test-token-uri", "auth_provider_x509_cert_url": "test-cert-url", "client_x509_cert_url": "test-client-cert-url"}` // #nosec G101 From e2286cb2658f0fac9634b2ee1e7d1184da75dd9a Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Mon, 22 Sep 2025 11:56:02 +0800 Subject: [PATCH 18/18] chore: fmt --- wren-launcher/commands/dbt.go | 2 +- wren-launcher/commands/launch.go | 4 ++-- wren-launcher/utils/docker.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/wren-launcher/commands/dbt.go b/wren-launcher/commands/dbt.go index 09f96d0d6f..60c85a6534 100644 --- a/wren-launcher/commands/dbt.go +++ b/wren-launcher/commands/dbt.go @@ -72,4 +72,4 @@ func DbtConvertProject(projectPath, outputDir, profileName, target string, usedB } return dbt.ConvertDbtProjectCore(convertOpts) -} \ No newline at end of file +} diff --git a/wren-launcher/commands/launch.go b/wren-launcher/commands/launch.go index 1f8ff2ede3..01fe9f13ed 100644 --- a/wren-launcher/commands/launch.go +++ b/wren-launcher/commands/launch.go @@ -537,7 +537,7 @@ func processDbtProject(projectDir string) (string, error) { includeStagingModels, err := askForIncludeStagingModels() if err != nil { pterm.Warning.Println("Could not get staging model preference, defaulting to 'No'.") - includeStagingModels = false + includeStagingModels = false } // Use the core conversion function from dbt package, passing the user's choice @@ -549,4 +549,4 @@ func processDbtProject(projectDir string) (string, error) { pterm.Info.Printf("Successfully processed dbt project to target directory: %s\n", targetDir) return result.LocalStoragePath, nil -} \ No newline at end of file +} diff --git a/wren-launcher/utils/docker.go b/wren-launcher/utils/docker.go index 280ee28791..f49d67c480 100644 --- a/wren-launcher/utils/docker.go +++ b/wren-launcher/utils/docker.go @@ -23,7 +23,7 @@ import ( const ( // please change the version when the version is updated - WREN_PRODUCT_VERSION string = "0.28.0" + WREN_PRODUCT_VERSION string = "0.28.0" DOCKER_COMPOSE_YAML_URL string = "https://raw.githubusercontent.com/Canner/WrenAI/" + WREN_PRODUCT_VERSION + "/docker/docker-compose.yaml" DOCKER_COMPOSE_ENV_URL string = "https://raw.githubusercontent.com/Canner/WrenAI/" + WREN_PRODUCT_VERSION + "/docker/.env.example" AI_SERVICE_CONFIG_URL string = "https://raw.githubusercontent.com/Canner/WrenAI/" + WREN_PRODUCT_VERSION + "/docker/config.example.yaml"