fix(cmd/gf): fix gen sharding dao in multiple shardingPattern tables …

cmd/gf/internal/cmd/cmd_z_unit_gen_dao_sharding_test.go

@@ -26,25 +26,34 @@ func Test_Gen_Dao_Sharding(t *testing.T) {
 			tableSingle = "single_table"
 			table1      = "users_0001"
 			table2      = "users_0002"
-			table3      = "users_0003"
+			table3      = "orders_0001"
+			table4      = "orders_0002"
 			sqlFilePath = gtest.DataPath(`gendao`, `sharding`, `sharding.sql`)
 		)
+		dropTableWithDb(db, tableSingle)
+		dropTableWithDb(db, table1)
+		dropTableWithDb(db, table2)
+		dropTableWithDb(db, table3)
+		dropTableWithDb(db, table4)
 		t.AssertNil(execSqlFile(db, sqlFilePath))
 		defer dropTableWithDb(db, tableSingle)
 		defer dropTableWithDb(db, table1)
 		defer dropTableWithDb(db, table2)
 		defer dropTableWithDb(db, table3)
+		defer dropTableWithDb(db, table4)
 
 		var (
 			path = gfile.Temp(guid.S())
-			//path  = "/Users/john/Temp/gen_dao_sharding"
+			// path  = "/Users/john/Temp/gen_dao_sharding"
 			group = "test"
 			in    = gendao.CGenDaoInput{
-				Path:  path,
-				Link:  link,
-				Group: group,
+				Path:   path,
+				Link:   link,
+				Group:  group,
+				Prefix: "",
 				ShardingPattern: []string{
 					`users_?`,
+					`orders_?`,
 				},
 			}
 		)
@@ -65,13 +74,16 @@ func Test_Gen_Dao_Sharding(t *testing.T) {
 
 		generatedFiles, err := gfile.ScanDir(path, "*.go", true)
 		t.AssertNil(err)
-		t.Assert(len(generatedFiles), 8)
+		t.Assert(len(generatedFiles), 12)
 		var (
 			daoSingleTableContent = gfile.GetContents(gfile.Join(path, "dao", "single_table.go"))
 			daoUsersContent       = gfile.GetContents(gfile.Join(path, "dao", "users.go"))
+			daoOrdersContent      = gfile.GetContents(gfile.Join(path, "dao", "orders.go"))
 		)
 		t.Assert(gstr.Contains(daoSingleTableContent, "SingleTable = singleTableDao{internal.NewSingleTableDao()}"), true)
-		t.Assert(gstr.Contains(daoUsersContent, "Users = usersDao{internal.NewUsersDao(userShardingHandler)}"), true)
+		t.Assert(gstr.Contains(daoUsersContent, "Users = usersDao{internal.NewUsersDao(usersShardingHandler)}"), true)
 		t.Assert(gstr.Contains(daoUsersContent, "m.Sharding(gdb.ShardingConfig{"), true)
+		t.Assert(gstr.Contains(daoOrdersContent, "Orders = ordersDao{internal.NewOrdersDao(ordersShardingHandler)}"), true)
+		t.Assert(gstr.Contains(daoOrdersContent, "m.Sharding(gdb.ShardingConfig{"), true)
 	})
 }

cmd/gf/internal/cmd/cmd_z_unit_gen_pbentity_test.go

@@ -367,3 +367,145 @@ func Test_Issue_3955(t *testing.T) {
 		}
 	})
 }
+
+func Test_Issue_4330_TypeMapping_Ineffective(t *testing.T) {
+	gtest.C(t, func(t *gtest.T) {
+		var (
+			err        error
+			db         = testDB
+			table      = "table_user"
+			sqlContent = fmt.Sprintf(
+				gtest.DataContent(`issue`, `3685`, `user.tpl.sql`),
+				table,
+			)
+		)
+		dropTableWithDb(db, table)
+		array := gstr.SplitAndTrim(sqlContent, ";")
+		for _, v := range array {
+			if _, err = db.Exec(ctx, v); err != nil {
+				t.AssertNil(err)
+			}
+		}
+		defer dropTableWithDb(db, table)
+
+		var (
+			path = gfile.Temp(guid.S())
+			in   = genpbentity.CGenPbEntityInput{
+				Path:              path,
+				Package:           "",
+				Link:              link,
+				Tables:            "",
+				Prefix:            "",
+				RemovePrefix:      "",
+				RemoveFieldPrefix: "",
+				NameCase:          "",
+				JsonCase:          "",
+				Option:            "",
+				TypeMapping: map[genpbentity.DBFieldTypeName]genpbentity.CustomAttributeType{
+					"json": {
+						Type:   "google.protobuf.Value",
+						Import: "google/protobuf/struct.proto",
+					},
+					"decimal": {
+						Type: "double",
+					},
+				},
+				FieldMapping: nil,
+			}
+		)
+		err = gutil.FillStructWithDefault(&in)
+		t.AssertNil(err)
+
+		err = gfile.Mkdir(path)
+		t.AssertNil(err)
+		defer gfile.Remove(path)
+
+		_, err = genpbentity.CGenPbEntity{}.PbEntity(ctx, in)
+		t.AssertNil(err)
+
+		// files
+		files, err := gfile.ScanDir(path, "*.proto", false)
+		t.AssertNil(err)
+		t.Assert(files, []string{
+			path + filepath.FromSlash("/table_user.proto"),
+		})
+
+		// contents
+		testPath := gtest.DataPath("issue", "4330")
+		expectFiles := []string{
+			testPath + filepath.FromSlash("/issue4330_double.proto"),
+		}
+		for i := range files {
+			t.Assert(gfile.GetContents(files[i]), gfile.GetContents(expectFiles[i]))
+		}
+	})
+}
+
+func Test_Gen_Pbentity_Sharding(t *testing.T) {
+	gtest.C(t, func(t *gtest.T) {
+		var (
+			err         error
+			db          = testDB
+			tableSingle = "single_table"
+			table1      = "users_0001"
+			table2      = "users_0002"
+			table3      = "orders_0001"
+			table4      = "orders_0002"
+			sqlFilePath = gtest.DataPath(`gendao`, `sharding`, `sharding.sql`)
+		)
+		dropTableWithDb(db, tableSingle)
+		dropTableWithDb(db, table1)
+		dropTableWithDb(db, table2)
+		dropTableWithDb(db, table3)
+		dropTableWithDb(db, table4)
+		t.AssertNil(execSqlFile(db, sqlFilePath))
+		defer dropTableWithDb(db, tableSingle)
+		defer dropTableWithDb(db, table1)
+		defer dropTableWithDb(db, table2)
+		defer dropTableWithDb(db, table3)
+		defer dropTableWithDb(db, table4)
+
+		var (
+			path = gfile.Temp(guid.S())
+			in   = genpbentity.CGenPbEntityInput{
+				Path:              path,
+				Package:           "unittest",
+				Link:              link,
+				Tables:            "",
+				RemovePrefix:      "",
+				RemoveFieldPrefix: "",
+				NameCase:          "",
+				JsonCase:          "",
+				Option:            "",
+				TypeMapping:       nil,
+				FieldMapping:      nil,
+				ShardingPattern: []string{
+					`users_?`,
+					`orders_?`,
+				},
+			}
+		)
+		err = gutil.FillStructWithDefault(&in)
+		t.AssertNil(err)
+
+		err = gfile.Mkdir(path)
+		t.AssertNil(err)
+		defer gfile.Remove(path)
+
+		_, err = genpbentity.CGenPbEntity{}.PbEntity(ctx, in)
+		t.AssertNil(err)
+
+		// files
+		t.AssertNil(err)
+		generatedFiles, err := gfile.ScanDir(path, "*.proto", true)
+		t.Assert(len(generatedFiles), 3)
+		var (
+			msgSingleTableContent = gfile.GetContents(gfile.Join(path, "single_table.proto"))
+			msgUsersContent       = gfile.GetContents(gfile.Join(path, "users.proto"))
+			msgOrdersContent      = gfile.GetContents(gfile.Join(path, "orders.proto"))
+		)
+		t.Assert(gstr.Contains(msgSingleTableContent, "message SingleTable {"), true)
+		t.Assert(gstr.Contains(msgUsersContent, "message Users {"), true)
+		t.Assert(gstr.Contains(msgOrdersContent, "message Orders {"), true)
+	})
+}

cmd/gf/internal/cmd/gendao/gendao.go

@@ -219,14 +219,18 @@ func doGenDaoForArray(ctx context.Context, index int, in CGenDaoInput) {
 					tableNames[i] = ""
 					continue
 				}
-				shardingNewTableSet.Add(newTableName)
+				// Add prefix to sharding table name, if not, the isSharding check would not match.
+				shardingNewTableSet.Add(in.Prefix + newTableName)
 			}
 		}
 		newTableName = in.Prefix + newTableName
-		newTableNames[i] = newTableName
+		if tableNames[i] != "" {
+			// If shardingNewTableSet contains newTableName (tableName is empty), it should not be added to tableNames, make it empty and filter later.
+			newTableNames[i] = newTableName
+		}
 	}
 	tableNames = garray.NewStrArrayFrom(tableNames).FilterEmpty().Slice()
-
+	newTableNames = garray.NewStrArrayFrom(newTableNames).FilterEmpty().Slice() // Filter empty table names. make sure that newTableNames and tableNames have the same length.
 	in.genItems.Scale()
 
 	// Dao: index and internal.

cmd/gf/internal/cmd/gendao/gendao_dao.go

@@ -127,6 +127,7 @@ func generateDaoIndex(in generateDaoIndexInput) {
 		tplView.ClearAssigns()
 		tplView.Assigns(gview.Params{
 			tplVarTableSharding:           in.IsSharding,
+			tplVarTableShardingPrefix:     in.NewTableName + "_",
 			tplVarImportPrefix:            in.ImportPrefix,
 			tplVarTableName:               in.TableName,
 			tplVarTableNameCamelCase:      in.TableNameCamelCase,

cmd/gf/internal/cmd/gendao/gendao_tag.go

@@ -97,6 +97,7 @@ generated json tag case for model struct, cases are as follows:
 	tplVarTableNameCamelCase      = `TplTableNameCamelCase`
 	tplVarTableNameCamelLowerCase = `TplTableNameCamelLowerCase`
 	tplVarTableSharding           = `TplTableSharding`
+	tplVarTableShardingPrefix     = `TplTableShardingPrefix`
 	tplVarPackageImports          = `TplPackageImports`
 	tplVarImportPrefix            = `TplImportPrefix`
 	tplVarStructDefine            = `TplStructDefine`

cmd/gf/internal/cmd/genpbentity/genpbentity.go

@@ -37,18 +37,19 @@ type (
 	CGenPbEntity      struct{}
 	CGenPbEntityInput struct {
 		g.Meta            `name:"pbentity" config:"{CGenPbEntityConfig}" brief:"{CGenPbEntityBrief}" eg:"{CGenPbEntityEg}" ad:"{CGenPbEntityAd}"`
-		Path              string `name:"path"              short:"p"  brief:"{CGenPbEntityBriefPath}" d:"manifest/protobuf/pbentity"`
-		Package           string `name:"package"           short:"k"  brief:"{CGenPbEntityBriefPackage}"`
-		GoPackage         string `name:"goPackage"           short:"g"  brief:"{CGenPbEntityBriefGoPackage}"`
-		Link              string `name:"link"              short:"l"  brief:"{CGenPbEntityBriefLink}"`
-		Tables            string `name:"tables"            short:"t"  brief:"{CGenPbEntityBriefTables}"`
-		Prefix            string `name:"prefix"            short:"f"  brief:"{CGenPbEntityBriefPrefix}"`
-		RemovePrefix      string `name:"removePrefix"      short:"r"  brief:"{CGenPbEntityBriefRemovePrefix}"`
-		RemoveFieldPrefix string `name:"removeFieldPrefix" short:"rf" brief:"{CGenPbEntityBriefRemoveFieldPrefix}"`
-		TablesEx          string `name:"tablesEx"          short:"x"  brief:"{CGenDaoBriefTablesEx}"`
-		NameCase          string `name:"nameCase"          short:"n"  brief:"{CGenPbEntityBriefNameCase}" d:"Camel"`
-		JsonCase          string `name:"jsonCase"          short:"j"  brief:"{CGenPbEntityBriefJsonCase}" d:"none"`
-		Option            string `name:"option"            short:"o"  brief:"{CGenPbEntityBriefOption}"`
+		Path              string   `name:"path"              short:"p"  brief:"{CGenPbEntityBriefPath}" d:"manifest/protobuf/pbentity"`
+		Package           string   `name:"package"           short:"k"  brief:"{CGenPbEntityBriefPackage}"`
+		GoPackage         string   `name:"goPackage"           short:"g"  brief:"{CGenPbEntityBriefGoPackage}"`
+		Link              string   `name:"link"              short:"l"  brief:"{CGenPbEntityBriefLink}"`
+		Tables            string   `name:"tables"            short:"t"  brief:"{CGenPbEntityBriefTables}"`
+		Prefix            string   `name:"prefix"            short:"f"  brief:"{CGenPbEntityBriefPrefix}"`
+		RemovePrefix      string   `name:"removePrefix"      short:"r"  brief:"{CGenPbEntityBriefRemovePrefix}"`
+		RemoveFieldPrefix string   `name:"removeFieldPrefix" short:"rf" brief:"{CGenPbEntityBriefRemoveFieldPrefix}"`
+		TablesEx          string   `name:"tablesEx"          short:"x"  brief:"{CGenDaoBriefTablesEx}"`
+		NameCase          string   `name:"nameCase"          short:"n"  brief:"{CGenPbEntityBriefNameCase}" d:"Camel"`
+		JsonCase          string   `name:"jsonCase"          short:"j"  brief:"{CGenPbEntityBriefJsonCase}" d:"none"`
+		Option            string   `name:"option"            short:"o"  brief:"{CGenPbEntityBriefOption}"`
+		ShardingPattern   []string `name:"shardingPattern"     short:"sp" brief:"{CGenDaoBriefShardingPattern}"`
 
 		TypeMapping  map[DBFieldTypeName]CustomAttributeType  `name:"typeMapping"  short:"y"  brief:"{CGenPbEntityBriefTypeMapping}"  orphan:"true"`
 		FieldMapping map[DBTableFieldName]CustomAttributeType `name:"fieldMapping" short:"fm" brief:"{CGenPbEntityBriefFieldMapping}" orphan:"true"`
@@ -122,6 +123,7 @@ CONFIGURATION SUPPORT
 	CGenPbEntityBriefTablesEx          = `generate all models exclude the specified tables, multiple prefix separated with ','`
 	CGenPbEntityBriefRemoveFieldPrefix = `remove specified prefix of the field, multiple prefix separated with ','`
 	CGenPbEntityBriefOption            = `extra protobuf options`
+	CGenPbEntityBriefShardingPattern   = `sharding pattern for table name, e.g. "users_?" will replace tables "users_001,users_002,..." to "users" pbentity`
 	CGenPbEntityBriefGroup             = `
 specifying the configuration group name of database for generated ORM instance,
 it's not necessary and the default value is "default"
@@ -252,6 +254,7 @@ func init() {
 		`CGenPbEntityBriefNameCase`:          CGenPbEntityBriefNameCase,
 		`CGenPbEntityBriefJsonCase`:          CGenPbEntityBriefJsonCase,
 		`CGenPbEntityBriefOption`:            CGenPbEntityBriefOption,
+		`CGenPbEntityBriefShardingPattern`:   CGenPbEntityBriefShardingPattern,
 		`CGenPbEntityBriefTypeMapping`:       CGenPbEntityBriefTypeMapping,
 		`CGenPbEntityBriefFieldMapping`:      CGenPbEntityBriefFieldMapping,
 	})
@@ -321,6 +324,7 @@ func doGenPbEntityForArray(ctx context.Context, index int, in CGenPbEntityInput)
 	}
 
 	tableNames := ([]string)(nil)
+	shardingNewTableSet := gset.NewStrSet()
 	if in.Tables != "" {
 		tableNames = gstr.SplitAndTrim(in.Tables, ",")
 	} else {
@@ -348,6 +352,31 @@ func doGenPbEntityForArray(ctx context.Context, index int, in CGenPbEntityInput)
 		for _, v := range removePrefixArray {
 			newTableName = gstr.TrimLeftStr(newTableName, v, 1)
 		}
+		var shardingTableName string
+		if len(in.ShardingPattern) > 0 {
+			for _, pattern := range in.ShardingPattern {
+				var (
+					match      []string
+					regPattern = gstr.Replace(pattern, "?", `(.+)`)
+				)
+				match, err = gregex.MatchString(regPattern, newTableName)
+				if err != nil {
+					mlog.Fatalf(`invalid sharding pattern "%s": %+v`, pattern, err)
+				}
+				if len(match) < 2 {
+					continue
+				}
+				shardingTableName = gstr.Replace(pattern, "?", "")
+				shardingTableName = gstr.Trim(shardingTableName, `_.-`)
+			}
+		}
+		if shardingTableName != "" {
+			if shardingNewTableSet.Contains(shardingTableName) {
+				continue
+			}
+			shardingNewTableSet.Add(shardingTableName)
+			newTableName = shardingTableName
+		}
 		generatePbEntityContentFile(ctx, CGenPbEntityInternalInput{
 			CGenPbEntityInput: in,
 			DB:                db,
@@ -441,14 +470,23 @@ func generateMessageFieldForPbEntity(index int, field *gdb.TableField, in CGenPb
 		err              error
 		ctx              = gctx.GetInitCtx()
 	)
-
 	if in.TypeMapping != nil && len(in.TypeMapping) > 0 {
+		// match typeMapping after local type transform.
+		// eg: double => string, varchar => string etc.
 		localTypeName, err = in.DB.CheckLocalTypeForField(ctx, field.Type, nil)
 		if err != nil {
 			panic(err)
 		}
 		if localTypeName != "" {
-			if typeMapping, ok := in.TypeMapping[strings.ToLower(string(localTypeName))]; ok {
+			if typeMappingLocal, localOk := in.TypeMapping[strings.ToLower(string(localTypeName))]; localOk {
+				localTypeNameStr = typeMappingLocal.Type
+				appendImport = typeMappingLocal.Import
+			}
+		}
+		// Try match unknown / string localTypeName with db type.
+		if localTypeName == "" || localTypeName == gdb.LocalTypeString {
+			formattedFieldType, _ := in.DB.GetFormattedDBTypeNameForField(field.Type)
+			if typeMapping, ok := in.TypeMapping[strings.ToLower(formattedFieldType)]; ok {
 				localTypeNameStr = typeMapping.Type
 				appendImport = typeMapping.Import
 			}