Merge pull request #5 from xxxsen/xxxsen/feature/optimize_category_se…

…arch

Xxxsen/feature/optimize category search

config/config.go

@@ -7,18 +7,24 @@ import (
 	"github.com/xxxsen/common/logger"
 )
 
+type CategoryPlugin struct {
+	Name    string   `json:"name"`
+	Plugins []string `json:"plugins"`
+}
+
 type Config struct {
-	ScanDir        string                 `json:"scan_dir"`
-	SaveDir        string                 `json:"save_dir"`
-	DataDir        string                 `json:"data_dir"`
-	Naming         string                 `json:"naming"`
-	PluginConfig   map[string]interface{} `json:"plugin_config"`
-	HandlerConfig  map[string]interface{} `json:"handler_config"`
-	Plugins        []string               `json:"plugins"`
-	Handlers       []string               `json:"handlers"`
-	ExtraMediaExts []string               `json:"extra_media_exts"`
-	LogConfig      logger.LogConfig       `json:"log_config"`
-	SwitchConfig   SwitchConfig           `json:"switch_config"`
+	ScanDir         string                 `json:"scan_dir"`
+	SaveDir         string                 `json:"save_dir"`
+	DataDir         string                 `json:"data_dir"`
+	Naming          string                 `json:"naming"`
+	PluginConfig    map[string]interface{} `json:"plugin_config"`
+	HandlerConfig   map[string]interface{} `json:"handler_config"`
+	Plugins         []string               `json:"plugins"`
+	CategoryPlugins []CategoryPlugin       `json:"category_plugins"`
+	Handlers        []string               `json:"handlers"`
+	ExtraMediaExts  []string               `json:"extra_media_exts"`
+	LogConfig       logger.LogConfig       `json:"log_config"`
+	SwitchConfig    SwitchConfig           `json:"switch_config"`
 }
 
 type SwitchConfig struct {
@@ -40,6 +46,10 @@ func defaultConfig() *Config {
 			"tktube",
 			"avsox",
 		},
+		CategoryPlugins: []CategoryPlugin{
+			//如果存在分配配置, 那么当番号被识别为特定分类的场景下, 将会使用分类插件直接查询
+			{Name: "FC2", Plugins: []string{"fc2", "18av", "freejavbt", "tktube", "avsox"}},
+		},
 		Handlers: []string{
 			"image_transcoder",
 			"poster_cropper",

main.go

@@ -6,10 +6,12 @@ import (
 	"fmt"
 	"log"
 	"path/filepath"
+	"strings"
 	"yamdc/capture"
 	"yamdc/config"
 	"yamdc/face"
 	"yamdc/ffmpeg"
+	"yamdc/number"
 	"yamdc/processor"
 	"yamdc/processor/handler"
 	"yamdc/searcher"
@@ -49,6 +51,9 @@ func main() {
 	logkit.Info("support plugins", zap.Strings("plugins", plugin.Plugins()))
 	logkit.Info("support handlers", zap.Strings("handlers", handler.Handlers()))
 	logkit.Info("current use plugins", zap.Strings("plugins", c.Plugins))
+	for _, ct := range c.CategoryPlugins {
+		logkit.Info("-- cat plugins", zap.String("cat", ct.Name), zap.Strings("plugins", ct.Plugins))
+	}
 	logkit.Info("current use handlers", zap.Strings("handlers", c.Handlers))
 	logkit.Info("use naming rule", zap.String("rule", c.Naming))
 	logkit.Info("scrape from dir", zap.String("dir", c.ScanDir))
@@ -65,11 +70,15 @@ func main() {
 	if err != nil {
 		logkit.Fatal("build searcher failed", zap.Error(err))
 	}
+	catSs, err := buildCatSearcher(c.CategoryPlugins, c.PluginConfig)
+	if err != nil {
+		logkit.Fatal("build cat searcher failed", zap.Error(err))
+	}
 	ps, err := buildProcessor(c.Handlers, c.HandlerConfig)
 	if err != nil {
 		logkit.Fatal("build processor failed", zap.Error(err))
 	}
-	cap, err := buildCapture(c, ss, ps)
+	cap, err := buildCapture(c, ss, catSs, ps)
 	if err != nil {
 		logkit.Fatal("build capture runner failed", zap.Error(err))
 	}
@@ -81,20 +90,32 @@ func main() {
 	logkit.Info("run capture kit finish, all file scrape succ")
 }
 
-func buildCapture(c *config.Config, ss []searcher.ISearcher, ps []processor.IProcessor) (*capture.Capture, error) {
+func buildCapture(c *config.Config, ss []searcher.ISearcher, catSs map[number.Category][]searcher.ISearcher, ps []processor.IProcessor) (*capture.Capture, error) {
 	opts := make([]capture.Option, 0, 10)
 	opts = append(opts,
 		capture.WithNamingRule(c.Naming),
 		capture.WithScanDir(c.ScanDir),
 		capture.WithSaveDir(c.SaveDir),
-		capture.WithSeacher(searcher.NewGroup(ss)),
+		capture.WithSeacher(searcher.NewCategorySearcher(ss, catSs)),
 		capture.WithProcessor(processor.NewGroup(ps)),
 		capture.WithEnableLinkMode(c.SwitchConfig.EnableLinkMode),
 		capture.WithExtraMediaExtList(c.ExtraMediaExts),
 	)
 	return capture.New(opts...)
 }
 
+func buildCatSearcher(cplgs []config.CategoryPlugin, m map[string]interface{}) (map[number.Category][]searcher.ISearcher, error) {
+	rs := make(map[number.Category][]searcher.ISearcher, len(cplgs))
+	for _, plg := range cplgs {
+		ss, err := buildSearcher(plg.Plugins, m)
+		if err != nil {
+			return nil, err
+		}
+		rs[number.Category(strings.ToUpper(plg.Name))] = ss
+	}
+	return rs, nil
+}
+
 func buildSearcher(plgs []string, m map[string]interface{}) ([]searcher.ISearcher, error) {
 	rs := make([]searcher.ISearcher, 0, len(plgs))
 	for _, name := range plgs {

number/category.go

@@ -2,7 +2,25 @@ package number
 
 import "strings"
 
+type Category string
+
+func (c Category) String() string {
+	return string(c)
+}
+
+const (
+	CatDefault Category = "DEFAULT"
+	CatFC2     Category = "FC2"
+)
+
 func IsFc2(number string) bool {
 	number = strings.ToUpper(number)
 	return strings.HasPrefix(number, "FC2")
 }
+
+func DetermineCategory(numberId string) Category {
+	if strings.HasPrefix(strings.ToUpper(numberId), "FC2") {
+		return CatFC2
+	}
+	return CatDefault //默认无分类
+}

number/model.go

@@ -5,17 +5,22 @@ import (
 )
 
 type Number struct {
-	number            string
+	numberId          string
 	isChineseSubtitle bool
 	isMultiCD         bool
 	multiCDIndex      int
 	isUncensorMovie   bool
 	is4k              bool
 	isLeak            bool
+	cat               Category
+}
+
+func (n *Number) GetCategory() Category {
+	return n.cat
 }
 
 func (n *Number) GetNumberID() string {
-	return n.number
+	return n.numberId
 }
 
 func (n *Number) GetIsChineseSubtitle() bool {

number/number.go

@@ -118,16 +118,17 @@ func Parse(str string) (*Number, error) {
 		return nil, fmt.Errorf("should not contain extname, str:%s", str)
 	}
 	rs := &Number{
-		number:            "",
+		numberId:          "",
 		isChineseSubtitle: false,
 		isMultiCD:         false,
 		multiCDIndex:      0,
 		isUncensorMovie:   false,
 	}
 	//提取后缀信息并对番号进行裁剪
 	number := resolveSuffixInfo(rs, str)
-	rs.number = strings.ToUpper(number)
+	rs.numberId = strings.ToUpper(number)
 	//通过番号直接填充信息(不进行裁剪)
 	resolveNumberInfo(rs, number)
+	rs.cat = DetermineCategory(rs.numberId)
 	return rs, nil
 }

number/number_test.go

@@ -9,65 +9,65 @@ import (
 func TestNumber(t *testing.T) {
 	checkList := map[string]*Number{
 		"HEYZO-3332.mp4": {
-			number:          "HEYZO-3332",
+			numberId:        "HEYZO-3332",
 			isUncensorMovie: true,
 		},
 		"052624_01.mp4": {
-			number:          "052624_01",
+			numberId:        "052624_01",
 			isUncensorMovie: true,
 		},
 		"052624_01-C.mp4": {
-			number:            "052624_01",
+			numberId:          "052624_01",
 			isChineseSubtitle: true,
 			isUncensorMovie:   true,
 		},
 		"052624_01-CD2.mp4": {
-			number:          "052624_01",
+			numberId:        "052624_01",
 			isUncensorMovie: true,
 			isMultiCD:       true,
 			multiCDIndex:    2,
 		},
 		"052624_01-CD3-C.mp4": {
-			number:            "052624_01",
+			numberId:          "052624_01",
 			isUncensorMovie:   true,
 			isMultiCD:         true,
 			multiCDIndex:      3,
 			isChineseSubtitle: true,
 		},
 		"052624_01_cd3_c.mp4": {
-			number:            "052624_01",
+			numberId:          "052624_01",
 			isUncensorMovie:   true,
 			isMultiCD:         true,
 			multiCDIndex:      3,
 			isChineseSubtitle: true,
 		},
 		"k0009-c_cd1-4k.mp4": {
-			number:            "K0009",
+			numberId:          "K0009",
 			isUncensorMovie:   true,
 			isMultiCD:         true,
 			multiCDIndex:      1,
 			isChineseSubtitle: true,
 			is4k:              true,
 		},
 		"n001-Cd1-4k.mp4": {
-			number:          "N001",
+			numberId:        "N001",
 			isUncensorMovie: true,
 			isMultiCD:       true,
 			multiCDIndex:    1,
 			is4k:            true,
 		},
 		"c-4k.mp4": {
-			number:            "C",
+			numberId:          "C",
 			isChineseSubtitle: false,
 			is4k:              true,
 		},
 		"-c-4k.mp4": {
-			number:            "",
+			numberId:          "",
 			isChineseSubtitle: true,
 			is4k:              true,
 		},
 		"abc-leak-c.mp4": {
-			number:            "ABC",
+			numberId:          "ABC",
 			isLeak:            true,
 			isChineseSubtitle: true,
 		},
@@ -83,3 +83,17 @@ func TestNumber(t *testing.T) {
 		assert.Equal(t, info.GetIs4K(), rs.GetIs4K())
 	}
 }
+
+func TestCategory(t *testing.T) {
+	{
+		n, err := Parse("fc2-ppv-12345")
+		assert.NoError(t, err)
+		assert.Equal(t, 1, len(n.GetCategory()))
+		assert.Equal(t, CatFC2, n.GetCategory()[0])
+	}
+	{
+		n, err := Parse("abc-0001")
+		assert.NoError(t, err)
+		assert.Equal(t, 0, len(n.GetCategory()))
+	}
+}

searcher/category_searcher.go

@@ -0,0 +1,40 @@
+package searcher
+
+import (
+	"context"
+	"yamdc/model"
+	"yamdc/number"
+
+	"github.com/xxxsen/common/logutil"
+	"go.uber.org/zap"
+)
+
+type categorySearcher struct {
+	defSearcher  []ISearcher
+	catSearchers map[number.Category][]ISearcher
+}
+
+func NewCategorySearcher(def []ISearcher, cats map[number.Category][]ISearcher) ISearcher {
+	return &categorySearcher{defSearcher: def, catSearchers: cats}
+}
+
+func (s *categorySearcher) Name() string {
+	return "category"
+}
+
+func (s *categorySearcher) Search(ctx context.Context, n *number.Number) (*model.AvMeta, bool, error) {
+	cat := n.GetCategory()
+	//没分类, 那么使用主链进行查询
+	//存在分类, 但是分类对应的链没有配置, 则使用主链进行查询
+	//如果已经存在分类链, 则不再进行降级
+	logger := logutil.GetLogger(ctx).With(zap.String("cat", string(cat)))
+	chain := s.defSearcher
+	if cat != number.CatDefault {
+		if c, ok := s.catSearchers[cat]; ok {
+			chain = c
+			logger.Debug("use cat chain for search")
+		}
+	}
+
+	return performGroupSearch(ctx, n, chain)
+}

searcher/group.go → searcher/group_searcher.go

@@ -21,11 +21,15 @@ func (g *group) Name() string {
 }
 
 func (g *group) Search(ctx context.Context, number *number.Number) (*model.AvMeta, bool, error) {
+	return performGroupSearch(ctx, number, g.ss)
+}
+
+func performGroupSearch(ctx context.Context, number *number.Number, ss []ISearcher) (*model.AvMeta, bool, error) {
 	var lastErr error
-	for _, s := range g.ss {
+	for _, s := range ss {
+		logutil.GetLogger(ctx).Debug("search number", zap.String("plugin", s.Name()))
 		meta, found, err := s.Search(ctx, number)
 		if err != nil {
-			logutil.GetLogger(context.Background()).Error("search fail", zap.String("searcher", s.Name()), zap.Error(err))
 			lastErr = err
 			continue
 		}

searcher/parser/date_parser.go

@@ -2,20 +2,20 @@ package parser
 
 import (
 	"context"
+	"time"
 	"yamdc/searcher/decoder"
-	"yamdc/searcher/utils"
 
 	"github.com/xxxsen/common/logutil"
 	"go.uber.org/zap"
 )
 
 func DefaultReleaseDateParser(ctx context.Context) decoder.NumberParseFunc {
 	return func(v string) int64 {
-		val, err := utils.ToTimestamp(v)
+		t, err := time.Parse(time.DateOnly, v)
 		if err != nil {
 			logutil.GetLogger(ctx).Error("decode release date failed", zap.Error(err), zap.String("data", v))
 			return 0
 		}
-		return val
+		return t.UnixMilli()
 	}
 }