diff --git a/config/config.go b/config/config.go index 4ef6e2b..8cde03a 100644 --- a/config/config.go +++ b/config/config.go @@ -7,18 +7,24 @@ import ( "github.com/xxxsen/common/logger" ) +type CategoryPlugin struct { + Name string `json:"name"` + Plugins []string `json:"plugins"` +} + type Config struct { - ScanDir string `json:"scan_dir"` - SaveDir string `json:"save_dir"` - DataDir string `json:"data_dir"` - Naming string `json:"naming"` - PluginConfig map[string]interface{} `json:"plugin_config"` - HandlerConfig map[string]interface{} `json:"handler_config"` - Plugins []string `json:"plugins"` - Handlers []string `json:"handlers"` - ExtraMediaExts []string `json:"extra_media_exts"` - LogConfig logger.LogConfig `json:"log_config"` - SwitchConfig SwitchConfig `json:"switch_config"` + ScanDir string `json:"scan_dir"` + SaveDir string `json:"save_dir"` + DataDir string `json:"data_dir"` + Naming string `json:"naming"` + PluginConfig map[string]interface{} `json:"plugin_config"` + HandlerConfig map[string]interface{} `json:"handler_config"` + Plugins []string `json:"plugins"` + CategoryPlugins []CategoryPlugin `json:"category_plugins"` + Handlers []string `json:"handlers"` + ExtraMediaExts []string `json:"extra_media_exts"` + LogConfig logger.LogConfig `json:"log_config"` + SwitchConfig SwitchConfig `json:"switch_config"` } type SwitchConfig struct { @@ -40,6 +46,10 @@ func defaultConfig() *Config { "tktube", "avsox", }, + CategoryPlugins: []CategoryPlugin{ + //如果存在分配配置, 那么当番号被识别为特定分类的场景下, 将会使用分类插件直接查询 + {Name: "FC2", Plugins: []string{"fc2", "18av", "freejavbt", "tktube", "avsox"}}, + }, Handlers: []string{ "image_transcoder", "poster_cropper", diff --git a/main.go b/main.go index 9c092e6..7a99957 100644 --- a/main.go +++ b/main.go @@ -6,10 +6,12 @@ import ( "fmt" "log" "path/filepath" + "strings" "yamdc/capture" "yamdc/config" "yamdc/face" "yamdc/ffmpeg" + "yamdc/number" "yamdc/processor" "yamdc/processor/handler" "yamdc/searcher" @@ -49,6 +51,9 @@ func main() { logkit.Info("support plugins", zap.Strings("plugins", plugin.Plugins())) logkit.Info("support handlers", zap.Strings("handlers", handler.Handlers())) logkit.Info("current use plugins", zap.Strings("plugins", c.Plugins)) + for _, ct := range c.CategoryPlugins { + logkit.Info("-- cat plugins", zap.String("cat", ct.Name), zap.Strings("plugins", ct.Plugins)) + } logkit.Info("current use handlers", zap.Strings("handlers", c.Handlers)) logkit.Info("use naming rule", zap.String("rule", c.Naming)) logkit.Info("scrape from dir", zap.String("dir", c.ScanDir)) @@ -65,11 +70,15 @@ func main() { if err != nil { logkit.Fatal("build searcher failed", zap.Error(err)) } + catSs, err := buildCatSearcher(c.CategoryPlugins, c.PluginConfig) + if err != nil { + logkit.Fatal("build cat searcher failed", zap.Error(err)) + } ps, err := buildProcessor(c.Handlers, c.HandlerConfig) if err != nil { logkit.Fatal("build processor failed", zap.Error(err)) } - cap, err := buildCapture(c, ss, ps) + cap, err := buildCapture(c, ss, catSs, ps) if err != nil { logkit.Fatal("build capture runner failed", zap.Error(err)) } @@ -81,13 +90,13 @@ func main() { logkit.Info("run capture kit finish, all file scrape succ") } -func buildCapture(c *config.Config, ss []searcher.ISearcher, ps []processor.IProcessor) (*capture.Capture, error) { +func buildCapture(c *config.Config, ss []searcher.ISearcher, catSs map[number.Category][]searcher.ISearcher, ps []processor.IProcessor) (*capture.Capture, error) { opts := make([]capture.Option, 0, 10) opts = append(opts, capture.WithNamingRule(c.Naming), capture.WithScanDir(c.ScanDir), capture.WithSaveDir(c.SaveDir), - capture.WithSeacher(searcher.NewGroup(ss)), + capture.WithSeacher(searcher.NewCategorySearcher(ss, catSs)), capture.WithProcessor(processor.NewGroup(ps)), capture.WithEnableLinkMode(c.SwitchConfig.EnableLinkMode), capture.WithExtraMediaExtList(c.ExtraMediaExts), @@ -95,6 +104,18 @@ func buildCapture(c *config.Config, ss []searcher.ISearcher, ps []processor.IPro return capture.New(opts...) } +func buildCatSearcher(cplgs []config.CategoryPlugin, m map[string]interface{}) (map[number.Category][]searcher.ISearcher, error) { + rs := make(map[number.Category][]searcher.ISearcher, len(cplgs)) + for _, plg := range cplgs { + ss, err := buildSearcher(plg.Plugins, m) + if err != nil { + return nil, err + } + rs[number.Category(strings.ToUpper(plg.Name))] = ss + } + return rs, nil +} + func buildSearcher(plgs []string, m map[string]interface{}) ([]searcher.ISearcher, error) { rs := make([]searcher.ISearcher, 0, len(plgs)) for _, name := range plgs { diff --git a/number/category.go b/number/category.go index 1a1bc40..34a4729 100644 --- a/number/category.go +++ b/number/category.go @@ -2,7 +2,25 @@ package number import "strings" +type Category string + +func (c Category) String() string { + return string(c) +} + +const ( + CatDefault Category = "DEFAULT" + CatFC2 Category = "FC2" +) + func IsFc2(number string) bool { number = strings.ToUpper(number) return strings.HasPrefix(number, "FC2") } + +func DetermineCategory(numberId string) Category { + if strings.HasPrefix(strings.ToUpper(numberId), "FC2") { + return CatFC2 + } + return CatDefault //默认无分类 +} diff --git a/number/model.go b/number/model.go index a2a6347..8dad6aa 100644 --- a/number/model.go +++ b/number/model.go @@ -5,17 +5,22 @@ import ( ) type Number struct { - number string + numberId string isChineseSubtitle bool isMultiCD bool multiCDIndex int isUncensorMovie bool is4k bool isLeak bool + cat Category +} + +func (n *Number) GetCategory() Category { + return n.cat } func (n *Number) GetNumberID() string { - return n.number + return n.numberId } func (n *Number) GetIsChineseSubtitle() bool { diff --git a/number/number.go b/number/number.go index 4d2a045..5cead0f 100644 --- a/number/number.go +++ b/number/number.go @@ -118,7 +118,7 @@ func Parse(str string) (*Number, error) { return nil, fmt.Errorf("should not contain extname, str:%s", str) } rs := &Number{ - number: "", + numberId: "", isChineseSubtitle: false, isMultiCD: false, multiCDIndex: 0, @@ -126,8 +126,9 @@ func Parse(str string) (*Number, error) { } //提取后缀信息并对番号进行裁剪 number := resolveSuffixInfo(rs, str) - rs.number = strings.ToUpper(number) + rs.numberId = strings.ToUpper(number) //通过番号直接填充信息(不进行裁剪) resolveNumberInfo(rs, number) + rs.cat = DetermineCategory(rs.numberId) return rs, nil } diff --git a/number/number_test.go b/number/number_test.go index 8a67f73..4512bf3 100644 --- a/number/number_test.go +++ b/number/number_test.go @@ -9,40 +9,40 @@ import ( func TestNumber(t *testing.T) { checkList := map[string]*Number{ "HEYZO-3332.mp4": { - number: "HEYZO-3332", + numberId: "HEYZO-3332", isUncensorMovie: true, }, "052624_01.mp4": { - number: "052624_01", + numberId: "052624_01", isUncensorMovie: true, }, "052624_01-C.mp4": { - number: "052624_01", + numberId: "052624_01", isChineseSubtitle: true, isUncensorMovie: true, }, "052624_01-CD2.mp4": { - number: "052624_01", + numberId: "052624_01", isUncensorMovie: true, isMultiCD: true, multiCDIndex: 2, }, "052624_01-CD3-C.mp4": { - number: "052624_01", + numberId: "052624_01", isUncensorMovie: true, isMultiCD: true, multiCDIndex: 3, isChineseSubtitle: true, }, "052624_01_cd3_c.mp4": { - number: "052624_01", + numberId: "052624_01", isUncensorMovie: true, isMultiCD: true, multiCDIndex: 3, isChineseSubtitle: true, }, "k0009-c_cd1-4k.mp4": { - number: "K0009", + numberId: "K0009", isUncensorMovie: true, isMultiCD: true, multiCDIndex: 1, @@ -50,24 +50,24 @@ func TestNumber(t *testing.T) { is4k: true, }, "n001-Cd1-4k.mp4": { - number: "N001", + numberId: "N001", isUncensorMovie: true, isMultiCD: true, multiCDIndex: 1, is4k: true, }, "c-4k.mp4": { - number: "C", + numberId: "C", isChineseSubtitle: false, is4k: true, }, "-c-4k.mp4": { - number: "", + numberId: "", isChineseSubtitle: true, is4k: true, }, "abc-leak-c.mp4": { - number: "ABC", + numberId: "ABC", isLeak: true, isChineseSubtitle: true, }, @@ -83,3 +83,17 @@ func TestNumber(t *testing.T) { assert.Equal(t, info.GetIs4K(), rs.GetIs4K()) } } + +func TestCategory(t *testing.T) { + { + n, err := Parse("fc2-ppv-12345") + assert.NoError(t, err) + assert.Equal(t, 1, len(n.GetCategory())) + assert.Equal(t, CatFC2, n.GetCategory()[0]) + } + { + n, err := Parse("abc-0001") + assert.NoError(t, err) + assert.Equal(t, 0, len(n.GetCategory())) + } +} diff --git a/searcher/category_searcher.go b/searcher/category_searcher.go new file mode 100644 index 0000000..a508ae8 --- /dev/null +++ b/searcher/category_searcher.go @@ -0,0 +1,40 @@ +package searcher + +import ( + "context" + "yamdc/model" + "yamdc/number" + + "github.com/xxxsen/common/logutil" + "go.uber.org/zap" +) + +type categorySearcher struct { + defSearcher []ISearcher + catSearchers map[number.Category][]ISearcher +} + +func NewCategorySearcher(def []ISearcher, cats map[number.Category][]ISearcher) ISearcher { + return &categorySearcher{defSearcher: def, catSearchers: cats} +} + +func (s *categorySearcher) Name() string { + return "category" +} + +func (s *categorySearcher) Search(ctx context.Context, n *number.Number) (*model.AvMeta, bool, error) { + cat := n.GetCategory() + //没分类, 那么使用主链进行查询 + //存在分类, 但是分类对应的链没有配置, 则使用主链进行查询 + //如果已经存在分类链, 则不再进行降级 + logger := logutil.GetLogger(ctx).With(zap.String("cat", string(cat))) + chain := s.defSearcher + if cat != number.CatDefault { + if c, ok := s.catSearchers[cat]; ok { + chain = c + logger.Debug("use cat chain for search") + } + } + + return performGroupSearch(ctx, n, chain) +} diff --git a/searcher/default.go b/searcher/default_searcher.go similarity index 100% rename from searcher/default.go rename to searcher/default_searcher.go diff --git a/searcher/group.go b/searcher/group_searcher.go similarity index 70% rename from searcher/group.go rename to searcher/group_searcher.go index 320ffcb..171adcd 100644 --- a/searcher/group.go +++ b/searcher/group_searcher.go @@ -21,11 +21,15 @@ func (g *group) Name() string { } func (g *group) Search(ctx context.Context, number *number.Number) (*model.AvMeta, bool, error) { + return performGroupSearch(ctx, number, g.ss) +} + +func performGroupSearch(ctx context.Context, number *number.Number, ss []ISearcher) (*model.AvMeta, bool, error) { var lastErr error - for _, s := range g.ss { + for _, s := range ss { + logutil.GetLogger(ctx).Debug("search number", zap.String("plugin", s.Name())) meta, found, err := s.Search(ctx, number) if err != nil { - logutil.GetLogger(context.Background()).Error("search fail", zap.String("searcher", s.Name()), zap.Error(err)) lastErr = err continue } diff --git a/searcher/parser/date_parser.go b/searcher/parser/date_parser.go index b5e26fe..c7586cf 100644 --- a/searcher/parser/date_parser.go +++ b/searcher/parser/date_parser.go @@ -2,8 +2,8 @@ package parser import ( "context" + "time" "yamdc/searcher/decoder" - "yamdc/searcher/utils" "github.com/xxxsen/common/logutil" "go.uber.org/zap" @@ -11,11 +11,11 @@ import ( func DefaultReleaseDateParser(ctx context.Context) decoder.NumberParseFunc { return func(v string) int64 { - val, err := utils.ToTimestamp(v) + t, err := time.Parse(time.DateOnly, v) if err != nil { logutil.GetLogger(ctx).Error("decode release date failed", zap.Error(err), zap.String("data", v)) return 0 } - return val + return t.UnixMilli() } } diff --git a/searcher/parser/duration_parser.go b/searcher/parser/duration_parser.go index c4b1a76..eae847b 100644 --- a/searcher/parser/duration_parser.go +++ b/searcher/parser/duration_parser.go @@ -2,16 +2,21 @@ package parser import ( "context" + "errors" "math" + "regexp" "strconv" "strings" "yamdc/searcher/decoder" - "yamdc/searcher/utils" "github.com/xxxsen/common/logutil" "go.uber.org/zap" ) +var ( + defaultDurationRegexp = regexp.MustCompile(`\s*(\d+)\s*.+`) +) + func DefaultHHMMSSDurationParser(ctx context.Context) decoder.NumberParseFunc { return func(v string) int64 { res := strings.Split(v, ":") @@ -35,7 +40,7 @@ func DefaultHHMMSSDurationParser(ctx context.Context) decoder.NumberParseFunc { func DefaultDurationParser(ctx context.Context) decoder.NumberParseFunc { return func(v string) int64 { - val, err := utils.ToDuration(v) + val, err := toDuration(v) if err != nil { logutil.GetLogger(ctx).Error("decode duration failed", zap.Error(err), zap.String("data", v)) return 0 @@ -43,3 +48,19 @@ func DefaultDurationParser(ctx context.Context) decoder.NumberParseFunc { return val } } + +func toDuration(timeStr string) (int64, error) { + re := defaultDurationRegexp + matches := re.FindStringSubmatch(timeStr) + if len(matches) <= 1 { + return 0, errors.New("invalid time format") + } + + number, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, err + } + seconds := number * 60 + + return int64(seconds), nil +} diff --git a/searcher/parser/duration_parser_test.go b/searcher/parser/duration_parser_test.go index 679812b..ff12b40 100644 --- a/searcher/parser/duration_parser_test.go +++ b/searcher/parser/duration_parser_test.go @@ -22,3 +22,20 @@ func TestHHMMSS(t *testing.T) { assert.Equal(t, tst.sec, out) } } + +func TestConv(t *testing.T) { + sts := []struct { + in string + out int64 + }{ + {"47分钟", 47 * 60}, + {" 10分钟", 600}, + {"140分", 140 * 60}, + {"117分鐘", 117 * 60}, + } + for _, st := range sts { + out, err := toDuration(st.in) + assert.NoError(t, err) + assert.Equal(t, st.out, out) + } +} diff --git a/searcher/plugin/18av.go b/searcher/plugin/18av.go index ebcf72c..b6ff69e 100644 --- a/searcher/plugin/18av.go +++ b/searcher/plugin/18av.go @@ -14,10 +14,6 @@ type av18 struct { DefaultPlugin } -func (p *av18) OnPrecheckRequest(ctx *PluginContext, n *number.Number) (bool, error) { - return number.IsFc2(n.GetNumberID()), nil -} - func (p *av18) OnMakeHTTPRequest(ctx *PluginContext, number *number.Number) (*http.Request, error) { uri := fmt.Sprintf("https://18av.me/cn/search.php?kw_type=key&kw=%s", number.GetNumberID()) ctx.SetKey("number", number.GetNumberID()) diff --git a/searcher/plugin/airav/airav.go b/searcher/plugin/airav/airav.go index 7991c38..24d65d6 100644 --- a/searcher/plugin/airav/airav.go +++ b/searcher/plugin/airav/airav.go @@ -7,8 +7,8 @@ import ( "strings" "yamdc/model" "yamdc/number" + "yamdc/searcher/parser" "yamdc/searcher/plugin" - "yamdc/searcher/utils" "github.com/xxxsen/common/logutil" "go.uber.org/zap" @@ -46,7 +46,7 @@ func (p *airav) OnDecodeHTTPData(ctx *plugin.PluginContext, data []byte) (*model Title: result.Name, Plot: result.Description, Actors: p.readActors(&result), - ReleaseDate: utils.ToTimestampOrDefault(result.PublishDate, 0), + ReleaseDate: parser.DefaultReleaseDateParser(ctx.GetContext())(result.PublishDate), Studio: p.readStudio(&result), Genres: p.readGenres(&result), Cover: &model.File{ diff --git a/searcher/plugin/fc2.go b/searcher/plugin/fc2.go index 44754c3..57d9892 100644 --- a/searcher/plugin/fc2.go +++ b/searcher/plugin/fc2.go @@ -23,10 +23,6 @@ type fc2 struct { DefaultPlugin } -func (p *fc2) OnPrecheckRequest(ctx *PluginContext, n *number.Number) (bool, error) { - return number.IsFc2(n.GetNumberID()), nil -} - func (p *fc2) OnMakeHTTPRequest(ctx *PluginContext, n *number.Number) (*http.Request, error) { number := strings.ToLower(n.GetNumberID()) res := defaultFc2NumberParser.FindStringSubmatch(number) diff --git a/searcher/plugin/jav321.go b/searcher/plugin/jav321.go index b94ee7c..7dd9b07 100644 --- a/searcher/plugin/jav321.go +++ b/searcher/plugin/jav321.go @@ -29,25 +29,6 @@ func (p *jav321) OnMakeHTTPRequest(ctx *PluginContext, number *number.Number) (* return req, nil } -// func (s *jav321) OnHandleHTTPRequest(ctx *PluginContext, invoker HTTPInvoker, req *http.Request) (*http.Response, error) { -// rsp, err := invoker(ctx, req) -// if err != nil { -// return nil, err -// } -// if rsp.StatusCode != http.StatusMovedPermanently { -// return nil, fmt.Errorf("number may not found, skip") -// } -// uri, err := rsp.Location() -// if err != nil { -// return nil, fmt.Errorf("read location failed, err:%w", err) -// } -// newReq, err := http.NewRequest(http.MethodGet, uri.String(), nil) -// if err != nil { -// return nil, err -// } -// return invoker(ctx, newReq) -// } - func (s *jav321) defaultStringProcessor(v string) string { v = strings.Trim(v, ": \t") return strings.TrimSpace(v) diff --git a/searcher/plugin/tktube.go b/searcher/plugin/tktube.go index f0651d0..5e99635 100644 --- a/searcher/plugin/tktube.go +++ b/searcher/plugin/tktube.go @@ -14,10 +14,6 @@ type tktube struct { DefaultPlugin } -func (p *tktube) OnPrecheckRequest(ctx *PluginContext, n *number.Number) (bool, error) { - return number.IsFc2(n.GetNumberID()), nil -} - func (p *tktube) OnMakeHTTPRequest(ctx *PluginContext, n *number.Number) (*http.Request, error) { nid := strings.ReplaceAll(n.GetNumberID(), "-", "--") ctx.SetKey("number", n.GetNumberID()) diff --git a/searcher/utils/conv_utils.go b/searcher/utils/conv_utils.go deleted file mode 100644 index fabbccd..0000000 --- a/searcher/utils/conv_utils.go +++ /dev/null @@ -1,43 +0,0 @@ -package utils - -import ( - "errors" - "regexp" - "strconv" - "time" -) - -var ( - defaultDurationRegexp = regexp.MustCompile(`\s*(\d+)\s*.+`) -) - -func ToTimestamp(date string) (int64, error) { - t, err := time.Parse(time.DateOnly, date) - if err != nil { - return 0, err - } - return t.UnixMilli(), nil -} - -func ToTimestampOrDefault(date string, def int64) int64 { - if v, err := ToTimestamp(date); err == nil { - return v - } - return def -} - -func ToDuration(timeStr string) (int64, error) { - re := defaultDurationRegexp - matches := re.FindStringSubmatch(timeStr) - if len(matches) <= 1 { - return 0, errors.New("invalid time format") - } - - number, err := strconv.Atoi(matches[1]) - if err != nil { - return 0, err - } - seconds := number * 60 - - return int64(seconds), nil -} diff --git a/searcher/utils/conv_utils_test.go b/searcher/utils/conv_utils_test.go deleted file mode 100644 index 477e846..0000000 --- a/searcher/utils/conv_utils_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package utils - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestConv(t *testing.T) { - sts := []struct { - in string - out int64 - }{ - {"47分钟", 47 * 60}, - {" 10分钟", 600}, - {"140分", 140 * 60}, - {"117分鐘", 117 * 60}, - } - for _, st := range sts { - out, err := ToDuration(st.in) - assert.NoError(t, err) - assert.Equal(t, st.out, out) - } -}