diff --git a/engine/actor.go b/engine/actor.go index 5fb42f34..206ae5e8 100644 --- a/engine/actor.go +++ b/engine/actor.go @@ -1,6 +1,7 @@ package engine import ( + goerrors "errors" "fmt" "sort" "sync" @@ -85,7 +86,7 @@ func (e *Engine) searchActor(keyword string, provider mt.Provider, fallback bool innerResults, innerErr := innerSearch(name) if innerErr != nil && // ignore InfoNotFound error. - innerErr != mt.ErrInfoNotFound { + !goerrors.Is(innerErr, mt.ErrInfoNotFound) { // add error to chain and handle it later. errors = append(errors, innerErr) continue @@ -109,12 +110,19 @@ func (e *Engine) SearchActor(keyword, name string, fallback bool) ([]*model.Acto return e.searchActor(keyword, provider, fallback) } -func (e *Engine) SearchActorAll(keyword string, fallback bool) (results []*model.ActorSearchResult, err error) { +func (e *Engine) SearchActorAll(keyword, lang string, fallback bool) (results []*model.ActorSearchResult, err error) { var ( mu sync.Mutex wg sync.WaitGroup ) - for _, provider := range e.actorProviders { + + availableProviders, err := e.GetActorProvidersByLanguage(lang) + if err != nil { + return + } + e.logger.Infof("Actor Keyword: %s, Language: %s, Providers: %v", keyword, lang, availableProviders) + + for _, provider := range availableProviders { wg.Add(1) go func(provider mt.ActorProvider) { defer wg.Done() diff --git a/engine/engine.go b/engine/engine.go index 4c038a77..ff0c8843 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -7,6 +7,7 @@ import ( "time" "go.uber.org/zap" + "golang.org/x/text/language" "gorm.io/gorm" "github.com/metatube-community/metatube-sdk-go/common/fetch" @@ -97,6 +98,27 @@ func (e *Engine) GetActorProviders() map[string]mt.ActorProvider { return e.actorProviders } +func (e *Engine) GetActorProvidersByLanguage(lang string) (map[string]mt.ActorProvider, error) { + if lang == "" { + return e.actorProviders, nil + } + + providers := make(map[string]mt.ActorProvider) + + tag, err := language.Parse(lang) + if err != nil { + return nil, err + } + + m := language.NewMatcher([]language.Tag{tag}) + for _, provider := range e.actorProviders { + if _, _, c := m.Match(provider.Language()); c >= language.Low { + providers[strings.ToUpper(provider.Name())] = provider + } + } + return providers, nil +} + func (e *Engine) GetActorProviderByURL(rawURL string) (mt.ActorProvider, error) { u, err := url.Parse(rawURL) if err != nil { @@ -135,6 +157,27 @@ func (e *Engine) GetMovieProviders() map[string]mt.MovieProvider { return e.movieProviders } +func (e *Engine) GetMovieProvidersByLanguage(lang string) (map[string]mt.MovieProvider, error) { + if lang == "" { + return e.movieProviders, nil + } + + providers := make(map[string]mt.MovieProvider) + + tag, err := language.Parse(lang) + if err != nil { + return nil, err + } + + m := language.NewMatcher([]language.Tag{tag}) + for _, provider := range e.movieProviders { + if _, _, c := m.Match(provider.Language()); c >= language.Low { + providers[strings.ToUpper(provider.Name())] = provider + } + } + return providers, nil +} + func (e *Engine) GetMovieProviderByURL(rawURL string) (mt.MovieProvider, error) { u, err := url.Parse(rawURL) if err != nil { diff --git a/engine/movie.go b/engine/movie.go index 5327c7ff..5bd91fac 100644 --- a/engine/movie.go +++ b/engine/movie.go @@ -85,7 +85,7 @@ func (e *Engine) SearchMovie(keyword, name string, fallback bool) ([]*model.Movi return e.searchMovie(keyword, provider, fallback) } -func (e *Engine) searchMovieAll(keyword string) (results []*model.MovieSearchResult, err error) { +func (e *Engine) searchMovieAll(keyword string, providers map[string]mt.MovieProvider) (results []*model.MovieSearchResult, err error) { type response struct { Results []*model.MovieSearchResult Error error @@ -96,7 +96,7 @@ func (e *Engine) searchMovieAll(keyword string) (results []*model.MovieSearchRes respCh := make(chan response) var wg sync.WaitGroup - for _, provider := range e.movieProviders { + for _, provider := range providers { wg.Add(1) // Goroutine started time. startTime := time.Now() @@ -139,11 +139,17 @@ func (e *Engine) searchMovieAll(keyword string) (results []*model.MovieSearchRes } // SearchMovieAll searches the keyword from all providers. -func (e *Engine) SearchMovieAll(keyword string, fallback bool) (results []*model.MovieSearchResult, err error) { +func (e *Engine) SearchMovieAll(keyword, lang string, fallback bool) (results []*model.MovieSearchResult, err error) { if keyword = number.Trim(keyword); keyword == "" { return nil, mt.ErrInvalidKeyword } + availableProviders, err := e.GetMovieProvidersByLanguage(lang) + if err != nil { + return + } + e.logger.Infof("Movie Keyword: %s, Language: %s, Providers: %v", keyword, lang, availableProviders) + defer func() { if err != nil { return @@ -180,12 +186,16 @@ func (e *Engine) SearchMovieAll(keyword string, fallback bool) (results []*model // overwrite error. err = nil // append results. - results = append(results, innerResults...) + for _, result := range innerResults { + if _, ok := availableProviders[strings.ToUpper(result.Provider)]; ok { + results = append(results, result) + } + } } }() } - results, err = e.searchMovieAll(keyword) + results, err = e.searchMovieAll(keyword, availableProviders) return } diff --git a/provider/1pondo/core/core.go b/provider/1pondo/core/core.go index 31c6fe37..4941e95b 100644 --- a/provider/1pondo/core/core.go +++ b/provider/1pondo/core/core.go @@ -12,6 +12,7 @@ import ( "github.com/gocolly/colly/v2" "github.com/nlnwa/whatwg-url/url" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -45,7 +46,7 @@ type Core struct { } func (core *Core) Init() *Core { - core.Scraper = scraper.NewDefaultScraper(core.DefaultName, core.BaseURL, core.DefaultPriority, + core.Scraper = scraper.NewDefaultScraper(core.DefaultName, core.BaseURL, core.DefaultPriority, language.Japanese, scraper.WithHeaders(map[string]string{ "Content-Type": "application/json", }), diff --git a/provider/airav/airav.go b/provider/airav/airav.go index ae852567..2b527c6f 100644 --- a/provider/airav/airav.go +++ b/provider/airav/airav.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -39,7 +40,7 @@ type AirAV struct { func New() *AirAV { return &AirAV{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, scraper.WithHeaders(map[string]string{ "Origin": baseURL, "Referer": baseURL, diff --git a/provider/arzon/arzon.go b/provider/arzon/arzon.go index f5b8a2b1..493b5bf5 100644 --- a/provider/arzon/arzon.go +++ b/provider/arzon/arzon.go @@ -9,6 +9,7 @@ import ( "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/fetch" "github.com/metatube-community/metatube-sdk-go/common/number" @@ -44,7 +45,7 @@ type ARZON struct { func New() *ARZON { return &ARZON{ Fetcher: fetch.Default(&fetch.Config{Referer: baseURL}), - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority), + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese), } } diff --git a/provider/avbase/avbase.go b/provider/avbase/avbase.go index 9cbcb05b..a366c5c2 100644 --- a/provider/avbase/avbase.go +++ b/provider/avbase/avbase.go @@ -11,6 +11,7 @@ import ( "time" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/number" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -50,7 +51,7 @@ type AVBase struct { func New() *AVBase { return &AVBase{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, scraper.WithHeaders(map[string]string{ "Referer": baseURL, })), diff --git a/provider/aventertainments/aventertainments.go b/provider/aventertainments/aventertainments.go index ef314d6d..9e8e22cf 100644 --- a/provider/aventertainments/aventertainments.go +++ b/provider/aventertainments/aventertainments.go @@ -9,6 +9,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/number" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -38,7 +39,7 @@ type AVE struct { } func New() *AVE { - return &AVE{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &AVE{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (ave *AVE) GetMovieInfoByID(id string) (info *model.MovieInfo, err error) { diff --git a/provider/caribbeancom/core/core.go b/provider/caribbeancom/core/core.go index 226d632d..cf3bdaef 100644 --- a/provider/caribbeancom/core/core.go +++ b/provider/caribbeancom/core/core.go @@ -12,6 +12,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -36,6 +37,7 @@ func (core *Core) Init() *Core { core.DefaultName, core.BaseURL, core.DefaultPriority, + language.Japanese, scraper.WithDetectCharset()) return core } diff --git a/provider/duga/duga.go b/provider/duga/duga.go index aa412cec..bb626661 100644 --- a/provider/duga/duga.go +++ b/provider/duga/duga.go @@ -10,6 +10,7 @@ import ( "time" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/number" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -40,7 +41,7 @@ type DUGA struct { func New() *DUGA { return &DUGA{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority), + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese), } } diff --git a/provider/fanza/fanza.go b/provider/fanza/fanza.go index 16c93ff8..b796efe6 100644 --- a/provider/fanza/fanza.go +++ b/provider/fanza/fanza.go @@ -17,6 +17,7 @@ import ( "github.com/gocolly/colly/v2" "github.com/iancoleman/orderedmap" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/comparer" "github.com/metatube-community/metatube-sdk-go/common/number" @@ -57,7 +58,7 @@ type FANZA struct { func New() *FANZA { return &FANZA{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, scraper.WithCookies(baseURL, []*http.Cookie{ {Name: "age_check_done", Value: "1"}, })), diff --git a/provider/fc2/fc2.go b/provider/fc2/fc2.go index 5fd89bdf..fddd3c59 100644 --- a/provider/fc2/fc2.go +++ b/provider/fc2/fc2.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -33,7 +34,7 @@ type FC2 struct { } func New() *FC2 { - return &FC2{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &FC2{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (fc2 *FC2) NormalizeMovieID(id string) string { diff --git a/provider/fc2hub/fc2hub.go b/provider/fc2hub/fc2hub.go index 25f6043c..b9e9621c 100644 --- a/provider/fc2hub/fc2hub.go +++ b/provider/fc2hub/fc2hub.go @@ -10,6 +10,7 @@ import ( "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -39,7 +40,7 @@ type FC2HUB struct { } func New() *FC2HUB { - return &FC2HUB{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &FC2HUB{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (fc2hub *FC2HUB) GetMovieInfoByID(id string) (info *model.MovieInfo, err error) { diff --git a/provider/gcolle/gcolle.go b/provider/gcolle/gcolle.go index bf73c593..61579a6f 100644 --- a/provider/gcolle/gcolle.go +++ b/provider/gcolle/gcolle.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -34,7 +35,10 @@ type Gcolle struct { } func New() *Gcolle { - return &Gcolle{scraper.NewDefaultScraper(Name, baseURL, Priority, scraper.WithDetectCharset())} + return &Gcolle{scraper.NewDefaultScraper( + Name, baseURL, Priority, language.Japanese, + scraper.WithDetectCharset(), + )} } func (gcl *Gcolle) NormalizeMovieID(id string) string { diff --git a/provider/getchu/getchu.go b/provider/getchu/getchu.go index 0a5fcc01..9ac38b8a 100644 --- a/provider/getchu/getchu.go +++ b/provider/getchu/getchu.go @@ -10,6 +10,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -34,7 +35,7 @@ type Getchu struct { } func New() *Getchu { - return &Getchu{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &Getchu{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (gcu *Getchu) NormalizeMovieID(id string) string { diff --git a/provider/gfriends/gfriends.go b/provider/gfriends/gfriends.go index c100f214..af3732f1 100644 --- a/provider/gfriends/gfriends.go +++ b/provider/gfriends/gfriends.go @@ -8,6 +8,7 @@ import ( "time" "github.com/iancoleman/orderedmap" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/fetch" "github.com/metatube-community/metatube-sdk-go/common/reverse" @@ -47,6 +48,8 @@ func (gf *GFriends) Name() string { return Name } func (gf *GFriends) Priority() int { return Priority } +func (gf *GFriends) Language() language.Tag { return language.Japanese } + func (gf *GFriends) URL() *url.URL { return _baseURL } func (gf *GFriends) NormalizeActorID(id string) string { return id /* AS IS */ } diff --git a/provider/h0930/core/core.go b/provider/h0930/core/core.go index 0434d913..68751d68 100644 --- a/provider/h0930/core/core.go +++ b/provider/h0930/core/core.go @@ -10,6 +10,7 @@ import ( "time" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -34,6 +35,7 @@ func (core *Core) Init() *Core { core.DefaultName, core.BaseURL, core.DefaultPriority, + language.Japanese, scraper.WithDetectCharset()) return core } diff --git a/provider/heydouga/heydouga.go b/provider/heydouga/heydouga.go index 769a5459..4e6b61b1 100644 --- a/provider/heydouga/heydouga.go +++ b/provider/heydouga/heydouga.go @@ -10,6 +10,7 @@ import ( "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -35,7 +36,7 @@ type HeyDouga struct { } func New() *HeyDouga { - return &HeyDouga{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &HeyDouga{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (hey *HeyDouga) NormalizeMovieID(id string) string { diff --git a/provider/heyzo/heyzo.go b/provider/heyzo/heyzo.go index 5620d078..0def0090 100644 --- a/provider/heyzo/heyzo.go +++ b/provider/heyzo/heyzo.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/js" "github.com/metatube-community/metatube-sdk-go/common/m3u8" @@ -43,7 +44,7 @@ type Heyzo struct { } func New() *Heyzo { - return &Heyzo{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &Heyzo{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (hzo *Heyzo) GetMovieReviewsByID(id string) (reviews []*model.MovieReviewDetail, err error) { diff --git a/provider/internal/scraper/scraper.go b/provider/internal/scraper/scraper.go index a8cd65ec..1b670da6 100644 --- a/provider/internal/scraper/scraper.go +++ b/provider/internal/scraper/scraper.go @@ -5,6 +5,7 @@ import ( "time" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/provider" ) @@ -15,12 +16,13 @@ var _ provider.Provider = (*Scraper)(nil) type Scraper struct { name string priority int + lang language.Tag baseURL *url.URL c *colly.Collector } // NewScraper returns Provider implemented *Scraper. -func NewScraper(name, baseURL string, priority int, opts ...Option) *Scraper { +func NewScraper(name, baseURL string, priority int, lang language.Tag, opts ...Option) *Scraper { u, err := url.Parse(baseURL) if err != nil { panic(err) @@ -28,6 +30,7 @@ func NewScraper(name, baseURL string, priority int, opts ...Option) *Scraper { s := &Scraper{ name: name, priority: priority, + lang: lang, baseURL: u, c: colly.NewCollector(), } @@ -41,8 +44,8 @@ func NewScraper(name, baseURL string, priority int, opts ...Option) *Scraper { } // NewDefaultScraper returns a *Scraper with default options enabled. -func NewDefaultScraper(name, baseURL string, priority int, opts ...Option) *Scraper { - return NewScraper(name, baseURL, priority, append([]Option{ +func NewDefaultScraper(name, baseURL string, priority int, lang language.Tag, opts ...Option) *Scraper { + return NewScraper(name, baseURL, priority, lang, append([]Option{ WithAllowURLRevisit(), WithIgnoreRobotsTxt(), WithRandomUserAgent(), @@ -55,6 +58,8 @@ func (s *Scraper) URL() *url.URL { return s.baseURL } func (s *Scraper) Priority() int { return s.priority } +func (s *Scraper) Language() language.Tag { return s.lang } + func (s *Scraper) NormalizeMovieID(id string) string { return id /* AS IS */ } func (s *Scraper) ParseMovieIDFromURL(string) (string, error) { panic("unimplemented") } diff --git a/provider/jav321/jav321.go b/provider/jav321/jav321.go index f24d3150..2328f6b7 100644 --- a/provider/jav321/jav321.go +++ b/provider/jav321/jav321.go @@ -11,6 +11,7 @@ import ( "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/number" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -40,7 +41,7 @@ type JAV321 struct { } func New() *JAV321 { - return &JAV321{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &JAV321{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (jav *JAV321) SetRequestTimeout(_ time.Duration) { diff --git a/provider/javbus/javbus.go b/provider/javbus/javbus.go index de760afb..dbbb342a 100644 --- a/provider/javbus/javbus.go +++ b/provider/javbus/javbus.go @@ -10,6 +10,7 @@ import ( "sync" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/number" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -41,7 +42,7 @@ type JavBus struct { func New() *JavBus { return &JavBus{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, scraper.WithDisableRedirects(), scraper.WithCookies(baseURL, []*http.Cookie{ // existmag=all diff --git a/provider/kin8tengoku/kin8tengoku.go b/provider/kin8tengoku/kin8tengoku.go index 0aba8007..9bc2abca 100644 --- a/provider/kin8tengoku/kin8tengoku.go +++ b/provider/kin8tengoku/kin8tengoku.go @@ -13,6 +13,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -38,7 +39,7 @@ type KIN8 struct { } func New() *KIN8 { - return &KIN8{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &KIN8{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (k8 *KIN8) NormalizeMovieID(id string) string { diff --git a/provider/madouqu/madouqu.go b/provider/madouqu/madouqu.go index 057e9227..e878228d 100644 --- a/provider/madouqu/madouqu.go +++ b/provider/madouqu/madouqu.go @@ -9,6 +9,7 @@ import ( "time" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/number" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -38,7 +39,7 @@ type MadouQu struct { } func New() *MadouQu { - return &MadouQu{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &MadouQu{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Chinese)} } func (mdq *MadouQu) SetRequestTimeout(_ time.Duration) { diff --git a/provider/mgstage/mgstage.go b/provider/mgstage/mgstage.go index a7f83090..013151c7 100644 --- a/provider/mgstage/mgstage.go +++ b/provider/mgstage/mgstage.go @@ -13,6 +13,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/number" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -45,7 +46,7 @@ type MGS struct { func New() *MGS { return &MGS{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, scraper.WithCookies(baseURL, []*http.Cookie{ {Name: "adc", Value: "1"}, })), diff --git a/provider/mywife/mywife.go b/provider/mywife/mywife.go index df453971..3afd4d6d 100644 --- a/provider/mywife/mywife.go +++ b/provider/mywife/mywife.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/model" "github.com/metatube-community/metatube-sdk-go/provider" @@ -31,7 +32,7 @@ type MyWife struct { } func New() *MyWife { - return &MyWife{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &MyWife{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (mw *MyWife) NormalizeMovieID(id string) string { diff --git a/provider/pcolle/pcolle.go b/provider/pcolle/pcolle.go index e687eab7..6b3aa1d5 100644 --- a/provider/pcolle/pcolle.go +++ b/provider/pcolle/pcolle.go @@ -10,6 +10,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -35,7 +36,7 @@ type Pcolle struct { func New() *Pcolle { return &Pcolle{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, scraper.WithCookies(baseURL, []*http.Cookie{ {Name: "AGE_CONF", Value: "1"}, })), diff --git a/provider/prestige/prestige.go b/provider/prestige/prestige.go index 4df96543..e396863b 100644 --- a/provider/prestige/prestige.go +++ b/provider/prestige/prestige.go @@ -11,6 +11,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/number" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -41,7 +42,7 @@ type PRESTIGE struct { func New() *PRESTIGE { return &PRESTIGE{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, scraper.WithCookies(baseURL, []*http.Cookie{ {Name: "coc", Value: "1"}, {Name: "age_auth", Value: "1"}, diff --git a/provider/provider.go b/provider/provider.go index 604d9bc4..73cf0e06 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -5,6 +5,8 @@ import ( "net/url" "time" + "golang.org/x/text/language" + "github.com/metatube-community/metatube-sdk-go/model" ) @@ -12,6 +14,9 @@ type Provider interface { // Name returns the name of the provider. Name() string + // Language returns the default language of the provider. + Language() language.Tag + // Priority returns the matching priority of the provider. Priority() int diff --git a/provider/sod/sod.go b/provider/sod/sod.go index e69903fe..a585e4e0 100644 --- a/provider/sod/sod.go +++ b/provider/sod/sod.go @@ -8,6 +8,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/fetch" "github.com/metatube-community/metatube-sdk-go/common/number" @@ -44,7 +45,7 @@ type SOD struct { func New() *SOD { return &SOD{ Fetcher: fetch.Default(&fetch.Config{Referer: baseURL}), - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority), + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese), } } diff --git a/provider/tokyo-hot/tokyo-hot.go b/provider/tokyo-hot/tokyo-hot.go index 1997d1a9..a29ddf6f 100644 --- a/provider/tokyo-hot/tokyo-hot.go +++ b/provider/tokyo-hot/tokyo-hot.go @@ -12,6 +12,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -40,7 +41,7 @@ type TokyoHot struct { } func New() *TokyoHot { - return &TokyoHot{scraper.NewDefaultScraper(Name, baseURL, Priority, + return &TokyoHot{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, // Temporary workaround for unknown CA issue. scraper.WithTransport(&http.Transport{ Proxy: http.DefaultTransport.(*http.Transport).Proxy, diff --git a/provider/xslist/xslist.go b/provider/xslist/xslist.go index ad1f5a5e..17e2a3da 100644 --- a/provider/xslist/xslist.go +++ b/provider/xslist/xslist.go @@ -10,6 +10,7 @@ import ( "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" dt "gorm.io/datatypes" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -39,7 +40,9 @@ type XsList struct { } func New() *XsList { - return &XsList{scraper.NewDefaultScraper(Name, baseURL, Priority, scraper.WithDisableCookies())} + return &XsList{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, + scraper.WithDisableCookies(), + )} } func (xsl *XsList) GetActorInfoByID(id string) (info *model.ActorInfo, err error) { diff --git a/provider/xxx-av/xxx-av.go b/provider/xxx-av/xxx-av.go index 1a4ea5ea..6e566913 100644 --- a/provider/xxx-av/xxx-av.go +++ b/provider/xxx-av/xxx-av.go @@ -11,6 +11,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -36,7 +37,7 @@ type TripleX struct { func New() *TripleX { return &TripleX{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, scraper.WithCookies(baseURL, []*http.Cookie{ {Name: "acc_accept_lang", Value: "japanese"}, })), diff --git a/route/search.go b/route/search.go index d6b285c4..0c727960 100644 --- a/route/search.go +++ b/route/search.go @@ -21,6 +21,7 @@ const ( type searchQuery struct { Q string `form:"q" binding:"required"` Provider string `form:"provider"` + Language string `form:"lang"` Fallback bool `form:"fallback"` } @@ -53,7 +54,7 @@ func getSearch(app *engine.Engine, typ searchType) gin.HandlerFunc { if isValidURL { results, err = app.GetActorInfoByURL(query.Q, true /* always lazy */) } else if searchAll { - results, err = app.SearchActorAll(query.Q, query.Fallback) + results, err = app.SearchActorAll(query.Q, query.Language, query.Fallback) } else { results, err = app.SearchActor(query.Q, query.Provider, query.Fallback) } @@ -61,7 +62,7 @@ func getSearch(app *engine.Engine, typ searchType) gin.HandlerFunc { if isValidURL { results, err = app.GetMovieInfoByURL(query.Q, true /* always lazy */) } else if searchAll { - results, err = app.SearchMovieAll(query.Q, query.Fallback) + results, err = app.SearchMovieAll(query.Q, query.Language, query.Fallback) } else { results, err = app.SearchMovie(query.Q, query.Provider, query.Fallback) }