Feature(search): add language option

engine/actor.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	goerrors "errors"
 	"fmt"
 	"sort"
 	"sync"
@@ -85,7 +86,7 @@ func (e *Engine) searchActor(keyword string, provider mt.Provider, fallback bool
 		innerResults, innerErr := innerSearch(name)
 		if innerErr != nil &&
 			// ignore InfoNotFound error.
-			innerErr != mt.ErrInfoNotFound {
+			!goerrors.Is(innerErr, mt.ErrInfoNotFound) {
 			// add error to chain and handle it later.
 			errors = append(errors, innerErr)
 			continue
@@ -109,12 +110,19 @@ func (e *Engine) SearchActor(keyword, name string, fallback bool) ([]*model.Acto
 	return e.searchActor(keyword, provider, fallback)
 }
 
-func (e *Engine) SearchActorAll(keyword string, fallback bool) (results []*model.ActorSearchResult, err error) {
+func (e *Engine) SearchActorAll(keyword, lang string, fallback bool) (results []*model.ActorSearchResult, err error) {
 	var (
 		mu sync.Mutex
 		wg sync.WaitGroup
 	)
-	for _, provider := range e.actorProviders {
+
+	availableProviders, err := e.GetActorProvidersByLanguage(lang)
+	if err != nil {
+		return
+	}
+	e.logger.Infof("Actor Keyword: %s, Language: %s, Providers: %v", keyword, lang, availableProviders)
+
+	for _, provider := range availableProviders {
 		wg.Add(1)
 		go func(provider mt.ActorProvider) {
 			defer wg.Done()

engine/engine.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"go.uber.org/zap"
+	"golang.org/x/text/language"
 	"gorm.io/gorm"
 
 	"github.com/metatube-community/metatube-sdk-go/common/fetch"
@@ -97,6 +98,27 @@ func (e *Engine) GetActorProviders() map[string]mt.ActorProvider {
 	return e.actorProviders
 }
 
+func (e *Engine) GetActorProvidersByLanguage(lang string) (map[string]mt.ActorProvider, error) {
+	if lang == "" {
+		return e.actorProviders, nil
+	}
+
+	providers := make(map[string]mt.ActorProvider)
+
+	tag, err := language.Parse(lang)
+	if err != nil {
+		return nil, err
+	}
+
+	m := language.NewMatcher([]language.Tag{tag})
+	for _, provider := range e.actorProviders {
+		if _, _, c := m.Match(provider.Language()); c >= language.Low {
+			providers[strings.ToUpper(provider.Name())] = provider
+		}
+	}
+	return providers, nil
+}
+
 func (e *Engine) GetActorProviderByURL(rawURL string) (mt.ActorProvider, error) {
 	u, err := url.Parse(rawURL)
 	if err != nil {
@@ -135,6 +157,27 @@ func (e *Engine) GetMovieProviders() map[string]mt.MovieProvider {
 	return e.movieProviders
 }
 
+func (e *Engine) GetMovieProvidersByLanguage(lang string) (map[string]mt.MovieProvider, error) {
+	if lang == "" {
+		return e.movieProviders, nil
+	}
+
+	providers := make(map[string]mt.MovieProvider)
+
+	tag, err := language.Parse(lang)
+	if err != nil {
+		return nil, err
+	}
+
+	m := language.NewMatcher([]language.Tag{tag})
+	for _, provider := range e.movieProviders {
+		if _, _, c := m.Match(provider.Language()); c >= language.Low {
+			providers[strings.ToUpper(provider.Name())] = provider
+		}
+	}
+	return providers, nil
+}
+
 func (e *Engine) GetMovieProviderByURL(rawURL string) (mt.MovieProvider, error) {
 	u, err := url.Parse(rawURL)
 	if err != nil {

engine/movie.go

@@ -85,7 +85,7 @@ func (e *Engine) SearchMovie(keyword, name string, fallback bool) ([]*model.Movi
 	return e.searchMovie(keyword, provider, fallback)
 }
 
-func (e *Engine) searchMovieAll(keyword string) (results []*model.MovieSearchResult, err error) {
+func (e *Engine) searchMovieAll(keyword string, providers map[string]mt.MovieProvider) (results []*model.MovieSearchResult, err error) {
 	type response struct {
 		Results   []*model.MovieSearchResult
 		Error     error
@@ -96,7 +96,7 @@ func (e *Engine) searchMovieAll(keyword string) (results []*model.MovieSearchRes
 	respCh := make(chan response)
 
 	var wg sync.WaitGroup
-	for _, provider := range e.movieProviders {
+	for _, provider := range providers {
 		wg.Add(1)
 		// Goroutine started time.
 		startTime := time.Now()
@@ -139,11 +139,17 @@ func (e *Engine) searchMovieAll(keyword string) (results []*model.MovieSearchRes
 }
 
 // SearchMovieAll searches the keyword from all providers.
-func (e *Engine) SearchMovieAll(keyword string, fallback bool) (results []*model.MovieSearchResult, err error) {
+func (e *Engine) SearchMovieAll(keyword, lang string, fallback bool) (results []*model.MovieSearchResult, err error) {
 	if keyword = number.Trim(keyword); keyword == "" {
 		return nil, mt.ErrInvalidKeyword
 	}
 
+	availableProviders, err := e.GetMovieProvidersByLanguage(lang)
+	if err != nil {
+		return
+	}
+	e.logger.Infof("Movie Keyword: %s, Language: %s, Providers: %v", keyword, lang, availableProviders)
+
 	defer func() {
 		if err != nil {
 			return
@@ -180,12 +186,16 @@ func (e *Engine) SearchMovieAll(keyword string, fallback bool) (results []*model
 				// overwrite error.
 				err = nil
 				// append results.
-				results = append(results, innerResults...)
+				for _, result := range innerResults {
+					if _, ok := availableProviders[strings.ToUpper(result.Provider)]; ok {
+						results = append(results, result)
+					}
+				}
 			}
 		}()
 	}
 
-	results, err = e.searchMovieAll(keyword)
+	results, err = e.searchMovieAll(keyword, availableProviders)
 	return
 }
 

provider/1pondo/core/core.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/gocolly/colly/v2"
 	"github.com/nlnwa/whatwg-url/url"
+	"golang.org/x/text/language"
 
 	"github.com/metatube-community/metatube-sdk-go/common/parser"
 	"github.com/metatube-community/metatube-sdk-go/model"
@@ -45,7 +46,7 @@ type Core struct {
 }
 
 func (core *Core) Init() *Core {
-	core.Scraper = scraper.NewDefaultScraper(core.DefaultName, core.BaseURL, core.DefaultPriority,
+	core.Scraper = scraper.NewDefaultScraper(core.DefaultName, core.BaseURL, core.DefaultPriority, language.Japanese,
 		scraper.WithHeaders(map[string]string{
 			"Content-Type": "application/json",
 		}),

provider/airav/airav.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/gocolly/colly/v2"
+	"golang.org/x/text/language"
 
 	"github.com/metatube-community/metatube-sdk-go/common/parser"
 	"github.com/metatube-community/metatube-sdk-go/model"
@@ -39,7 +40,7 @@ type AirAV struct {
 
 func New() *AirAV {
 	return &AirAV{
-		Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority,
+		Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese,
 			scraper.WithHeaders(map[string]string{
 				"Origin":  baseURL,
 				"Referer": baseURL,

provider/arzon/arzon.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/gocolly/colly/v2"
 	"golang.org/x/net/html"
+	"golang.org/x/text/language"
 
 	"github.com/metatube-community/metatube-sdk-go/common/fetch"
 	"github.com/metatube-community/metatube-sdk-go/common/number"
@@ -44,7 +45,7 @@ type ARZON struct {
 func New() *ARZON {
 	return &ARZON{
 		Fetcher: fetch.Default(&fetch.Config{Referer: baseURL}),
-		Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority),
+		Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese),
 	}
 }
 

provider/avbase/avbase.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/gocolly/colly/v2"
+	"golang.org/x/text/language"
 
 	"github.com/metatube-community/metatube-sdk-go/common/number"
 	"github.com/metatube-community/metatube-sdk-go/common/parser"
@@ -50,7 +51,7 @@ type AVBase struct {
 
 func New() *AVBase {
 	return &AVBase{
-		Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority,
+		Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese,
 			scraper.WithHeaders(map[string]string{
 				"Referer": baseURL,
 			})),

provider/aventertainments/aventertainments.go

@@ -9,6 +9,7 @@ import (
 	"github.com/antchfx/htmlquery"
 	"github.com/gocolly/colly/v2"
 	"golang.org/x/net/html"
+	"golang.org/x/text/language"
 
 	"github.com/metatube-community/metatube-sdk-go/common/number"
 	"github.com/metatube-community/metatube-sdk-go/common/parser"
@@ -38,7 +39,7 @@ type AVE struct {
 }
 
 func New() *AVE {
-	return &AVE{scraper.NewDefaultScraper(Name, baseURL, Priority)}
+	return &AVE{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)}
 }
 
 func (ave *AVE) GetMovieInfoByID(id string) (info *model.MovieInfo, err error) {

provider/caribbeancom/core/core.go

@@ -12,6 +12,7 @@ import (
 	"github.com/antchfx/htmlquery"
 	"github.com/gocolly/colly/v2"
 	"golang.org/x/net/html"
+	"golang.org/x/text/language"
 
 	"github.com/metatube-community/metatube-sdk-go/common/parser"
 	"github.com/metatube-community/metatube-sdk-go/model"
@@ -36,6 +37,7 @@ func (core *Core) Init() *Core {
 		core.DefaultName,
 		core.BaseURL,
 		core.DefaultPriority,
+		language.Japanese,
 		scraper.WithDetectCharset())
 	return core
 }

provider/duga/duga.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/gocolly/colly/v2"
+	"golang.org/x/text/language"
 
 	"github.com/metatube-community/metatube-sdk-go/common/number"
 	"github.com/metatube-community/metatube-sdk-go/common/parser"
@@ -40,7 +41,7 @@ type DUGA struct {
 
 func New() *DUGA {
 	return &DUGA{
-		Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority),
+		Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese),
 	}
 }
 

provider/fanza/fanza.go

@@ -17,6 +17,7 @@ import (
 	"github.com/gocolly/colly/v2"
 	"github.com/iancoleman/orderedmap"
 	"golang.org/x/net/html"
+	"golang.org/x/text/language"
 
 	"github.com/metatube-community/metatube-sdk-go/common/comparer"
 	"github.com/metatube-community/metatube-sdk-go/common/number"
@@ -57,7 +58,7 @@ type FANZA struct {
 
 func New() *FANZA {
 	return &FANZA{
-		Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority,
+		Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese,
 			scraper.WithCookies(baseURL, []*http.Cookie{
 				{Name: "age_check_done", Value: "1"},
 			})),

provider/fc2/fc2.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/gocolly/colly/v2"
+	"golang.org/x/text/language"
 
 	"github.com/metatube-community/metatube-sdk-go/common/parser"
 	"github.com/metatube-community/metatube-sdk-go/model"
@@ -33,7 +34,7 @@ type FC2 struct {
 }
 
 func New() *FC2 {
-	return &FC2{scraper.NewDefaultScraper(Name, baseURL, Priority)}
+	return &FC2{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)}
 }
 
 func (fc2 *FC2) NormalizeMovieID(id string) string {

provider/fc2hub/fc2hub.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/gocolly/colly/v2"
 	"golang.org/x/net/html"
+	"golang.org/x/text/language"
 
 	"github.com/metatube-community/metatube-sdk-go/common/parser"
 	"github.com/metatube-community/metatube-sdk-go/model"
@@ -39,7 +40,7 @@ type FC2HUB struct {
 }
 
 func New() *FC2HUB {
-	return &FC2HUB{scraper.NewDefaultScraper(Name, baseURL, Priority)}
+	return &FC2HUB{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)}
 }
 
 func (fc2hub *FC2HUB) GetMovieInfoByID(id string) (info *model.MovieInfo, err error) {

provider/gcolle/gcolle.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/gocolly/colly/v2"
+	"golang.org/x/text/language"
 
 	"github.com/metatube-community/metatube-sdk-go/common/parser"
 	"github.com/metatube-community/metatube-sdk-go/model"
@@ -34,7 +35,10 @@ type Gcolle struct {
 }
 
 func New() *Gcolle {
-	return &Gcolle{scraper.NewDefaultScraper(Name, baseURL, Priority, scraper.WithDetectCharset())}
+	return &Gcolle{scraper.NewDefaultScraper(
+		Name, baseURL, Priority, language.Japanese,
+		scraper.WithDetectCharset(),
+	)}
 }
 
 func (gcl *Gcolle) NormalizeMovieID(id string) string {

provider/getchu/getchu.go

@@ -10,6 +10,7 @@ import (
 	"github.com/antchfx/htmlquery"
 	"github.com/gocolly/colly/v2"
 	"golang.org/x/net/html"
+	"golang.org/x/text/language"
 
 	"github.com/metatube-community/metatube-sdk-go/common/parser"
 	"github.com/metatube-community/metatube-sdk-go/model"
@@ -34,7 +35,7 @@ type Getchu struct {
 }
 
 func New() *Getchu {
-	return &Getchu{scraper.NewDefaultScraper(Name, baseURL, Priority)}
+	return &Getchu{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)}
 }
 
 func (gcu *Getchu) NormalizeMovieID(id string) string {

provider/gfriends/gfriends.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/iancoleman/orderedmap"
+	"golang.org/x/text/language"
 
 	"github.com/metatube-community/metatube-sdk-go/common/fetch"
 	"github.com/metatube-community/metatube-sdk-go/common/reverse"
@@ -47,6 +48,8 @@ func (gf *GFriends) Name() string { return Name }
 
 func (gf *GFriends) Priority() int { return Priority }
 
+func (gf *GFriends) Language() language.Tag { return language.Japanese }
+
 func (gf *GFriends) URL() *url.URL { return _baseURL }
 
 func (gf *GFriends) NormalizeActorID(id string) string { return id /* AS IS */ }

provider/h0930/core/core.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/gocolly/colly/v2"
+	"golang.org/x/text/language"
 
 	"github.com/metatube-community/metatube-sdk-go/common/parser"
 	"github.com/metatube-community/metatube-sdk-go/model"
@@ -34,6 +35,7 @@ func (core *Core) Init() *Core {
 		core.DefaultName,
 		core.BaseURL,
 		core.DefaultPriority,
+		language.Japanese,
 		scraper.WithDetectCharset())
 	return core
 }