Merge pull request #17 from xxxsen/xxxsen/feature/support_proxy

支持代理

client/client.go

@@ -1,36 +1,67 @@
 package client
 
 import (
+	"fmt"
 	"net/http"
 	"net/http/cookiejar"
-	"time"
+	"net/url"
 
 	"github.com/imroc/req/v3"
 )
 
+var defaultInst IHTTPClient
+
+func init() {
+	SetDefault(MustNewClient()) //初始化default, 避免无初始化使用直接炸了
+}
+
+func SetDefault(c IHTTPClient) {
+	defaultInst = c
+}
+
+func DefaultClient() IHTTPClient {
+	return defaultInst
+}
+
 type IHTTPClient interface {
 	Do(req *http.Request) (*http.Response, error)
 }
 
-type Client struct {
+type clientWrap struct {
 	client *http.Client
 }
 
-func NewClient() IHTTPClient {
+func NewClient(opts ...Option) (IHTTPClient, error) {
+	c := applyOpts(opts...)
 	// 第三方客户端用着不是很习惯, 考虑到我们需要用到的功能都是在transport里面,
 	// 所以这里直接把第三方客户端的transport提出来用...
 	reqClient := req.NewClient()
-	reqClient.ImpersonateChrome()
+	reqClient.ImpersonateChrome() //fixme: 部分逻辑看着, 有使用到底层的client, 但是, 貌似不使用这部分东西也能正常绕过cf?
 	t := reqClient.Transport
 	jar, _ := cookiejar.New(nil)
 	client := &http.Client{
 		Transport: t,
 		Jar:       jar,
-		Timeout:   10 * time.Second,
+		Timeout:   c.timeout,
+	}
+	if len(c.proxy) > 0 {
+		proxyUrl, err := url.Parse(c.proxy)
+		if err != nil {
+			return nil, fmt.Errorf("parse proxy link failed, err:%w", err)
+		}
+		t.Proxy = http.ProxyURL(proxyUrl) // set proxy
+	}
+	return &clientWrap{client: client}, nil
+}
+
+func MustNewClient(opts ...Option) IHTTPClient {
+	c, err := NewClient(opts...)
+	if err != nil {
+		panic(err)
 	}
-	return &Client{client: client}
+	return c
 }
 
-func (c *Client) Do(req *http.Request) (*http.Response, error) {
+func (c *clientWrap) Do(req *http.Request) (*http.Response, error) {
 	return c.client.Do(req)
 }

client/config.go

@@ -0,0 +1,33 @@
+package client
+
+import "time"
+
+type config struct {
+	timeout time.Duration
+	proxy   string
+}
+
+type Option func(c *config)
+
+func WithTimeout(t time.Duration) Option {
+	return func(c *config) {
+		c.timeout = t
+	}
+}
+
+func WithProxy(link string) Option {
+	return func(c *config) {
+		c.proxy = link
+	}
+}
+
+func applyOpts(opts ...Option) *config {
+	c := &config{}
+	for _, opt := range opts {
+		opt(c)
+	}
+	if c.timeout == 0 {
+		c.timeout = 10 * time.Second
+	}
+	return c
+}

config/config.go

@@ -17,6 +17,17 @@ type Dependency struct {
 	RelPath string `json:"rel_path"`
 }
 
+type ProxyConfig struct {
+	Addr     string `json:"addr"`
+	User     string `json:"user"`
+	Password string `json:"password"`
+}
+
+type NetworkConfig struct {
+	Timeout int64  `json:"timeout"` //单位为秒
+	Proxy   string `json:"proxy"`
+}
+
 type Config struct {
 	ScanDir         string                 `json:"scan_dir"`
 	SaveDir         string                 `json:"save_dir"`
@@ -30,6 +41,7 @@ type Config struct {
 	ExtraMediaExts  []string               `json:"extra_media_exts"`
 	LogConfig       logger.LogConfig       `json:"log_config"`
 	Dependencies    []Dependency           `json:"dependencies"`
+	NetworkConfig   NetworkConfig          `json:"network_config"`
 }
 
 func defaultConfig() *Config {

downloadmgr/download_manager_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestDownloa(t *testing.T) {
-	m := NewManager(client.NewClient())
+	m := NewManager(client.MustNewClient())
 	err := m.Download("https://github.com/Kagami/go-face-testdata/raw/master/models/shape_predictor_5_face_landmarks.dat", "testdata/abc.dat")
 	assert.NoError(t, err)
 }

main.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"path/filepath"
 	"strings"
+	"time"
 	"yamdc/capture"
 	"yamdc/client"
 	"yamdc/config"
@@ -22,6 +23,7 @@ import (
 	"yamdc/searcher"
 	"yamdc/store"
 	"yamdc/translator"
+	"yamdc/translator/googletranslator"
 
 	"github.com/xxxsen/common/logger"
 	"github.com/xxxsen/common/logutil"
@@ -43,6 +45,9 @@ func main() {
 	if err := precheckDir(c); err != nil {
 		logkit.Fatal("precheck dir failed", zap.Error(err))
 	}
+	if err := setupHTTPClient(c); err != nil {
+		logkit.Fatal("setup http client failed", zap.Error(err))
+	}
 	logkit.Info("check dependencies...")
 	if err := ensureDependencies(c.DataDir, c.Dependencies); err != nil {
 		logkit.Fatal("ensure dependencies failed", zap.Error(err))
@@ -55,8 +60,8 @@ func main() {
 	logkit.Info("read env flags", zap.Any("flag", *envflag.GetFlag()))
 
 	store.SetStorage(store.MustNewSqliteStorage(filepath.Join(c.DataDir, "cache", "cache.db")))
-	if err := translator.Init(); err != nil {
-		logkit.Error("init translater failed", zap.Error(err))
+	if err := setupTranslator(c); err != nil {
+		logkit.Error("setup translator failed", zap.Error(err)) //非关键路径
 	}
 	if err := initFace(filepath.Join(c.DataDir, "models")); err != nil {
 		logkit.Error("init face recognizer failed", zap.Error(err))
@@ -72,7 +77,6 @@ func main() {
 	logkit.Info("scrape from dir", zap.String("dir", c.ScanDir))
 	logkit.Info("save to dir", zap.String("dir", c.SaveDir))
 	logkit.Info("use data dir", zap.String("dir", c.DataDir))
-	logkit.Info("current switch options", zap.Any("options", envflag.GetFlag()))
 	logkit.Info("check current feature list")
 	logkit.Info("-- ffmpeg", zap.Bool("enable", ffmpeg.IsFFMpegEnabled()))
 	logkit.Info("-- ffprobe", zap.Bool("enable", ffmpeg.IsFFProbeEnabled()))
@@ -188,7 +192,7 @@ func ensureDependencies(datadir string, cdeps []config.Dependency) error {
 			Target: filepath.Join(datadir, item.RelPath),
 		})
 	}
-	return dependency.Resolve(client.NewClient(), deps)
+	return dependency.Resolve(client.DefaultClient(), deps)
 }
 
 func initFace(models string) error {
@@ -219,3 +223,28 @@ func initFace(models string) error {
 	face.SetFaceRec(face.NewGroup(impls))
 	return nil
 }
+
+func setupHTTPClient(c *config.Config) error {
+	opts := make([]client.Option, 0, 4)
+	if c.NetworkConfig.Timeout > 0 {
+		opts = append(opts, client.WithTimeout(time.Duration(c.NetworkConfig.Timeout)*time.Second))
+	}
+	if pxy := c.NetworkConfig.Proxy; len(pxy) > 0 {
+		opts = append(opts, client.WithProxy(pxy))
+	}
+	clientImpl, err := client.NewClient(opts...)
+	if err != nil {
+		return err
+	}
+	client.SetDefault(clientImpl)
+	return nil
+}
+
+func setupTranslator(c *config.Config) error {
+	t, err := googletranslator.New(googletranslator.WithProxyUrl(c.NetworkConfig.Proxy))
+	if err != nil {
+		return err
+	}
+	translator.SetTranslator(t)
+	return nil
+}

processor/handler/translate_handler.go

@@ -14,14 +14,14 @@ func (p *translaterHandler) Name() string {
 	return HTranslater
 }
 
-func (p *translaterHandler) translateSingle(name string, in string, item *model.SingleTranslateItem) error {
+func (p *translaterHandler) translateSingle(ctx context.Context, name string, in string, item *model.SingleTranslateItem) error {
 	if len(in) == 0 {
 		return nil
 	}
 	if !item.Enable {
 		return nil
 	}
-	res, err := translator.Translate(in, "auto", "zh")
+	res, err := translator.Translate(ctx, in, "auto", "zh")
 	if err != nil {
 		return fmt.Errorf("translate failed, name:%s, err:%w", name, err)
 	}
@@ -34,8 +34,8 @@ func (p *translaterHandler) Handle(ctx context.Context, fc *model.FileContext) e
 		return nil
 	}
 	var errs []error
-	errs = append(errs, p.translateSingle("title", fc.Meta.Title, &fc.Meta.ExtInfo.TranslateInfo.Title))
-	errs = append(errs, p.translateSingle("plot", fc.Meta.Plot, &fc.Meta.ExtInfo.TranslateInfo.Plot))
+	errs = append(errs, p.translateSingle(ctx, "title", fc.Meta.Title, &fc.Meta.ExtInfo.TranslateInfo.Title))
+	errs = append(errs, p.translateSingle(ctx, "plot", fc.Meta.Plot, &fc.Meta.ExtInfo.TranslateInfo.Plot))
 
 	for _, err := range errs {
 		if err != nil {

searcher/default_searcher.go

@@ -40,7 +40,7 @@ func MustNewDefaultSearcher(name string, plg api.IPlugin) ISearcher {
 }
 
 func defaultInvoker() api.HTTPInvoker {
-	basicClient := client.NewClient()
+	basicClient := client.DefaultClient()
 	return func(ctx context.Context, req *http.Request) (*http.Response, error) {
 		return basicClient.Do(req)
 	}

translator/googletranslator/config.go

@@ -0,0 +1,13 @@
+package googletranslator
+
+type config struct {
+	proxy string
+}
+
+type Option func(c *config)
+
+func WithProxyUrl(p string) Option {
+	return func(c *config) {
+		c.proxy = p
+	}
+}

translator/googletranslator/google_translator.go

@@ -0,0 +1,33 @@
+package googletranslator
+
+import (
+	"context"
+	"yamdc/translator"
+
+	gt "github.com/Conight/go-googletrans"
+)
+
+type googleTranslator struct {
+	t *gt.Translator
+}
+
+func New(opts ...Option) (translator.ITranslator, error) {
+	c := &config{}
+	for _, opt := range opts {
+		opt(c)
+	}
+	t := gt.New(gt.Config{
+		Proxy: c.proxy,
+	})
+	return &googleTranslator{
+		t: t,
+	}, nil
+}
+
+func (t *googleTranslator) Translate(_ context.Context, wording, src, dst string) (string, error) {
+	res, err := t.t.Translate(wording, src, dst)
+	if err != nil {
+		return "", err
+	}
+	return res.Text, nil
+}

translator/googletranslator/google_translator_test.go

@@ -0,0 +1,19 @@
+package googletranslator
+
+import (
+	"context"
+	"testing"
+	"yamdc/translator"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTranslate(t *testing.T) {
+	impl, err := New()
+	assert.NoError(t, err)
+	translator.SetTranslator(impl)
+	assert.NoError(t, err)
+	res, err := translator.Translate(context.Background(), "hello world", "auto", "zh")
+	assert.NoError(t, err)
+	t.Logf("result:%s", res)
+}

translator/translator.go

@@ -1,43 +1,23 @@
 package translator
 
 import (
-	translator "github.com/Conight/go-googletrans"
+	"context"
 )
 
-var defaultTranslator *Translator
-
-func Init() error {
-	inst, err := New()
-	if err != nil {
-		return err
-	}
-	defaultTranslator = inst
-	return nil
-}
-
-func IsTranslatorEnabled() bool {
-	return defaultTranslator != nil
+type ITranslator interface {
+	Translate(ctx context.Context, wording string, srclang, dstlang string) (string, error)
 }
 
-type Translator struct {
-	t *translator.Translator
+func SetTranslator(t ITranslator) {
+	defaultTranslator = t
 }
 
-func New() (*Translator, error) {
-	t := translator.New()
-	return &Translator{
-		t: t,
-	}, nil
-}
+var defaultTranslator ITranslator
 
-func (t *Translator) Translate(origin, src, dst string) (string, error) {
-	res, err := t.t.Translate(origin, src, dst)
-	if err != nil {
-		return "", err
-	}
-	return res.Text, nil
+func IsTranslatorEnabled() bool {
+	return defaultTranslator != nil
 }
 
-func Translate(origin, src, dst string) (string, error) {
-	return defaultTranslator.Translate(origin, src, dst)
+func Translate(ctx context.Context, origin, src, dst string) (string, error) {
+	return defaultTranslator.Translate(ctx, origin, src, dst)
 }