feat(xignitefeeder): migrate API endpoints from QUICK to MINKABU (#622)

* feat(xignitefeeder): migrate API endpoints from QUICK to MINKABU

Author: dakimura

contrib/xignitefeeder/README.md

@@ -17,11 +17,9 @@ bgworkers:
       # exchange list
       exchanges:
         - XTKS # Tokyo Stock Exchange
-        - XJAS # Jasdaq
         #- XNGO # Nagoya Stock Exchange
         #- XSAP # Sapporo Stock Exchange
         #- XFKA # Fukuoka Stock Exchange
-        #- XTAM # Tokyo PRO Market
       # Xignite feeder also retrieves data of Index Symbols (ex. ＴＯＰＩＸ（東証１部株価指数）) every day.
       # To get target indices, index groups that the indices belong are necessary.
       # (cf. https://www.marketdata-cloud.quick-co.jp/Products/QUICKIndexHistorical/Overview/ListSymbols )
@@ -42,6 +40,17 @@ bgworkers:
       update_time: "22:00:00" # (UTC). = every day at 07:00:00 (JST)
       # XigniteFeeder writes data to "{identifier}/{timeframe}/TICK" TimeBucketKey
       timeframe: "1Sec"
+      # Base URL of the API
+      base_url: "https://stg-api.mk-smapi.jp/"
+      # Endpoint of each Xigntie API
+      endpoint:
+        equity_realtime_get_quotes: "MINKABUEquityRealTime.json/GetQuotes"
+        equity_realtime_list_symbols: "MINKABUEquityRealTime.json/ListSymbols"
+        equity_realtime_get_bars: "MINKABUEquityRealTime.json/GetBars"
+        equity_historical_get_quotes_range: "MINKABUEquityHistorical.json/GetQuotesRange"
+        index_realtime_get_bars: "MINKABUIndexRealTime.json/GetBars"
+        index_historical_list_symbols: "MINKABUIndexHistorical.json/ListSymbols"
+        index_historical_get_quotes_range: "MINKABUIndexHistorical.json/GetQuotesRange"
       # Auth token for Xignite API
       # This config can be manually overridden by "XIGNITE_FEEDER_API_TOKEN" environmental variable.
       token: "D***0"

contrib/xignitefeeder/api/client.go

@@ -15,36 +15,17 @@ import (
 	"github.com/alpacahq/marketstore/v4/utils/log"
 )
 
-const (
-	// XigniteBaseURL is a Base URL for Quick Xignite API
-	// (https://www.marketdata-cloud.quick-co.jp/Products/)
-	XigniteBaseURL = "https://api.marketdata-cloud.quick-co.jp"
-	// GetQuotesURL is the URL of Get Quotes endpoint
-	// (https://www.marketdata-cloud.quick-co.jp/Products/QUICKEquityRealTime/Overview/GetQuotes)
-	GetQuotesURL = XigniteBaseURL + "/QUICKEquityRealTime.json/GetQuotes"
-	// ListSymbolsURL is the URL of List symbols endpoint
-	// (https://www.marketdata-cloud.quick-co.jp/Products/QUICKEquityRealTime/Overview/ListSymbols)
-	ListSymbolsURL = XigniteBaseURL + "/QUICKEquityRealTime.json/ListSymbols"
-	// ListIndexSymbolsURL is the URL of List symbols endpoint
-	// (https://www.marketdata-cloud.quick-co.jp/Products/QUICKIndexHistorical/Overview/ListSymbols)
-	// /QUICKEquityRealTime.json/ListSymbols : list symbols for a exchange
-	// /QUICKIndexHistorical.json/ListSymbols : list index symbols for an index group (ex. TOPIX).
-	ListIndexSymbolsURL = XigniteBaseURL + "/QUICKIndexHistorical.json/ListSymbols"
-	// GetBarsURL is the URL of Get Bars endpoint
-	// (https://www.marketdata-cloud.quick-co.jp/Products/QUICKEquityRealTime/Overview/GetBars)
-	GetBarsURL = XigniteBaseURL + "/QUICKEquityRealTime.json/GetBars"
-	// GetIndexBarsURL is the URL of QuickIndexRealTime/GetBars endpoint
-	// (https://www.marketdata-cloud.quick-co.jp/Products/QUICKIndexRealTime/Overview/GetBars)
-	GetIndexBarsURL = XigniteBaseURL + "/QUICKIndexRealTime.json/GetBars"
-	// GetQuotesRangeURL is the URL of Get Quotes Range endpoint
-	// (https://www.marketdata-cloud.quick-co.jp/Products/QUICKEquityHistorical/Overview/GetQuotesRange)
-	GetQuotesRangeURL = XigniteBaseURL + "/QUICKEquityHistorical.json/GetQuotesRange"
-	// GetIndexQuotesRangeURL is the URL of Get Index Quotes Range endpoint
-	// (https://www.marketdata-cloud.quick-co.jp/Products/QUICKIndexHistorical/Overview/GetQuotesRange)
-	GetIndexQuotesRangeURL = XigniteBaseURL + "/QUICKIndexHistorical.json/GetQuotesRange"
-
-	SuccessOutcome = "Success"
-)
+const SuccessOutcome = "Success"
+
+type Endpoints struct {
+	EquityRealTimeGetQuotes        string
+	EquityRealTimeListSymbols      string
+	EquityRealTimeGetBars          string
+	EquityHistoricalGetQuotesRange string
+	IndexRealTimeGetBars           string
+	IndexHistoricalListSymbols     string
+	IndexHistoricalGetQuotesRange  string
+}
 
 // Client calls an endpoint and returns the parsed response.
 type Client interface {
@@ -62,17 +43,21 @@ type Client interface {
 }
 
 // NewDefaultAPIClient initializes Xignite API client with the specified API token and HTTP timeout[sec].
-func NewDefaultAPIClient(token string, timeoutSec int) *DefaultClient {
+func NewDefaultAPIClient(token string, timeoutSec int, baseURL string, endpoints Endpoints) *DefaultClient {
 	return &DefaultClient{
 		httpClient: &http.Client{Timeout: time.Duration(timeoutSec) * time.Second},
 		token:      token,
+		baseURL:    baseURL,
+		endpoints:  endpoints,
 	}
 }
 
 // DefaultClient is the Xignite API client with a default http client.
 type DefaultClient struct {
 	httpClient *http.Client
 	token      string
+	baseURL    string
+	endpoints  Endpoints
 }
 
 // GetRealTimeQuotes calls GetQuotes endpoint of Xignite API with specified identifiers
@@ -86,7 +71,8 @@ func (c *DefaultClient) GetRealTimeQuotes(ctx context.Context, identifiers []str
 		"Identifiers":    {strings.Join(identifiers, ",")},
 	}
 	req, err := http.NewRequestWithContext(ctx,
-		"POST", GetQuotesURL, strings.NewReader(form.Encode()))
+		"POST", fmt.Sprintf("%s%s", c.baseURL, c.endpoints.EquityRealTimeGetQuotes),
+		strings.NewReader(form.Encode()))
 	if err != nil {
 		return response, errors.Wrap(err, "failed to create an http request")
 	}
@@ -121,7 +107,8 @@ func (c *DefaultClient) GetRealTimeQuotes(ctx context.Context, identifiers []str
 // https://www.marketdata-cloud.quick-co.jp/Products/QUICKEquityRealTime/Overview/ListSymbols
 // exchange: XTKS, XNGO, XSAP, XFKA, XJAS, XTAM
 func (c *DefaultClient) ListSymbols(ctx context.Context, exchange string) (response ListSymbolsResponse, err error) {
-	apiURL := ListSymbolsURL + fmt.Sprintf("?_token=%s&Exchange=%s", c.token, exchange)
+	apiURL := fmt.Sprintf("%s%s?_token=%s&Exchange=%s",
+		c.baseURL, c.endpoints.EquityRealTimeListSymbols, c.token, exchange)
 	req, err := http.NewRequestWithContext(ctx, "GET", apiURL, http.NoBody)
 	if err != nil {
 		return response, errors.Wrap(err, "failed to create an http request")
@@ -145,7 +132,8 @@ func (c *DefaultClient) ListSymbols(ctx context.Context, exchange string) (respo
 // indexGroup: INDXJPX, IND_NIKKEI.
 func (c *DefaultClient) ListIndexSymbols(ctx context.Context, indexGroup string,
 ) (response ListIndexSymbolsResponse, err error) {
-	apiURL := ListIndexSymbolsURL + fmt.Sprintf("?_token=%s&GroupName=%s", c.token, indexGroup)
+	apiURL := fmt.Sprintf("%s%s?_token=%s&GroupName=%s",
+		c.baseURL, c.endpoints.IndexHistoricalListSymbols, c.token, indexGroup)
 	req, err := http.NewRequestWithContext(ctx, "GET", apiURL, http.NoBody)
 	if err != nil {
 		return response, errors.Wrap(err, "failed to create an http request")
@@ -182,7 +170,8 @@ func getBarsURLValues(token, identifier string, start, end time.Time) url.Values
 func (c *DefaultClient) GetRealTimeBars(ctx context.Context, identifier string, start, end time.Time,
 ) (response GetBarsResponse, err error) {
 	form := getBarsURLValues(c.token, identifier, start, end)
-	req, err := getBarsRequest(ctx, GetBarsURL, form.Encode())
+	req, err := getBarsRequest(ctx, fmt.Sprintf("%s%s", c.baseURL, c.endpoints.EquityRealTimeGetBars),
+		form.Encode())
 	if err != nil {
 		return response, err
 	}
@@ -202,7 +191,8 @@ func (c *DefaultClient) GetRealTimeBars(ctx context.Context, identifier string,
 func (c *DefaultClient) GetIndexBars(ctx context.Context, identifier string, start, end time.Time,
 ) (response GetIndexBarsResponse, err error) {
 	form := getBarsURLValues(c.token, identifier, start, end)
-	req, err := getBarsRequest(ctx, GetIndexBarsURL, form.Encode())
+	req, err := getBarsRequest(ctx, fmt.Sprintf("%s%s", c.baseURL, c.endpoints.IndexRealTimeGetBars),
+		form.Encode())
 	if err != nil {
 		return response, err
 	}
@@ -233,7 +223,8 @@ func getBarsRequest(ctx context.Context, url, body string) (*http.Request, error
 func (c *DefaultClient) GetQuotesRange(ctx context.Context, identifier string, startDate, endDate time.Time,
 ) (response GetQuotesRangeResponse, err error) {
 	formValues := quotesRangeFormValues(c.token, identifier, startDate, endDate)
-	req, err := quotesRangeReq(ctx, GetQuotesRangeURL, formValues.Encode())
+	req, err := quotesRangeReq(ctx, fmt.Sprintf("%s%s", c.baseURL, c.endpoints.EquityHistoricalGetQuotesRange),
+		formValues.Encode())
 	if err != nil {
 		return response, err
 	}
@@ -257,7 +248,8 @@ func (c *DefaultClient) GetQuotesRange(ctx context.Context, identifier string, s
 func (c *DefaultClient) GetIndexQuotesRange(ctx context.Context, identifier string, startDate, endDate time.Time,
 ) (response GetIndexQuotesRangeResponse, err error) {
 	formValues := quotesRangeFormValues(c.token, identifier, startDate, endDate)
-	req, err := quotesRangeReq(ctx, GetIndexQuotesRangeURL, formValues.Encode())
+	req, err := quotesRangeReq(ctx, fmt.Sprintf("%s%s", c.baseURL, c.endpoints.IndexHistoricalGetQuotesRange),
+		formValues.Encode())
 	if err != nil {
 		return response, err
 	}

contrib/xignitefeeder/configs/config.go

@@ -30,6 +30,8 @@ type DefaultConfig struct {
 	Timeframe           string    `json:"timeframe"`
 	APIToken            string    `json:"token"`
 	Timeout             int       `json:"timeout"`
+	BaseURL             string    `json:"base_url"`
+	Endpoint            Endpoint  `json:"endpoint"`
 	OpenTime            time.Time
 	CloseTime           time.Time
 	ClosedDaysOfTheWeek []time.Weekday
@@ -54,6 +56,16 @@ type DefaultConfig struct {
 	} `json:"recentBackfill"`
 }
 
+type Endpoint struct {
+	EquityRealTimeGetQuotes        string `json:"equity_realtime_get_quotes"`
+	EquityRealTimeListSymbols      string `json:"equity_realtime_list_symbols"`
+	EquityRealTimeGetBars          string `json:"equity_realtime_get_bars"`
+	EquityHistoricalGetQuotesRange string `json:"equity_historical_get_quotes_range"`
+	IndexRealTimeGetBars           string `json:"index_realtime_get_bars"`
+	IndexHistoricalListSymbols     string `json:"index_historical_list_symbols"`
+	IndexHistoricalGetQuotesRange  string `json:"index_historical_get_quotes_range"`
+}
+
 // NewConfig casts a map object to Config struct and returns it through json marshal->unmarshal.
 func NewConfig(config map[string]interface{}) (*DefaultConfig, error) {
 	data, err := json.Marshal(config)
@@ -74,10 +86,40 @@ func NewConfig(config map[string]interface{}) (*DefaultConfig, error) {
 	if err := validate(ret); err != nil {
 		return nil, fmt.Errorf("config validation error: %w", err)
 	}
+	ret.BaseURL, ret.Endpoint = endpointWithDefault(ret.BaseURL, ret.Endpoint)
 
 	return ret, nil
 }
 
+func endpointWithDefault(baseURL string, endpoint Endpoint) (string, Endpoint) {
+	if baseURL == "" {
+		baseURL = "https://api.marketdata-cloud.quick-co.jp/"
+	}
+	if endpoint.EquityRealTimeGetQuotes == "" {
+		endpoint.EquityRealTimeGetQuotes = "QUICKEquityRealTime.json/GetQuotes"
+	}
+	if endpoint.EquityRealTimeListSymbols == "" {
+		endpoint.EquityRealTimeListSymbols = "QUICKEquityRealTime.json/ListSymbols"
+	}
+	if endpoint.EquityRealTimeGetBars == "" {
+		endpoint.EquityRealTimeGetBars = "QUICKEquityRealTime.json/GetBars"
+	}
+	if endpoint.EquityHistoricalGetQuotesRange == "" {
+		endpoint.EquityHistoricalGetQuotesRange = "QUICKEquityHistorical.json/GetQuotesRange"
+	}
+	if endpoint.IndexRealTimeGetBars == "" {
+		endpoint.IndexRealTimeGetBars = "QUICKIndexRealTime.json/GetBars"
+	}
+	if endpoint.IndexHistoricalListSymbols == "" {
+		endpoint.IndexHistoricalListSymbols = "QUICKIndexHistorical.json/ListSymbols"
+	}
+	if endpoint.IndexHistoricalGetQuotesRange == "" {
+		endpoint.IndexHistoricalGetQuotesRange = "QUICKIndexHistorical.json/GetQuotesRange"
+	}
+
+	return baseURL, endpoint
+}
+
 func validate(cfg *DefaultConfig) error {
 	if len(cfg.Exchanges) < 1 && len(cfg.IndexGroups) < 1 {
 		return errors.New("must have 1 or more stock exchanges or index group in the config file")

contrib/xignitefeeder/configs/config_test.go

@@ -41,6 +41,17 @@ func TestNewConfig(t *testing.T) {
 				ClosedDays:          []time.Time{},
 				UpdateTime:          time.Date(0, 1, 1, 20, 0, 0, 0, time.UTC),
 				APIToken:            "ABCDEFGHIJKLMNOPQRSTUVWXYZ789012",
+				// default urls
+				BaseURL: "https://api.marketdata-cloud.quick-co.jp/",
+				Endpoint: configs.Endpoint{
+					EquityRealTimeGetQuotes:        "QUICKEquityRealTime.json/GetQuotes",
+					EquityRealTimeListSymbols:      "QUICKEquityRealTime.json/ListSymbols",
+					EquityRealTimeGetBars:          "QUICKEquityRealTime.json/GetBars",
+					EquityHistoricalGetQuotesRange: "QUICKEquityHistorical.json/GetQuotesRange",
+					IndexRealTimeGetBars:           "QUICKIndexRealTime.json/GetBars",
+					IndexHistoricalListSymbols:     "QUICKIndexHistorical.json/ListSymbols",
+					IndexHistoricalGetQuotesRange:  "QUICKIndexHistorical.json/GetQuotesRange",
+				},
 			},
 			wantErr: false,
 		},
@@ -63,6 +74,17 @@ func TestNewConfig(t *testing.T) {
 				ClosedDays:          []time.Time{},
 				UpdateTime:          time.Date(0, 1, 1, 12, 34, 56, 0, time.UTC),
 				APIToken:            "hellohellohellohellohellohello12",
+				// default urls
+				BaseURL: "https://api.marketdata-cloud.quick-co.jp/",
+				Endpoint: configs.Endpoint{
+					EquityRealTimeGetQuotes:        "QUICKEquityRealTime.json/GetQuotes",
+					EquityRealTimeListSymbols:      "QUICKEquityRealTime.json/ListSymbols",
+					EquityRealTimeGetBars:          "QUICKEquityRealTime.json/GetBars",
+					EquityHistoricalGetQuotesRange: "QUICKEquityHistorical.json/GetQuotesRange",
+					IndexRealTimeGetBars:           "QUICKIndexRealTime.json/GetBars",
+					IndexHistoricalListSymbols:     "QUICKIndexHistorical.json/ListSymbols",
+					IndexHistoricalGetQuotesRange:  "QUICKIndexHistorical.json/GetQuotesRange",
+				},
 			},
 			wantErr: false,
 		},

contrib/xignitefeeder/symbols/manager.go

@@ -87,6 +87,12 @@ func (m *ManagerImpl) UpdateSymbols(ctx context.Context) {
 		var identifiers []string
 		for _, securityDescription := range resp.ArrayOfSecurityDescription {
 			symbol := securityDescription.Symbol
+			if len(symbol) >= 5 {
+				// ignore 5-digit stock code
+				log.Info(fmt.Sprintf("Ignore 5-digit stock code: %s", symbol))
+				continue
+			}
+
 			if _, found := m.NotQuoteStockList[symbol]; found {
 				// ignore symbols in not_quote_stock_list
 				continue

contrib/xignitefeeder/symbols/manager_test.go

@@ -21,6 +21,7 @@ func (mac *MockListSymbolsAPIClient) ListSymbols(_ context.Context, exchange str
 			ArrayOfSecurityDescription: []api.SecurityDescription{
 				{Symbol: "1234"},
 				{Symbol: "5678"},
+				{Symbol: "90123"}, // 5-digit stock code should be ignored
 			},
 		}, nil
 	}

contrib/xignitefeeder/xignitefeeder.go

@@ -29,7 +29,15 @@ func NewBgWorker(conf map[string]interface{}) (bgworker.BgWorker, error) {
 	log.Info("loaded Xignite Feeder config...")
 
 	// init Xignite API client
-	apiClient := api.NewDefaultAPIClient(config.APIToken, config.Timeout)
+	apiClient := api.NewDefaultAPIClient(config.APIToken, config.Timeout, config.BaseURL, api.Endpoints{
+		EquityRealTimeGetQuotes:        config.Endpoint.EquityRealTimeGetQuotes,
+		EquityRealTimeListSymbols:      config.Endpoint.EquityRealTimeListSymbols,
+		EquityRealTimeGetBars:          config.Endpoint.EquityRealTimeGetBars,
+		EquityHistoricalGetQuotesRange: config.Endpoint.EquityHistoricalGetQuotesRange,
+		IndexRealTimeGetBars:           config.Endpoint.IndexRealTimeGetBars,
+		IndexHistoricalListSymbols:     config.Endpoint.IndexHistoricalListSymbols,
+		IndexHistoricalGetQuotesRange:  config.Endpoint.IndexHistoricalGetQuotesRange,
+	})
 
 	// init Market Time Checker
 	var timeChecker feed.MarketTimeChecker