diff --git a/go.work b/go.work index 48e0f3ae..4fcabaab 100644 --- a/go.work +++ b/go.work @@ -12,5 +12,6 @@ use ( ./src/go/libs/loa-api ./src/go/libs/loa-db ./src/go/libs/monitoring + ./src/go/libs/ratelimit ./src/go/libs/schedule ) diff --git a/go.work.sum b/go.work.sum index 4e1014f6..c8822336 100644 --- a/go.work.sum +++ b/go.work.sum @@ -53,6 +53,7 @@ golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= diff --git a/src/go/apps/auction-item-stat-scraper/scraper/scraper.go b/src/go/apps/auction-item-stat-scraper/scraper/scraper.go index 5700ca17..a9e692cb 100644 --- a/src/go/apps/auction-item-stat-scraper/scraper/scraper.go +++ b/src/go/apps/auction-item-stat-scraper/scraper/scraper.go @@ -10,20 +10,25 @@ import ( "github.com/KubrickCode/loa-work/src/go/libs/loaApi/request" "github.com/KubrickCode/loa-work/src/go/libs/loadb" "github.com/KubrickCode/loa-work/src/go/libs/loadb/models" - "golang.org/x/time/rate" + "github.com/KubrickCode/loa-work/src/go/libs/ratelimit" +) + +const ( + defaultRateLimitInterval = time.Second + defaultRateLimitBurst = 1 ) type Scraper struct { client request.APIClient db loadb.DB - rateLimiter *rate.Limiter + rateLimiter ratelimit.Limiter } func NewScraper(client request.APIClient, db loadb.DB) *Scraper { return &Scraper{ client: client, db: db, - rateLimiter: rate.NewLimiter(rate.Every(time.Second), 1), + rateLimiter: ratelimit.NewLimiterPerDuration(defaultRateLimitInterval, defaultRateLimitBurst), } } diff --git a/src/go/apps/market-item-category-scraper/scraper/scraper.go b/src/go/apps/market-item-category-scraper/scraper/scraper.go index 870428c6..d0097bc0 100644 --- a/src/go/apps/market-item-category-scraper/scraper/scraper.go +++ b/src/go/apps/market-item-category-scraper/scraper/scraper.go @@ -1,26 +1,36 @@ package scraper import ( + "context" "errors" "fmt" "log" + "time" "github.com/KubrickCode/loa-work/src/go/libs/loaApi/request" "github.com/KubrickCode/loa-work/src/go/libs/loadb" "github.com/KubrickCode/loa-work/src/go/libs/loadb/models" + "github.com/KubrickCode/loa-work/src/go/libs/ratelimit" +) + +const ( + defaultRateLimitInterval = time.Second + defaultRateLimitBurst = 1 ) var ErrNoMarketItemCategories = errors.New("no market item categories found") type Scraper struct { - client request.APIClient - db loadb.DB + client request.APIClient + db loadb.DB + rateLimiter ratelimit.Limiter } func NewScraper(client request.APIClient, db loadb.DB) *Scraper { return &Scraper{ - client: client, - db: db, + client: client, + db: db, + rateLimiter: ratelimit.NewLimiterPerDuration(defaultRateLimitInterval, defaultRateLimitBurst), } } @@ -41,6 +51,10 @@ func (s *Scraper) Start() error { } func (s *Scraper) getCategories() ([]*models.MarketItemCategory, error) { + if err := s.rateLimiter.Wait(context.Background()); err != nil { + return nil, fmt.Errorf("rate limiter error: %w", err) + } + resp, err := s.client.GetCategoryList() if err != nil { return nil, err diff --git a/src/go/apps/market-item-category-scraper/scraper/scraper_test.go b/src/go/apps/market-item-category-scraper/scraper/scraper_test.go index 5fe99e80..88dd59b1 100644 --- a/src/go/apps/market-item-category-scraper/scraper/scraper_test.go +++ b/src/go/apps/market-item-category-scraper/scraper/scraper_test.go @@ -1,12 +1,21 @@ package scraper import ( + "context" "errors" "testing" "github.com/KubrickCode/loa-work/src/go/libs/loaApi" + "github.com/KubrickCode/loa-work/src/go/libs/ratelimit" ) +// noopLimiter implements ratelimit.Limiter with no delay +type noopLimiter struct{} + +func (l *noopLimiter) Wait(ctx context.Context) error { + return nil +} + type mockAPIClient struct { getAuctionItemListFunc func(params *loaApi.GetAuctionItemListParams) (*loaApi.GetAuctionItemListResponse, error) getCategoryListFunc func() (*loaApi.GetCategoryListResponse, error) @@ -65,8 +74,9 @@ func TestGetCategories_Success(t *testing.T) { } scraper := &Scraper{ - client: mockClient, - db: nil, + client: mockClient, + db: nil, + rateLimiter: &noopLimiter{}, } categories, err := scraper.getCategories() @@ -95,17 +105,14 @@ func TestGetCategories_APIError(t *testing.T) { } scraper := &Scraper{ - client: mockClient, - db: nil, + client: mockClient, + db: nil, + rateLimiter: &noopLimiter{}, } _, err := scraper.getCategories() - if err == nil { - t.Fatal("Expected error, got nil") - } - - if err != expectedErr { - t.Errorf("Expected error %v, got %v", expectedErr, err) + if !errors.Is(err, expectedErr) { + t.Fatalf("Expected error %v, got %v", expectedErr, err) } } @@ -119,8 +126,9 @@ func TestGetCategories_EmptyResponse(t *testing.T) { } scraper := &Scraper{ - client: mockClient, - db: nil, + client: mockClient, + db: nil, + rateLimiter: &noopLimiter{}, } _, err := scraper.getCategories() @@ -195,3 +203,40 @@ func TestGetFlattenCategories_EmptySubCategories(t *testing.T) { t.Errorf("Expected name 'Category without subs', got %s", flattened[0].Name) } } + +func TestNewScraper_RateLimiterInitialization(t *testing.T) { + mockClient := &mockAPIClient{} + scraper := NewScraper(mockClient, nil) + + if scraper.rateLimiter == nil { + t.Fatal("Expected rateLimiter to be initialized, got nil") + } +} + +func TestRateLimiter_InterfaceCompliance(t *testing.T) { + mockClient := &mockAPIClient{ + getCategoryListFunc: func() (*loaApi.GetCategoryListResponse, error) { + return &loaApi.GetCategoryListResponse{ + Categories: []loaApi.Category{ + {Code: 10000, CodeName: "Test"}, + }, + }, nil + }, + } + + // Test with real limiter + scraper := &Scraper{ + client: mockClient, + db: nil, + rateLimiter: ratelimit.NewLimiterPerDuration(0, 1), // instant for test + } + + categories, err := scraper.getCategories() + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if len(categories) != 1 { + t.Errorf("Expected 1 category, got %d", len(categories)) + } +} diff --git a/src/go/apps/market-item-scraper/scraper/scraper.go b/src/go/apps/market-item-scraper/scraper/scraper.go index ed99e134..a85fd8f6 100644 --- a/src/go/apps/market-item-scraper/scraper/scraper.go +++ b/src/go/apps/market-item-scraper/scraper/scraper.go @@ -1,24 +1,34 @@ package scraper import ( + "context" "fmt" "log" + "time" "github.com/KubrickCode/loa-work/src/go/libs/loaApi" "github.com/KubrickCode/loa-work/src/go/libs/loaApi/request" "github.com/KubrickCode/loa-work/src/go/libs/loadb" "github.com/KubrickCode/loa-work/src/go/libs/loadb/models" + "github.com/KubrickCode/loa-work/src/go/libs/ratelimit" +) + +const ( + defaultRateLimitInterval = time.Second + defaultRateLimitBurst = 1 ) type Scraper struct { - client request.APIClient - db loadb.DB + client request.APIClient + db loadb.DB + rateLimiter ratelimit.Limiter } func NewScraper(client request.APIClient, db loadb.DB) *Scraper { return &Scraper{ - client: client, - db: db, + client: client, + db: db, + rateLimiter: ratelimit.NewLimiterPerDuration(defaultRateLimitInterval, defaultRateLimitBurst), } } @@ -62,6 +72,10 @@ func (s *Scraper) getItemsToSave(categories []*models.MarketItemCategory) ([]*mo pageNo := 1 for { + if err := s.rateLimiter.Wait(context.Background()); err != nil { + return nil, fmt.Errorf("rate limiter error: %w", err) + } + resp, err := s.client.GetMarketItemList(&loaApi.GetMarketItemListParams{ CategoryCode: category.Code, PageNo: pageNo, diff --git a/src/go/apps/market-item-scraper/scraper/scraper_test.go b/src/go/apps/market-item-scraper/scraper/scraper_test.go new file mode 100644 index 00000000..3f3470e3 --- /dev/null +++ b/src/go/apps/market-item-scraper/scraper/scraper_test.go @@ -0,0 +1,506 @@ +package scraper + +import ( + "context" + "database/sql" + "errors" + "testing" + "time" + + "github.com/KubrickCode/loa-work/src/go/libs/loaApi" + "github.com/KubrickCode/loa-work/src/go/libs/loadb" + "github.com/KubrickCode/loa-work/src/go/libs/loadb/models" + "github.com/KubrickCode/loa-work/src/go/libs/ratelimit" +) + +// noopLimiter implements ratelimit.Limiter with no delay +type noopLimiter struct{} + +func (l *noopLimiter) Wait(ctx context.Context) error { + return nil +} + +type mockAPIClient struct { + getAuctionItemListFunc func(params *loaApi.GetAuctionItemListParams) (*loaApi.GetAuctionItemListResponse, error) + getCategoryListFunc func() (*loaApi.GetCategoryListResponse, error) + getMarketItemFunc func(params *loaApi.GetMarketItemParams) (*loaApi.GetMarketItemResponse, error) + getMarketItemListFunc func(params *loaApi.GetMarketItemListParams) (*loaApi.GetMarketItemListResponse, error) +} + +func (m *mockAPIClient) GetAuctionItemList(params *loaApi.GetAuctionItemListParams) (*loaApi.GetAuctionItemListResponse, error) { + if m.getAuctionItemListFunc != nil { + return m.getAuctionItemListFunc(params) + } + return nil, errors.New("not implemented") +} + +func (m *mockAPIClient) GetCategoryList() (*loaApi.GetCategoryListResponse, error) { + if m.getCategoryListFunc != nil { + return m.getCategoryListFunc() + } + return nil, errors.New("not implemented") +} + +func (m *mockAPIClient) GetMarketItem(params *loaApi.GetMarketItemParams) (*loaApi.GetMarketItemResponse, error) { + if m.getMarketItemFunc != nil { + return m.getMarketItemFunc(params) + } + return nil, errors.New("not implemented") +} + +func (m *mockAPIClient) GetMarketItemList(params *loaApi.GetMarketItemListParams) (*loaApi.GetMarketItemListResponse, error) { + if m.getMarketItemListFunc != nil { + return m.getMarketItemListFunc(params) + } + return nil, errors.New("not implemented") +} + +type mockMarketItemCategoryRepository struct { + findByIDFunc func(id int) (*models.MarketItemCategory, error) + findItemScraperEnabledAllFunc func() ([]*models.MarketItemCategory, error) + upsertManyFunc func(categories []*models.MarketItemCategory) error +} + +func (m *mockMarketItemCategoryRepository) FindByID(id int) (*models.MarketItemCategory, error) { + if m.findByIDFunc != nil { + return m.findByIDFunc(id) + } + return nil, errors.New("not implemented") +} + +func (m *mockMarketItemCategoryRepository) FindItemScraperEnabledAll() ([]*models.MarketItemCategory, error) { + if m.findItemScraperEnabledAllFunc != nil { + return m.findItemScraperEnabledAllFunc() + } + return nil, errors.New("not implemented") +} + +func (m *mockMarketItemCategoryRepository) UpsertMany(categories []*models.MarketItemCategory) error { + if m.upsertManyFunc != nil { + return m.upsertManyFunc(categories) + } + return errors.New("not implemented") +} + +type mockMarketItemRepository struct { + findAllFunc func() ([]*models.MarketItem, error) + findByNameFunc func(name string) (*models.MarketItem, error) + findStatScraperEnabledAllFunc func() ([]*models.MarketItem, error) + updateStatFunc func(item *models.MarketItem) error + upsertManyFunc func(items []*models.MarketItem) error +} + +func (m *mockMarketItemRepository) FindAll() ([]*models.MarketItem, error) { + if m.findAllFunc != nil { + return m.findAllFunc() + } + return nil, errors.New("not implemented") +} + +func (m *mockMarketItemRepository) FindByName(name string) (*models.MarketItem, error) { + if m.findByNameFunc != nil { + return m.findByNameFunc(name) + } + return nil, errors.New("not implemented") +} + +func (m *mockMarketItemRepository) FindStatScraperEnabledAll() ([]*models.MarketItem, error) { + if m.findStatScraperEnabledAllFunc != nil { + return m.findStatScraperEnabledAllFunc() + } + return nil, errors.New("not implemented") +} + +func (m *mockMarketItemRepository) UpdateStat(item *models.MarketItem) error { + if m.updateStatFunc != nil { + return m.updateStatFunc(item) + } + return errors.New("not implemented") +} + +func (m *mockMarketItemRepository) UpsertMany(items []*models.MarketItem) error { + if m.upsertManyFunc != nil { + return m.upsertManyFunc(items) + } + return errors.New("not implemented") +} + +type mockDB struct { + marketItemCategoryRepo loadb.MarketItemCategoryRepository + marketItemRepo loadb.MarketItemRepository +} + +func (m *mockDB) AuctionItem() loadb.AuctionItemRepository { + return nil +} + +func (m *mockDB) AuctionItemCategory() loadb.AuctionItemCategoryRepository { + return nil +} + +func (m *mockDB) AuctionItemStat() loadb.AuctionItemStatRepository { + return nil +} + +func (m *mockDB) DB() *sql.DB { + return nil +} + +func (m *mockDB) Item() loadb.ItemRepository { + return nil +} + +func (m *mockDB) MarketItem() loadb.MarketItemRepository { + return m.marketItemRepo +} + +func (m *mockDB) MarketItemCategory() loadb.MarketItemCategoryRepository { + return m.marketItemCategoryRepo +} + +func (m *mockDB) MarketItemStat() loadb.MarketItemStatRepository { + return nil +} + +func (m *mockDB) WithTransaction(action func(tx loadb.DB) error) error { + return errors.New("not implemented") +} + +func TestNewScraper_RateLimiterInitialization(t *testing.T) { + mockClient := &mockAPIClient{} + mockDB := &mockDB{} + scraper := NewScraper(mockClient, mockDB) + + if scraper.rateLimiter == nil { + t.Fatal("Expected rateLimiter to be initialized, got nil") + } +} + +func TestRateLimiter_MultipleAPICalls(t *testing.T) { + callCount := 0 + mockClient := &mockAPIClient{ + getMarketItemListFunc: func(params *loaApi.GetMarketItemListParams) (*loaApi.GetMarketItemListResponse, error) { + callCount++ + return &loaApi.GetMarketItemListResponse{ + Items: []loaApi.MarketItem{ + { + BundleCount: 10, + Grade: "전설", + Icon: "http://example.com/icon.png", + ID: callCount, + Name: "Test Item", + }, + }, + PageSize: 10, + }, nil + }, + } + + mockDB := &mockDB{ + marketItemCategoryRepo: &mockMarketItemCategoryRepository{ + findItemScraperEnabledAllFunc: func() ([]*models.MarketItemCategory, error) { + return []*models.MarketItemCategory{ + {ID: 1, Code: 10000, Name: "Category 1"}, + {ID: 2, Code: 20000, Name: "Category 2"}, + {ID: 3, Code: 30000, Name: "Category 3"}, + }, nil + }, + }, + marketItemRepo: &mockMarketItemRepository{ + upsertManyFunc: func(items []*models.MarketItem) error { + return nil + }, + }, + } + + scraper := &Scraper{ + client: mockClient, + db: mockDB, + rateLimiter: ratelimit.NewLimiter(10, 1), // 10 req/sec for faster test + } + + start := time.Now() + _, err := scraper.getItemsToSave([]*models.MarketItemCategory{ + {ID: 1, Code: 10000, Name: "Category 1"}, + {ID: 2, Code: 20000, Name: "Category 2"}, + {ID: 3, Code: 30000, Name: "Category 3"}, + }) + elapsed := time.Since(start) + + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if callCount != 3 { + t.Errorf("Expected 3 API calls, got %d", callCount) + } + + expectedMinDuration := 200 * time.Millisecond + if elapsed < expectedMinDuration { + t.Errorf("Expected at least %v for 3 calls with 10 req/sec rate limiter, got %v", expectedMinDuration, elapsed) + } +} + +func TestGetCategoriesToScrape_Success(t *testing.T) { + expectedCategories := []*models.MarketItemCategory{ + {ID: 1, Code: 10000, Name: "Category 1"}, + {ID: 2, Code: 20000, Name: "Category 2"}, + } + + mockDB := &mockDB{ + marketItemCategoryRepo: &mockMarketItemCategoryRepository{ + findItemScraperEnabledAllFunc: func() ([]*models.MarketItemCategory, error) { + return expectedCategories, nil + }, + }, + } + + scraper := &Scraper{ + client: &mockAPIClient{}, + db: mockDB, + rateLimiter: &noopLimiter{}, + } + + categories, err := scraper.getCategoriesToScrape() + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if len(categories) != 2 { + t.Errorf("Expected 2 categories, got %d", len(categories)) + } + + if categories[0].Code != 10000 { + t.Errorf("Expected first category code 10000, got %d", categories[0].Code) + } +} + +func TestGetCategoriesToScrape_EmptyResult(t *testing.T) { + mockDB := &mockDB{ + marketItemCategoryRepo: &mockMarketItemCategoryRepository{ + findItemScraperEnabledAllFunc: func() ([]*models.MarketItemCategory, error) { + return []*models.MarketItemCategory{}, nil + }, + }, + } + + scraper := &Scraper{ + client: &mockAPIClient{}, + db: mockDB, + rateLimiter: &noopLimiter{}, + } + + _, err := scraper.getCategoriesToScrape() + if err == nil { + t.Fatal("Expected error for empty categories, got nil") + } +} + +func TestGetCategoriesToScrape_DBError(t *testing.T) { + expectedErr := errors.New("database connection failed") + mockDB := &mockDB{ + marketItemCategoryRepo: &mockMarketItemCategoryRepository{ + findItemScraperEnabledAllFunc: func() ([]*models.MarketItemCategory, error) { + return nil, expectedErr + }, + }, + } + + scraper := &Scraper{ + client: &mockAPIClient{}, + db: mockDB, + rateLimiter: &noopLimiter{}, + } + + _, err := scraper.getCategoriesToScrape() + if err == nil { + t.Fatal("Expected error, got nil") + } + + if err != expectedErr { + t.Errorf("Expected error %v, got %v", expectedErr, err) + } +} + +func TestGetItemsToSave_Success(t *testing.T) { + mockClient := &mockAPIClient{ + getMarketItemListFunc: func(params *loaApi.GetMarketItemListParams) (*loaApi.GetMarketItemListResponse, error) { + return &loaApi.GetMarketItemListResponse{ + Items: []loaApi.MarketItem{ + { + BundleCount: 10, + Grade: "전설", + Icon: "http://example.com/icon1.png", + ID: 123, + Name: "Item 1", + }, + { + BundleCount: 5, + Grade: "영웅", + Icon: "http://example.com/icon2.png", + ID: 456, + Name: "Item 2", + }, + }, + PageSize: 10, + }, nil + }, + } + + scraper := &Scraper{ + client: mockClient, + db: &mockDB{}, + rateLimiter: &noopLimiter{}, + } + + categories := []*models.MarketItemCategory{ + {ID: 1, Code: 10000, Name: "Category 1"}, + } + + items, err := scraper.getItemsToSave(categories) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if len(items) != 2 { + t.Errorf("Expected 2 items, got %d", len(items)) + } + + if items[0].Name != "Item 1" { + t.Errorf("Expected first item name 'Item 1', got %s", items[0].Name) + } + + if items[0].RefID != 123 { + t.Errorf("Expected first item RefID 123, got %d", items[0].RefID) + } + + if items[1].Grade != "영웅" { + t.Errorf("Expected second item grade '영웅', got %s", items[1].Grade) + } +} + +func TestGetItemsToSave_DuplicateItems(t *testing.T) { + callCount := 0 + mockClient := &mockAPIClient{ + getMarketItemListFunc: func(params *loaApi.GetMarketItemListParams) (*loaApi.GetMarketItemListResponse, error) { + callCount++ + if callCount == 1 { + return &loaApi.GetMarketItemListResponse{ + Items: []loaApi.MarketItem{ + { + BundleCount: 10, + Grade: "전설", + Icon: "http://example.com/icon1.png", + ID: 123, + Name: "Item 1", + }, + }, + PageSize: 10, + }, nil + } + return &loaApi.GetMarketItemListResponse{ + Items: []loaApi.MarketItem{ + { + BundleCount: 10, + Grade: "전설", + Icon: "http://example.com/icon1.png", + ID: 789, + Name: "Item 1", + }, + }, + PageSize: 10, + }, nil + }, + } + + scraper := &Scraper{ + client: mockClient, + db: &mockDB{}, + rateLimiter: &noopLimiter{}, + } + + categories := []*models.MarketItemCategory{ + {ID: 1, Code: 10000, Name: "Category 1"}, + {ID: 2, Code: 20000, Name: "Category 2"}, + } + + items, err := scraper.getItemsToSave(categories) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if len(items) != 1 { + t.Errorf("Expected 1 item (duplicate filtered), got %d", len(items)) + } +} + +func TestGetItemsToSave_APIError(t *testing.T) { + expectedErr := errors.New("API connection failed") + mockClient := &mockAPIClient{ + getMarketItemListFunc: func(params *loaApi.GetMarketItemListParams) (*loaApi.GetMarketItemListResponse, error) { + return nil, expectedErr + }, + } + + scraper := &Scraper{ + client: mockClient, + db: &mockDB{}, + rateLimiter: &noopLimiter{}, + } + + categories := []*models.MarketItemCategory{ + {ID: 1, Code: 10000, Name: "Category 1"}, + } + + _, err := scraper.getItemsToSave(categories) + if err == nil { + t.Fatal("Expected error, got nil") + } +} + +func TestGetItemsToSave_Pagination(t *testing.T) { + pageNo := 0 + mockClient := &mockAPIClient{ + getMarketItemListFunc: func(params *loaApi.GetMarketItemListParams) (*loaApi.GetMarketItemListResponse, error) { + pageNo++ + if pageNo == 1 { + return &loaApi.GetMarketItemListResponse{ + Items: []loaApi.MarketItem{ + {ID: 1, Name: "Item 1", Grade: "전설", BundleCount: 10}, + {ID: 2, Name: "Item 2", Grade: "영웅", BundleCount: 5}, + }, + PageSize: 2, + }, nil + } + return &loaApi.GetMarketItemListResponse{ + Items: []loaApi.MarketItem{ + {ID: 3, Name: "Item 3", Grade: "희귀", BundleCount: 3}, + }, + PageSize: 2, + }, nil + }, + } + + scraper := &Scraper{ + client: mockClient, + db: &mockDB{}, + rateLimiter: &noopLimiter{}, + } + + categories := []*models.MarketItemCategory{ + {ID: 1, Code: 10000, Name: "Category 1"}, + } + + items, err := scraper.getItemsToSave(categories) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if len(items) != 3 { + t.Errorf("Expected 3 items from pagination, got %d", len(items)) + } + + if pageNo != 2 { + t.Errorf("Expected 2 pages to be fetched, got %d", pageNo) + } +} diff --git a/src/go/apps/market-item-stat-scraper/scraper/scraper.go b/src/go/apps/market-item-stat-scraper/scraper/scraper.go index ee19a477..4210a659 100644 --- a/src/go/apps/market-item-stat-scraper/scraper/scraper.go +++ b/src/go/apps/market-item-stat-scraper/scraper/scraper.go @@ -10,21 +10,26 @@ import ( "github.com/KubrickCode/loa-work/src/go/libs/loaApi/request" "github.com/KubrickCode/loa-work/src/go/libs/loadb" "github.com/KubrickCode/loa-work/src/go/libs/loadb/models" + "github.com/KubrickCode/loa-work/src/go/libs/ratelimit" "github.com/ericlagergren/decimal" - "golang.org/x/time/rate" +) + +const ( + defaultRateLimitInterval = time.Second + defaultRateLimitBurst = 1 ) type Scraper struct { client request.APIClient db loadb.DB - rateLimiter *rate.Limiter + rateLimiter ratelimit.Limiter } func NewScraper(client request.APIClient, db loadb.DB) *Scraper { return &Scraper{ client: client, db: db, - rateLimiter: rate.NewLimiter(rate.Every(time.Second), 1), + rateLimiter: ratelimit.NewLimiterPerDuration(defaultRateLimitInterval, defaultRateLimitBurst), } } diff --git a/src/go/libs/ratelimit/go.mod b/src/go/libs/ratelimit/go.mod new file mode 100644 index 00000000..16c1eb02 --- /dev/null +++ b/src/go/libs/ratelimit/go.mod @@ -0,0 +1,5 @@ +module github.com/KubrickCode/loa-work/src/go/libs/ratelimit + +go 1.23 + +require golang.org/x/time v0.9.0 diff --git a/src/go/libs/ratelimit/go.sum b/src/go/libs/ratelimit/go.sum new file mode 100644 index 00000000..aa0ec267 --- /dev/null +++ b/src/go/libs/ratelimit/go.sum @@ -0,0 +1,2 @@ +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= diff --git a/src/go/libs/ratelimit/limiter.go b/src/go/libs/ratelimit/limiter.go new file mode 100644 index 00000000..e6062397 --- /dev/null +++ b/src/go/libs/ratelimit/limiter.go @@ -0,0 +1,32 @@ +package ratelimit + +import ( + "context" + "time" + + "golang.org/x/time/rate" +) + +type Limiter interface { + Wait(ctx context.Context) error +} + +type limiter struct { + rl *rate.Limiter +} + +func NewLimiter(rps float64, burst int) Limiter { + return &limiter{ + rl: rate.NewLimiter(rate.Limit(rps), burst), + } +} + +func NewLimiterPerDuration(d time.Duration, burst int) Limiter { + return &limiter{ + rl: rate.NewLimiter(rate.Every(d), burst), + } +} + +func (l *limiter) Wait(ctx context.Context) error { + return l.rl.Wait(ctx) +} diff --git a/src/go/libs/ratelimit/limiter_test.go b/src/go/libs/ratelimit/limiter_test.go new file mode 100644 index 00000000..52132b57 --- /dev/null +++ b/src/go/libs/ratelimit/limiter_test.go @@ -0,0 +1,59 @@ +package ratelimit + +import ( + "context" + "testing" + "time" +) + +func TestNewLimiter(t *testing.T) { + limiter := NewLimiter(10, 1) + if limiter == nil { + t.Fatal("Expected limiter to be created, got nil") + } +} + +func TestNewLimiterPerDuration(t *testing.T) { + limiter := NewLimiterPerDuration(time.Second, 1) + if limiter == nil { + t.Fatal("Expected limiter to be created, got nil") + } +} + +func TestLimiter_Wait_RateLimiting(t *testing.T) { + limiter := NewLimiter(10, 1) // 10 req/sec + + start := time.Now() + + for i := 0; i < 3; i++ { + if err := limiter.Wait(context.Background()); err != nil { + t.Fatalf("Wait() returned unexpected error: %v", err) + } + } + + elapsed := time.Since(start) + + // 3 calls at 10 req/sec should take at least 200ms (first is immediate, then 2 waits) + expectedMin := 200 * time.Millisecond + if elapsed < expectedMin { + t.Errorf("Expected at least %v for 3 calls, got %v", expectedMin, elapsed) + } +} + +func TestLimiter_Wait_ContextCancellation(t *testing.T) { + limiter := NewLimiter(0.1, 1) // Very slow: 1 req per 10 sec + + // First call is immediate (uses burst) + if err := limiter.Wait(context.Background()); err != nil { + t.Fatalf("First Wait() returned unexpected error: %v", err) + } + + // Second call should wait, but we cancel context + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + err := limiter.Wait(ctx) + if err == nil { + t.Error("Expected context cancellation error, got nil") + } +}