Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/go/apps/auction-item-stat-scraper/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/KubrickCode/loa-work/src/go/apps/auction-item-stat-scraper/converter"
"github.com/KubrickCode/loa-work/src/go/apps/auction-item-stat-scraper/scraper"
"github.com/KubrickCode/loa-work/src/go/libs/env"
"github.com/KubrickCode/loa-work/src/go/libs/loaApi/request"
"github.com/KubrickCode/loa-work/src/go/libs/loadb"
"github.com/KubrickCode/loa-work/src/go/libs/monitoring"
"github.com/KubrickCode/loa-work/src/go/libs/schedule"
Expand All @@ -27,7 +28,8 @@ func main() {
log.Fatal(err)
}

scraper := scraper.NewScraper(db)
client := request.NewClient()
scraper := scraper.NewScraper(client, db)
converter := converter.NewConverter(db)

combinedTask := func() error {
Expand Down
11 changes: 8 additions & 3 deletions src/go/apps/auction-item-stat-scraper/scraper/scraper.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,17 @@ import (
)

type Scraper struct {
client request.APIClient
db loadb.DB
rateLimiter *rate.Limiter
}

func NewScraper(db loadb.DB) *Scraper {
return &Scraper{db: db, rateLimiter: rate.NewLimiter(rate.Every(time.Second), 1)}
func NewScraper(client request.APIClient, db loadb.DB) *Scraper {
return &Scraper{
client: client,
db: db,
rateLimiter: rate.NewLimiter(rate.Every(time.Second), 1),
}
}

func (s *Scraper) Start() error {
Expand Down Expand Up @@ -67,7 +72,7 @@ func (s *Scraper) getItemStatsToCreate(category *models.AuctionItemCategory, ite
return nil, fmt.Errorf("rate limiter error: %w", err)
}

auctionItemListResp, err := request.GetAuctionItemList(&loaApi.GetAuctionItemListParams{
auctionItemListResp, err := s.client.GetAuctionItemList(&loaApi.GetAuctionItemListParams{
CategoryCode: category.Code,
ItemName: item.Name,
PageNo: 1,
Expand Down
4 changes: 3 additions & 1 deletion src/go/apps/market-item-category-scraper/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"log"

"github.com/KubrickCode/loa-work/src/go/apps/market-item-category-scraper/scraper"
"github.com/KubrickCode/loa-work/src/go/libs/loaApi/request"
"github.com/KubrickCode/loa-work/src/go/libs/loadb"
)

Expand All @@ -13,7 +14,8 @@ func main() {
log.Fatal(err)
}

scraper := scraper.NewScraper(db)
client := request.NewClient()
scraper := scraper.NewScraper(client, db)

err = scraper.Start()
if err != nil {
Expand Down
17 changes: 12 additions & 5 deletions src/go/apps/market-item-category-scraper/scraper/scraper.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package scraper

import (
"errors"
"fmt"
"log"

Expand All @@ -9,12 +10,18 @@ import (
"github.com/KubrickCode/loa-work/src/go/libs/loadb/models"
)

var ErrNoMarketItemCategories = errors.New("no market item categories found")

type Scraper struct {
db loadb.DB
client request.APIClient
db loadb.DB
}

func NewScraper(db loadb.DB) *Scraper {
return &Scraper{db: db}
func NewScraper(client request.APIClient, db loadb.DB) *Scraper {
return &Scraper{
client: client,
db: db,
}
}

func (s *Scraper) Start() error {
Expand All @@ -34,15 +41,15 @@ func (s *Scraper) Start() error {
}

func (s *Scraper) getCategories() ([]*models.MarketItemCategory, error) {
resp, err := request.GetCategoryList()
resp, err := s.client.GetCategoryList()
if err != nil {
return nil, err
}

categories := GetFlattenCategories(resp.Categories)

if len(categories) == 0 {
return nil, fmt.Errorf("no market item categories found")
return nil, ErrNoMarketItemCategories
}

return categories, nil
Expand Down
197 changes: 197 additions & 0 deletions src/go/apps/market-item-category-scraper/scraper/scraper_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,197 @@
package scraper

import (
"errors"
"testing"

"github.com/KubrickCode/loa-work/src/go/libs/loaApi"
)

type mockAPIClient struct {
getAuctionItemListFunc func(params *loaApi.GetAuctionItemListParams) (*loaApi.GetAuctionItemListResponse, error)
getCategoryListFunc func() (*loaApi.GetCategoryListResponse, error)
getMarketItemFunc func(params *loaApi.GetMarketItemParams) (*loaApi.GetMarketItemResponse, error)
getMarketItemListFunc func(params *loaApi.GetMarketItemListParams) (*loaApi.GetMarketItemListResponse, error)
}

func (m *mockAPIClient) GetAuctionItemList(params *loaApi.GetAuctionItemListParams) (*loaApi.GetAuctionItemListResponse, error) {
if m.getAuctionItemListFunc != nil {
return m.getAuctionItemListFunc(params)
}
return nil, errors.New("not implemented")
}

func (m *mockAPIClient) GetCategoryList() (*loaApi.GetCategoryListResponse, error) {
if m.getCategoryListFunc != nil {
return m.getCategoryListFunc()
}
return nil, errors.New("not implemented")
}

func (m *mockAPIClient) GetMarketItem(params *loaApi.GetMarketItemParams) (*loaApi.GetMarketItemResponse, error) {
if m.getMarketItemFunc != nil {
return m.getMarketItemFunc(params)
}
return nil, errors.New("not implemented")
}

func (m *mockAPIClient) GetMarketItemList(params *loaApi.GetMarketItemListParams) (*loaApi.GetMarketItemListResponse, error) {
if m.getMarketItemListFunc != nil {
return m.getMarketItemListFunc(params)
}
return nil, errors.New("not implemented")
}

func TestGetCategories_Success(t *testing.T) {
mockClient := &mockAPIClient{
getCategoryListFunc: func() (*loaApi.GetCategoryListResponse, error) {
return &loaApi.GetCategoryListResponse{
Categories: []loaApi.Category{
{
Code: 10000,
CodeName: "Test Category 1",
Subs: []loaApi.SubCategory{
{Code: 10001, CodeName: "Test Sub Category 1"},
},
},
{
Code: 20000,
CodeName: "Test Category 2",
Subs: []loaApi.SubCategory{},
},
},
}, nil
},
}

scraper := &Scraper{
client: mockClient,
db: nil,
}

categories, err := scraper.getCategories()
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}

if len(categories) != 3 {
t.Errorf("Expected 3 categories (2 parent + 1 sub), got %d", len(categories))
}

expectedCodes := []int{10000, 10001, 20000}
for i, category := range categories {
if category.Code != expectedCodes[i] {
t.Errorf("Expected category code %d, got %d", expectedCodes[i], category.Code)
}
}
}

func TestGetCategories_APIError(t *testing.T) {
expectedErr := errors.New("API connection failed")
mockClient := &mockAPIClient{
getCategoryListFunc: func() (*loaApi.GetCategoryListResponse, error) {
return nil, expectedErr
},
}

scraper := &Scraper{
client: mockClient,
db: nil,
}

_, err := scraper.getCategories()
if err == nil {
t.Fatal("Expected error, got nil")
}

if err != expectedErr {
t.Errorf("Expected error %v, got %v", expectedErr, err)
}
}

func TestGetCategories_EmptyResponse(t *testing.T) {
mockClient := &mockAPIClient{
getCategoryListFunc: func() (*loaApi.GetCategoryListResponse, error) {
return &loaApi.GetCategoryListResponse{
Categories: []loaApi.Category{},
}, nil
},
}

scraper := &Scraper{
client: mockClient,
db: nil,
}

_, err := scraper.getCategories()
if err == nil {
t.Fatal("Expected error for empty categories, got nil")
}

if !errors.Is(err, ErrNoMarketItemCategories) {
t.Errorf("Expected ErrNoMarketItemCategories, got %v", err)
}
}

func TestGetFlattenCategories(t *testing.T) {
categories := []loaApi.Category{
{
Code: 10000,
CodeName: "Parent 1",
Subs: []loaApi.SubCategory{
{Code: 10001, CodeName: "Child 1-1"},
{Code: 10002, CodeName: "Child 1-2"},
},
},
{
Code: 20000,
CodeName: "Parent 2",
Subs: []loaApi.SubCategory{
{Code: 20001, CodeName: "Child 2-1"},
},
},
}

flattened := GetFlattenCategories(categories)

if len(flattened) != 5 {
t.Errorf("Expected 5 flattened categories, got %d", len(flattened))
}

expectedCodes := []int{10000, 10001, 10002, 20000, 20001}
for i, category := range flattened {
if category.Code != expectedCodes[i] {
t.Errorf("At index %d: expected code %d, got %d", i, expectedCodes[i], category.Code)
}
}

for _, category := range flattened {
if category.ID != 0 {
t.Errorf("Expected ID to be 0 (new record), got %d", category.ID)
}
}
}

func TestGetFlattenCategories_EmptySubCategories(t *testing.T) {
categories := []loaApi.Category{
{
Code: 10000,
CodeName: "Category without subs",
Subs: []loaApi.SubCategory{},
},
}

flattened := GetFlattenCategories(categories)

if len(flattened) != 1 {
t.Errorf("Expected 1 category, got %d", len(flattened))
}

if flattened[0].Code != 10000 {
t.Errorf("Expected code 10000, got %d", flattened[0].Code)
}

if flattened[0].Name != "Category without subs" {
t.Errorf("Expected name 'Category without subs', got %s", flattened[0].Name)
}
}
4 changes: 3 additions & 1 deletion src/go/apps/market-item-scraper/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"log"

"github.com/KubrickCode/loa-work/src/go/apps/market-item-scraper/scraper"
"github.com/KubrickCode/loa-work/src/go/libs/loaApi/request"
"github.com/KubrickCode/loa-work/src/go/libs/loadb"
)

Expand All @@ -13,7 +14,8 @@ func main() {
log.Fatal(err)
}

scraper := scraper.NewScraper(db)
client := request.NewClient()
scraper := scraper.NewScraper(client, db)

err = scraper.Start()
if err != nil {
Expand Down
12 changes: 8 additions & 4 deletions src/go/apps/market-item-scraper/scraper/scraper.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,15 @@ import (
)

type Scraper struct {
db loadb.DB
client request.APIClient
db loadb.DB
}

func NewScraper(db loadb.DB) *Scraper {
return &Scraper{db: db}
func NewScraper(client request.APIClient, db loadb.DB) *Scraper {
return &Scraper{
client: client,
db: db,
}
}

func (s *Scraper) Start() error {
Expand Down Expand Up @@ -58,7 +62,7 @@ func (s *Scraper) getItemsToSave(categories []*models.MarketItemCategory) ([]*mo
pageNo := 1

for {
resp, err := request.GetMarketItemList(&loaApi.GetMarketItemListParams{
resp, err := s.client.GetMarketItemList(&loaApi.GetMarketItemListParams{
CategoryCode: category.Code,
PageNo: pageNo,
})
Expand Down
4 changes: 3 additions & 1 deletion src/go/apps/market-item-stat-scraper/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/KubrickCode/loa-work/src/go/apps/market-item-stat-scraper/converter"
"github.com/KubrickCode/loa-work/src/go/apps/market-item-stat-scraper/scraper"
"github.com/KubrickCode/loa-work/src/go/libs/env"
"github.com/KubrickCode/loa-work/src/go/libs/loaApi/request"
"github.com/KubrickCode/loa-work/src/go/libs/loadb"
"github.com/KubrickCode/loa-work/src/go/libs/monitoring"
"github.com/KubrickCode/loa-work/src/go/libs/schedule"
Expand All @@ -27,7 +28,8 @@ func main() {
log.Fatal(err)
}

scraper := scraper.NewScraper(db)
client := request.NewClient()
scraper := scraper.NewScraper(client, db)
converter := converter.NewConverter(db)

combinedTask := func() error {
Expand Down
Loading
Loading