Skip to content

Commit

Permalink
v4.6.0
Browse files Browse the repository at this point in the history 
- Added more tests
- Added ValueError handler for check_if_current_date_has_passed function
- Adjusted 'find_missing_dates' function
  • Loading branch information
@sakan811
sakan811 committed Jun 18, 2024
1 parent 641c9c8 commit 56ca951
Show file tree
Hide file tree
Showing 14 changed files with 823 additions and 497 deletions.
51 changes: 32 additions & 19 deletions japan_avg_hotel_price_finder/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,15 @@ def check_if_current_date_has_passed(year: int, month: int, day: int, timezone=N
else:
today = datetime.datetime.today()
today_for_check = today.strftime('%Y-%m-%d')
current_date_for_check = datetime.datetime(year, month, day).strftime('%Y-%m-%d')
if current_date_for_check < today_for_check:
return True
else:

try:
current_date_for_check = datetime.datetime(year, month, day).strftime('%Y-%m-%d')
if current_date_for_check < today_for_check:
return True
else:
return False
except ValueError:
logger.error("Invalid date. Returning False")
return False


Expand Down Expand Up @@ -67,7 +72,7 @@ def find_missing_dates_in_db(sqlite_db: str) -> list:
dates_in_db, end_date, start_date = find_dates_of_the_month_in_db(sqlite_db, days_in_month, month,
year)

missing_dates = find_missing_dates(dates_in_db, days_in_month, today, month, year)
missing_dates = find_missing_dates(dates_in_db, days_in_month, month, year)
logger.warning(f"Missing dates in {start_date} to {end_date}: {missing_dates}")
else:
date_obj = datetime.datetime.strptime(row[0], '%Y-%m')
Expand All @@ -80,7 +85,7 @@ def find_missing_dates_in_db(sqlite_db: str) -> list:
dates_in_db, end_date, start_date = find_dates_of_the_month_in_db(sqlite_db, days_in_month, month,
year)

missing_dates = find_missing_dates(dates_in_db, days_in_month, today, month, year)
missing_dates = find_missing_dates(dates_in_db, days_in_month, month, year)
logger.warning(f"Missing dates in {start_date} to {end_date}: {missing_dates}")

return missing_dates
Expand Down Expand Up @@ -183,32 +188,40 @@ def scrape_with_basic_scraper(db: str, date, to_sqlite: bool = False):
def find_missing_dates(
dates_in_db: set[str],
days_in_month: int,
today: datetime,
month: int,
year: int) -> list[str]:
year: int,
timezone=None) -> list[str]:
"""
Find missing dates of the given month.
Only check date from today onward.
:param dates_in_db: Dates of that month in the database of the current AsOf Date.
Date format: '%Y-%m-%d'.
:param days_in_month: Total days in the given month.
:param today: Today's date as a Datetime object.
:param month: Month.
:param year: Year.
:param timezone: Timezone, default is None, mostly for testing purpose.
:returns: Missing Dates as a list.
"""
logger.info(f"Find missing date of {calendar.month_name[month]} {year}.")
if timezone:
today = datetime.datetime.now(timezone)
else:
today = datetime.datetime.today()

dates_in_db_date_obj = [datetime.datetime.strptime(date, '%Y-%m-%d').date() for date in dates_in_db]
filtered_dates = [date for date in dates_in_db_date_obj if date >= today.date()]

today_date_obj = today.date()
missing_dates = []
for day in range(1, days_in_month + 1):
date_str = datetime.datetime(year, month, day).strftime('%Y-%m-%d')
if date_str not in dates_in_db:
if month == today.month:
# Handle the case when the month to scrape is the current month.
if day < today.day:
logger.warning(f"This day has passed. Skip {date_str}")
else:
missing_dates.append(date_str)
else:
missing_dates.append(date_str)
date_to_check = datetime.datetime(year, month, day)
date_to_check_str = date_to_check.strftime('%Y-%m-%d')
date_to_check_date_obj = date_to_check.date()
if date_to_check_date_obj < today_date_obj:
logger.warning(f"{date_to_check_str} has passed. Skip this date.")
else:
if date_to_check_date_obj not in filtered_dates:
missing_dates.append(date_to_check_str)
return missing_dates


Expand Down
233 changes: 233 additions & 0 deletions tests/test_graphql/test_check_info.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,233 @@
from unittest.mock import Mock

import pytest

from japan_avg_hotel_price_finder.graphql_scraper import check_info


def test_returns_correct_total_page_number_and_data_mapping():
# Given
response_mock = Mock()
response_mock.status_code = 200
response_mock.json.return_value = {
'data': {
'searchQueries': {
'search': {
'pagination': {'nbResultsTotal': 1},
'breadcrumbs': [{}, {}, {'name': 'Test City'}],
'flexibleDatesConfig': {
'dateRangeCalendar': {
'checkin': ['2023-01-01'],
'checkout': ['2023-01-02']
}
},
'searchMeta': {
'nbAdults': 2,
'nbChildren': 1,
'nbRooms': 1
},
'results': [{
'blocks': [{
'finalPrice': {'currency': 'USD'}
}]
}]
}
}
}
}
entered_city = "Test City"
entered_check_in = "2023-01-01"
entered_check_out = "2023-01-02"
entered_selected_currency = "USD"
entered_num_adult = 2
entered_num_children = 1
entered_num_room = 1

# When
result = check_info(
response_mock, entered_city, entered_check_in, entered_check_out,
entered_selected_currency, entered_num_adult, entered_num_children,
entered_num_room
)

# Then
assert result == (1, {
"city": "Test City",
"check_in": "2023-01-01",
"check_out": "2023-01-02",
"num_adult": 2,
"num_children": 1,
"num_room": 1,
"selected_currency": "USD"
})


def test_handles_response_with_missing_or_null_fields_gracefully():
# Given
response_mock = Mock()
response_mock.status_code = 200
response_mock.json.return_value = {
'data': {
'searchQueries': {
'search': {
'pagination': {'nbResultsTotal': 1},
'breadcrumbs': [{}, {}, {'name': None}],
'flexibleDatesConfig': {
'dateRangeCalendar': {
'checkin': [None],
'checkout': [None]
}
},
'searchMeta': {
'nbAdults': None,
'nbChildren': None,
'nbRooms': None
},
'results': [{
'blocks': [{
'finalPrice': {'currency': None}
}]
}]
}
}
}
}
entered_city = "Test City"
entered_check_in = "2023-01-01"
entered_check_out = "2023-01-02"
entered_selected_currency = "USD"
entered_num_adult = 2
entered_num_children = 1
entered_num_room = 1

# When
error_message = ''
try:
check_info(
response_mock, entered_city, entered_check_in, entered_check_out,
entered_selected_currency, entered_num_adult, entered_num_children,
entered_num_room
)
except SystemExit as e:
error_message = str(e)

# Then
assert error_message == "Error City not match: Test City != None"


def test_data_mapping_dictionary_keys():
# Given
response_mock = Mock()
response_mock.status_code = 200
response_mock.json.return_value = {
'data': {
'searchQueries': {
'search': {
'pagination': {'nbResultsTotal': 1},
'breadcrumbs': [{}, {}, {'name': 'Test City'}],
'flexibleDatesConfig': {
'dateRangeCalendar': {
'checkin': ['2023-01-01'],
'checkout': ['2023-01-02']
}
},
'searchMeta': {
'nbAdults': 2,
'nbChildren': 1,
'nbRooms': 1
},
'results': [{
'blocks': [{
'finalPrice': {'currency': 'USD'}
}]
}]
}
}
}
}
entered_city = "Test City"
entered_check_in = "2023-01-01"
entered_check_out = "2023-01-02"
entered_selected_currency = "USD"
entered_num_adult = 2
entered_num_children = 1
entered_num_room = 1

# When
result = check_info(
response_mock, entered_city, entered_check_in, entered_check_out,
entered_selected_currency, entered_num_adult, entered_num_children,
entered_num_room
)

# Then
assert result == (1, {
"city": "Test City",
"check_in": "2023-01-01",
"check_out": "2023-01-02",
"num_adult": 2,
"num_children": 1,
"num_room": 1,
"selected_currency": "USD"
})


def test_data_mapping_extraction():
# Given
response_mock = Mock()
response_mock.status_code = 200
response_mock.json.return_value = {
'data': {
'searchQueries': {
'search': {
'pagination': {'nbResultsTotal': 1},
'breadcrumbs': [{}, {}, {'name': 'Test City'}],
'flexibleDatesConfig': {
'dateRangeCalendar': {
'checkin': ['2023-01-01'],
'checkout': ['2023-01-02']
}
},
'searchMeta': {
'nbAdults': 2,
'nbChildren': 1,
'nbRooms': 1
},
'results': [{
'blocks': [{
'finalPrice': {'currency': 'USD'}
}]
}]
}
}
}
}
entered_city = "Test City"
entered_check_in = "2023-01-01"
entered_check_out = "2023-01-02"
entered_selected_currency = "USD"
entered_num_adult = 2
entered_num_children = 1
entered_num_room = 1

# When
result = check_info(
response_mock, entered_city, entered_check_in, entered_check_out,
entered_selected_currency, entered_num_adult, entered_num_children,
entered_num_room
)

# Then
assert result == (1, {
"city": "Test City",
"check_in": "2023-01-01",
"check_out": "2023-01-02",
"num_adult": 2,
"num_children": 1,
"num_room": 1,
"selected_currency": "USD"
})


if __name__ == '__main__':
pytest.main()
Loading

0 comments on commit 56ca951

Please sign in to comment.