diff --git a/.gitignore b/.gitignore index c058249..95b132f 100644 --- a/.gitignore +++ b/.gitignore @@ -170,3 +170,5 @@ cython_debug/ /prototype.py /prototype.py /avg_japan_hotel_price_test.db +/load_to_sqlite.py +/*.json \ No newline at end of file diff --git a/japan_avg_hotel_price_finder/migrate_to_sqlite.py b/japan_avg_hotel_price_finder/migrate_to_sqlite.py index d5cd486..da0af58 100644 --- a/japan_avg_hotel_price_finder/migrate_to_sqlite.py +++ b/japan_avg_hotel_price_finder/migrate_to_sqlite.py @@ -46,24 +46,34 @@ def migrate_data_to_sqlite(df_filtered: pd.DataFrame) -> None: con.execute(query) - data_type = { - 'Hotel': 'text primary key', - 'Price': 'real', - 'Review': 'real', - 'Price/Review': 'real', - 'City': 'text', - 'Date': 'text', - 'AsOf': 'text' - } + hotel_price_dtype: dict = get_hotel_price_dtype() # Save the DataFrame to a table named 'HotelPrice' - df_filtered.to_sql('HotelPrice', con=con, if_exists='append', index=False, dtype=data_type) + df_filtered.to_sql('HotelPrice', con=con, if_exists='append', index=False, dtype=hotel_price_dtype) logger.info(f'Data has been saved to {db}') create_average_room_price_by_date_view(db) +def get_hotel_price_dtype() -> dict: + """ + Get HotelPrice datatype. + :return: HotelPrice datatype. + """ + logger.info('Get HotelPrice datatype...') + hotel_price_dtype = { + 'Hotel': 'text not null primary key', + 'Price': 'real not null', + 'Review': 'real not null', + 'Price/Review': 'real not null', + 'City': 'text not null', + 'Date': 'text not null', + 'AsOf': 'text not null' + } + return hotel_price_dtype + + def create_average_room_price_by_date_view(db: str) -> None: """ Create AverageRoomPriceByDate view diff --git a/main.py b/main.py index 582f613..5cb312c 100644 --- a/main.py +++ b/main.py @@ -22,7 +22,7 @@ from set_details import Details logger.add('japan_avg_hotel_price_month.log', - format="{time:YYYY-MM-DD at HH:mm:ss} | {level} | {thread} | {name} | {module} | {function} | {line} | {message}", + format="{time:YYYY-MM-DD at HH:mm:ss} | {level} | {name} | {module} | {function} | {line} | {message}", mode='w') # Initialize argument parser @@ -37,7 +37,9 @@ month = args.month details = Details(month=month) -if args.thread_pool: +if args.thread_pool and args.month_end: + logger.warning('Cannot use both --thread_pool and --month_end at the same time. Please use one of them at a time.') +elif args.thread_pool: logger.info('Using thread pool scraper') thread_scrape = ThreadPoolScraper(details) to_sqlite = args.to_sqlite diff --git a/set_details.py b/set_details.py index a42cce5..2054155 100644 --- a/set_details.py +++ b/set_details.py @@ -29,8 +29,8 @@ class Details: """ # Set booking details. city: str = 'Osaka' - check_in: str = '2024-12-28' - check_out: str = '2024-12-29' + check_in: str = '2024-09-29' + check_out: str = '2024-09-30' group_adults: int = 1 num_rooms: int = 1 group_children: int = 0