From 11fd3e9034840db179a1d9b59b5aafdea6c33308 Mon Sep 17 00:00:00 2001 From: Jakub Sucheta Date: Thu, 14 Dec 2023 17:38:14 +0100 Subject: [PATCH 1/8] series 50%, fav dzialaja --- FILMAN-CRAWLER/main.py | 21 +- FILMAN-CRAWLER/tasks/scrap_series.py | 106 +++++++++++ ...atched.py => scrap_user_watched_movies.py} | 22 ++- .../tasks/scrap_user_watched_series.py | 180 ++++++++++++++++++ FILMAN-DISCORD/main.py | 22 +-- FILMAN-SERVER/main.py | 73 ++++++- FILMAN-SERVER/movie.py | 4 +- FILMAN-SERVER/series.py | 87 +++++++++ FILMAN-SERVER/tasks.py | 3 +- FILMAN-SERVER/watched.py | 75 +++++++- 10 files changed, 555 insertions(+), 38 deletions(-) create mode 100644 FILMAN-CRAWLER/tasks/scrap_series.py rename FILMAN-CRAWLER/tasks/{scrap_user_watched.py => scrap_user_watched_movies.py} (90%) create mode 100644 FILMAN-CRAWLER/tasks/scrap_user_watched_series.py create mode 100644 FILMAN-SERVER/series.py diff --git a/FILMAN-CRAWLER/main.py b/FILMAN-CRAWLER/main.py index 497cc07..12dbfb5 100644 --- a/FILMAN-CRAWLER/main.py +++ b/FILMAN-CRAWLER/main.py @@ -7,8 +7,8 @@ from fake_useragent import UserAgent from tasks.scrap_movie import Scraper as movie_scrapper -from tasks.scrap_user_watched import Scraper as user_watched_scrapper - +from tasks.scrap_user_watched_movies import Scraper as user_watched_movies_scrapper +from tasks.scrap_user_watched_series import Scraper as user_watched_series_scrapper from concurrent.futures import ThreadPoolExecutor @@ -93,6 +93,7 @@ def fetch_tasks_from_endpoint(): allowed_tasks = [ "scrap_movie", "check_user_new_movies", + "check_user_new_series", ] try: @@ -149,13 +150,27 @@ def main(): executor.submit(m_scraper.scrap, task) if task.type == "check_user_new_movies": - m_scraper = user_watched_scrapper( + m_scraper = user_watched_movies_scrapper( headers=HEADERS, movie_id=task.job, endpoint_url=CORE_ENDPOINT, ) executor.submit(m_scraper.scrap, task) + if task.type == "check_user_new_series": + logging.info("check_user_new_series") + logging.info("check_user_new_series") + logging.info("check_user_new_series") + logging.info("check_user_new_series") + s_scraper = user_watched_series_scrapper( + headers=HEADERS, + series_id=task.job, + endpoint_url=CORE_ENDPOINT, + ) + executor.submit(s_scraper.scrap, task) + + + else: logging.info("No tasks found") wait_time = min(wait_time * 2, max_wait) diff --git a/FILMAN-CRAWLER/tasks/scrap_series.py b/FILMAN-CRAWLER/tasks/scrap_series.py new file mode 100644 index 0000000..f1bca38 --- /dev/null +++ b/FILMAN-CRAWLER/tasks/scrap_series.py @@ -0,0 +1,106 @@ +import logging +import requests +import ujson + + +logging.basicConfig( + format="%(asctime)s %(levelname)-8s %(message)s", + level=logging.INFO, + datefmt="%Y-%m-%d %H:%M:%S", +) + + +class Task: + def __init__( + self, id_task, status, type, job, unix_timestamp, unix_timestamp_last_update + ): + self.id_task = id_task + self.status = status + self.type = type + self.job = job + self.unix_timestamp = unix_timestamp + self.unix_timestamp_last_update = unix_timestamp_last_update + + def __str__(self): + return ( + f"{self.id_task} {self.status} {self.type} {self.job} {self.unix_timestamp}" + ) + + +class Scraper: + def __init__(self, headers=None, series_id=None, endpoint_url=None): + self.headers = headers + self.series_id = series_id + self.endpoint_url = endpoint_url + + def fetch(self, url): + response = requests.get(url, headers=self.headers) + if response.status_code != 200: + logging.error(f"Error fetching {url}: HTTP {response.status_code}") + return None + return response.text + + def scrap(self, task): + info_url = f"https://www.filmweb.pl/api/v1/title/{task.job}/info" + rating_url = f"https://www.filmweb.pl/api/v1/film/{task.job}/rating" + + info_data = self.fetch(info_url) + rating_data = self.fetch(rating_url) + + if info_data is None or rating_data is None: + return False + + info_data = ujson.loads(info_data) + rating_data = ujson.loads(rating_data) + + title = info_data.get("title", None) + year = int(info_data.get("year", None)) + other_year = int(info_data.get("otherYear", None)) + poster_url = info_data.get("posterPath", "https://vectorified.com/images/no-data-icon-23.png") + community_rate = rating_data.get("rate", None) + + if title is None or year is None or poster_url is None: + return False + + + + def update_data(self, series_id, title, year, other_year ,poster_url, community_rate, id_task): + r = requests.post( + f"{self.endpoint_url}/series/update", + headers=self.headers, + json={ + "id": int(series_id), + "title": str(title), + "year": int(year), + "other_year": int(other_year), + "poster_uri": str(poster_url), + "community_rate": float(community_rate), + }, + ) + + # r = requests.post( + # f"{self.endpoint_url}/movie/update", + # headers=self.headers, + # json={ + # "id": int(movie_id), + # "title": str(title), + # "year": int(year), + # "poster_uri": str(poster_url), + # "community_rate": float(community_rate), + # }, + # ) + + if r.status_code != 200: + logging.error(f"Error updating series data: HTTP {r.status_code}") + logging.error(r.text) + return False + + r = requests.get( + f"{self.endpoint_url}/task/update?id_task={id_task}&status=done", + headers=self.headers, + ) + + if r.status_code != 200: + return False + + return True diff --git a/FILMAN-CRAWLER/tasks/scrap_user_watched.py b/FILMAN-CRAWLER/tasks/scrap_user_watched_movies.py similarity index 90% rename from FILMAN-CRAWLER/tasks/scrap_user_watched.py rename to FILMAN-CRAWLER/tasks/scrap_user_watched_movies.py index 8a9c49c..d059b88 100644 --- a/FILMAN-CRAWLER/tasks/scrap_user_watched.py +++ b/FILMAN-CRAWLER/tasks/scrap_user_watched_movies.py @@ -22,18 +22,18 @@ def __str__(self): class Watched: def __init__( - self, id_watched, id_filmweb, movie_id, rate, comment, favourite, unix_timestamp + self, id_watched, id_filmweb, movie_id, rate, comment, favorite, unix_timestamp ): self.id_watched = id_watched self.id_filmweb = id_filmweb self.movie_id = movie_id self.rate = rate self.comment = comment - self.favourite = favourite + self.favorite = favorite self.unix_timestamp = unix_timestamp def __str__(self): - return f"{self.id_watched} {self.id_filmweb} {self.movie_id} {self.rate} {self.comment} {self.favourite} {self.unix_timestamp}" + return f"{self.id_watched} {self.id_filmweb} {self.movie_id} {self.rate} {self.comment} {self.favorite} {self.unix_timestamp}" def to_dict(self): return { @@ -42,7 +42,7 @@ def to_dict(self): "movie_id": self.movie_id, "rate": self.rate, "comment": self.comment, - "favourite": self.favourite, + "favorite": self.favorite, "unix_timestamp": self.unix_timestamp, } @@ -83,6 +83,7 @@ def scrap(self, task): first_time_scrap = user_already_watched_data is None if last_100_watched_data is None: + self.update_task_status_done(task.id_task) return "Private profile" last_100_watched_data = ujson.loads(last_100_watched_data) @@ -103,6 +104,7 @@ def scrap(self, task): new_movies_ids.append(filmweb_watched[0]) if len(new_movies_ids) == 0: + self.update_task_status_done(task.id_task) return "No new movies" new_movies_watched = [] @@ -123,7 +125,7 @@ def scrap(self, task): movie_id=movie_id, rate=movie_rate_data["rate"], comment=movie_rate_data.get("comment", None), - favourite=bool(movie_rate_data.get("favourite", False)), + favorite=bool(movie_rate_data.get("favorite", False)), unix_timestamp=movie_rate_data["timestamp"], ) @@ -160,11 +162,19 @@ def update_data( logging.error(r.text) return "Error adding movies" + self.update_task_status_done(id_task) + + if r.status_code != 200: + return "Error updating task" + + return True + + def update_task_status_done(self, id_task: int): r = requests.get( f"{self.endpoint_url}/task/update?id_task={id_task}&status=done" ) if r.status_code != 200: - return "Error updating task" + return False return True diff --git a/FILMAN-CRAWLER/tasks/scrap_user_watched_series.py b/FILMAN-CRAWLER/tasks/scrap_user_watched_series.py new file mode 100644 index 0000000..fdca7af --- /dev/null +++ b/FILMAN-CRAWLER/tasks/scrap_user_watched_series.py @@ -0,0 +1,180 @@ +import logging +import requests +import ujson + + +class Task: + def __init__( + self, id_task, status, type, job, unix_timestamp, unix_timestamp_last_update + ): + self.id_task = id_task + self.status = status + self.type = type + self.job = job + self.unix_timestamp = unix_timestamp + self.unix_timestamp_last_update = unix_timestamp_last_update + + def __str__(self): + return ( + f"{self.id_task} {self.status} {self.type} {self.job} {self.unix_timestamp}" + ) + + +class Watched: + def __init__( + self, id_watched, id_filmweb, series_id, rate, comment, favorite, unix_timestamp + ): + self.id_watched = id_watched + self.id_filmweb = id_filmweb + self.series_id = series_id + self.rate = rate + self.comment = comment + self.favorite = favorite + self.unix_timestamp = unix_timestamp + + def __str__(self): + return f"{self.id_watched} {self.id_filmweb} {self.series_id} {self.rate} {self.comment} {self.favorite} {self.unix_timestamp}" + + def to_dict(self): + return { + "id_watched": self.id_watched, + "id_filmweb": self.id_filmweb, + "series_id": self.series_id, + "rate": self.rate, + "comment": self.comment, + "favorite": self.favorite, + "unix_timestamp": self.unix_timestamp, + } + + +# https://www.filmweb.pl/api/v1/user/tirstus/vote/film + +# https://www.filmweb.pl/api/v1/user/tirstus/vote/film/783759 + +logging.basicConfig( + format="%(asctime)s %(levelname)-8s %(message)s", + level=logging.INFO, + datefmt="%Y-%m-%d %H:%M:%S", +) + + +class Scraper: + def __init__(self, headers=None, series_id=None, endpoint_url=None): + self.headers = headers + self.series_id = series_id + self.endpoint_url = endpoint_url + + def fetch(self, url): + response = requests.get(url, headers=self.headers) + if response.status_code != 200: + logging.error(f"Error fetching {url}: HTTP {response.status_code}") + return None + return response.text + + def scrap(self, task): + last_100_watched = f"https://www.filmweb.pl/api/v1/user/{task.job}/vote/serial" + user_already_watched = ( + f"{self.endpoint_url}/user/watched/series/all?id_filmweb={task.job}" + ) + + last_100_watched_data = self.fetch(last_100_watched) + user_already_watched_data = self.fetch(user_already_watched) + + first_time_scrap = user_already_watched_data is None + + if last_100_watched_data is None: + self.update_task_status_done(task.id_task) + return "Private profile" + + last_100_watched_data = ujson.loads(last_100_watched_data) + + if user_already_watched_data is None: + user_already_watched_data = [] + else: + user_already_watched_data = ujson.loads(user_already_watched_data) + + user_already_watched_data = list( + map(lambda x: x["series_id"], user_already_watched_data) + ) + + new_series_id = [] + + for filmweb_watched in last_100_watched_data: + if filmweb_watched[0] not in user_already_watched_data: + new_series_id.append(filmweb_watched[0]) + + if len(new_series_id) == 0: + self.update_task_status_done(task.id_task) + return "No new series" + + new_series_watched = [] + + for series_id in new_series_id: + series_rate_data = self.fetch( + f"https://www.filmweb.pl/api/v1/user/{task.job}/vote/serial/{series_id}" + ) + + if series_rate_data is None: + continue + + series_rate_data = ujson.loads(series_rate_data) + + series_watched = Watched( + None, + task.job, + series_id=series_id, + rate=series_rate_data["rate"], + comment=series_rate_data.get("comment", None), + favorite=bool(series_rate_data.get("favorite", False)), + unix_timestamp=series_rate_data["timestamp"], + ) + + new_series_watched.append(series_watched) + + return self.update_data( + task.job, + new_series_watched, + task.id_task, + first_time_scrap, + ) + + def update_data( + self, + filmweb_id: str, + series_watched: list, + id_task: int, + without_discord: bool, + ): + if len(filmweb_id) == 0: + return "No new series" + + r = requests.post( + f"{self.endpoint_url}/user/watched/series/add_many", + json={ + "id_filmweb": filmweb_id, + "series": [series.to_dict() for series in series_watched], + "without_discord": without_discord, + }, + ) + + if r.status_code != 200: + logging.error(f"Error adding series: HTTP {r.status_code}") + logging.error(r.text) + return "Error adding series" + + self.update_task_status_done(id_task) + + if r.status_code != 200: + return "Error updating task" + + return True + + def update_task_status_done(self, id_task: int): + r = requests.get( + f"{self.endpoint_url}/task/update?id_task={id_task}&status=done" + ) + + if r.status_code != 200: + return False + + return True diff --git a/FILMAN-DISCORD/main.py b/FILMAN-DISCORD/main.py index f81a493..739413f 100644 --- a/FILMAN-DISCORD/main.py +++ b/FILMAN-DISCORD/main.py @@ -158,7 +158,7 @@ async def send_discord_message( print("movie_title", movie["title"]) print("movie_year", movie["year"]) print("comment", user_rate["comment"]) - print("favourite", user_rate["favourite"]) + print("favorite", user_rate["favorite"]) print("rate", user_rate["rate"]) movie_url = filmweb_movie_url_generator( @@ -188,28 +188,20 @@ async def send_discord_message( inline=True, ) - if user_rate["comment"]: + if user_rate["favorite"] != 0: embed.add_field( - name="Komentarz", - value=f"{user_rate['comment']}", + name="Ulubiony", + value=f"❤️ Tak", inline=True, ) - if user_rate["favourite"] != 0: + if user_rate["comment"]: embed.add_field( - name="Ulubiony", - value=f"❤️ Tak", + name="Komentarz", + value=f"{user_rate['comment']}", inline=True, ) - # if user_rate["comment"] or user_rate["favourite"] != 0: - # blank_field = "\u200b" - # embed.add_field( - # name=blank_field, - # value=blank_field, - # inline=False, - # ) - if user_rate["rate"] != 0: embed.add_field( name=f"Ocena `{user_info['id_filmweb']}`", diff --git a/FILMAN-SERVER/main.py b/FILMAN-SERVER/main.py index 4b1a1c0..4143895 100644 --- a/FILMAN-SERVER/main.py +++ b/FILMAN-SERVER/main.py @@ -9,6 +9,10 @@ from movie import Movie as MovieManagerMovie from movie import MovieManager + +from series import Series as SeriesManagerSeries +from series import SeriesManager + from watched import WatchedManager from users import UserManager from discord_m import DiscordManager @@ -108,7 +112,7 @@ class MovieWatched(BaseModel): movie_id: int rate: int comment: Optional[str] = None - favourite: bool = False + favorite: bool = False unix_timestamp: int @@ -128,12 +132,44 @@ async def add_many_movies(movies_in: MovieWatchedList): movie.movie_id, movie.rate, movie.comment, - movie.favourite, + movie.favorite, movie.unix_timestamp, movies_in.without_discord, ) +class SeriesWatched(BaseModel): + id_watched: Optional[int] = None + id_filmweb: str + series_id: int + rate: int + comment: Optional[str] = None + favorite: bool = False + unix_timestamp: int + + +class SeriesWatchedList(BaseModel): + id_filmweb: str + series: list[SeriesWatched] + without_discord: Optional[bool] = False + + +@app.post("/user/watched/series/add_many") +async def add_many_series(series_in: SeriesWatchedList): + watched_manager = WatchedManager() + + for series in series_in.series: + result = watched_manager.add_watched_series( + series_in.id_filmweb, + series.series_id, + series.rate, + series.comment, + series.favorite, + series.unix_timestamp, + series_in.without_discord, + ) + + @app.post("/user/create") async def create_user(user_in: UserIn): user_manager = UserManager() @@ -187,6 +223,24 @@ async def get_user_destinations(id_discord: int): else: return result +################################################## +# SERIES +################################################## + +class SeriesUpdateIn(BaseModel): + id: int + title: str + year: int + other_year: Optional[int] = None + poster_uri: str + community_rate: float + +@app.get("/series/update") +async def update_series(series_update_in: SeriesUpdateIn): + series_manager = SeriesManager() + + # tu skonczyles + ################################################## # MOVIE @@ -269,7 +323,8 @@ async def scrap_all_users(): return {"message": "OK"} else: raise HTTPException(status_code=500, detail=result) - + + @app.get("/task/scrap/all/movies") async def scrap_all_movies(): tasks_manager = TasksManager() @@ -279,6 +334,14 @@ async def scrap_all_movies(): else: raise HTTPException(status_code=500, detail=result) +@app.get("/tasks/scrap/all/series") +async def scrap_all_series(): + tasks_manager = TasksManager() + result = tasks_manager.add_scrap_series_task() + if result is True: + return {"message": "OK"} + else: + raise HTTPException(status_code=500, detail=result) @app.get("/tasks/update/stuck") async def update_stuck_tasks(): @@ -338,7 +401,7 @@ async def add_user_to_guild(guild_in: GuildIn): if result == "User not found in database": raise HTTPException(status_code=404, detail=result) - + if result == "Server not configured": raise HTTPException(status_code=405, detail=result) @@ -359,4 +422,4 @@ async def configure_guild(guild_configure_in: GuildConfigureIn): if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=8000, log_level="debug") \ No newline at end of file + uvicorn.run(app, host="0.0.0.0", port=8000, log_level="debug") diff --git a/FILMAN-SERVER/movie.py b/FILMAN-SERVER/movie.py index b10de3a..9a10f7b 100644 --- a/FILMAN-SERVER/movie.py +++ b/FILMAN-SERVER/movie.py @@ -9,9 +9,7 @@ def __init__(self, **kwargs): self.id = kwargs.get("id") self.title = kwargs.get("title") self.year = kwargs.get("year") - self.poster_uri = kwargs.get( - "poster_uri" - ) # https://fwcdn.pl/fpo/20/32/32032/6933343.$.jpg + self.poster_uri = kwargs.get("poster_uri") self.community_rate = kwargs.get("community_rate") def __str__(self): diff --git a/FILMAN-SERVER/series.py b/FILMAN-SERVER/series.py new file mode 100644 index 0000000..67a1179 --- /dev/null +++ b/FILMAN-SERVER/series.py @@ -0,0 +1,87 @@ +import time + +from db import Database +from tasks import TasksManager + + +class Series: + def __init__(self, **kwargs): + self.id = kwargs.get("id") + self.title = kwargs.get("title") + self.year = kwargs.get("year") + self.other_year = kwargs.get("other_year") + self.poster_uri = kwargs.get("poster_uri") + self.community_rate = kwargs.get("community_rate") + + def __str__(self): + return f"{self.title} ({self.year})" + + +class SeriesManager: + def __init__(self): + pass + + def add_series_to_db(self, series): + db = Database() + + db.cursor.execute(f"SELECT * FROM series WHERE id = %s", (series.id,)) + + result = db.cursor.fetchone() + + if result is None: + db.cursor.execute( + f"INSERT INTO series (id, updated_unixtime, title, year, other_year, poster_uri, community_rate) VALUES (%s, %s, %s, %s, %s, %s, %s)", + ( + series.id, + int(time.time()), + series.title, + series.year, + series.other_year, + series.poster_uri, + series.community_rate, + ), + ) + + db.connection.commit() + db.connection.close() + + return True + + db.cursor.execute( + f"UPDATE series SET updated_unixtime = %s, title = %s, year = %s, other_year = %s, poster_uri = %s, community_rate = %s WHERE id = %s", + ( + int(time.time()), + series.title, + series.year, + series.other_year, + series.poster_uri, + series.community_rate, + series.id, + ), + ) + + db.connection.commit() + db.connection.close() + + return True + + def get_series_by_id(self, id): + db = Database() + db.cursor.execute(f"SELECT * FROM series WHERE id = {id}") + + result = db.cursor.fetchone() + + if result is None: + task_manager = TasksManager() + task_manager.new_task("scrap_series", id) + + return None + else: + return Series( + id=result[0], + title=result[3], + year=result[4], + other_year=result[5], + poster_uri=result[6], + community_rate=result[7], + ) diff --git a/FILMAN-SERVER/tasks.py b/FILMAN-SERVER/tasks.py index 74f0bf6..c99ba83 100644 --- a/FILMAN-SERVER/tasks.py +++ b/FILMAN-SERVER/tasks.py @@ -113,6 +113,7 @@ def add_scrap_users_task(self): for user in all_users: self.new_task("check_user_new_movies", user.id_filmweb) + self.new_task("check_user_new_series", user.id_filmweb) return True @@ -151,7 +152,7 @@ def get_and_update_tasks(self, types: list = None, status: str = "waiting"): types_placeholder = ", ".join(["%s"] * len(types)) db.cursor.execute( - f"SELECT * FROM tasks WHERE type IN ({types_placeholder}) AND status = %s ORDER BY id_task ASC LIMIT 10", + f"SELECT * FROM tasks WHERE type IN ({types_placeholder}) AND status = %s ORDER BY id_task ASC LIMIT 5", (*types, status), ) diff --git a/FILMAN-SERVER/watched.py b/FILMAN-SERVER/watched.py index 7a86c41..e7f91de 100644 --- a/FILMAN-SERVER/watched.py +++ b/FILMAN-SERVER/watched.py @@ -1,19 +1,21 @@ from db import Database from movie import Movie, MovieManager +from series import Series, SeriesManager from tasks import TasksManager from users import UserManager from discord_m import DiscordManager + class Watched: def __init__( - self, id_watched, id_filmweb, movie_id, rate, comment, favourite, unix_timestamp + self, id_watched, id_filmweb, movie_id, rate, comment, favorite, unix_timestamp ): self.id_watched = id_watched self.id_filmweb = id_filmweb self.movie_id = movie_id self.rate = rate self.comment = comment - self.favourite = favourite + self.favorite = favorite self.unix_timestamp = unix_timestamp def __str__(self): @@ -48,13 +50,76 @@ def update_movie_data( return True + def add_watched_series( + self, + id_filmweb: str, + series_id: int, + rate: int, + comment: str, + favorite: bool, + unix_timestamp: int, + without_discord: bool, + ): + db = Database() + series_manager = SeriesManager() + + # check series is in db + db.cursor.execute(f"SELECT * FROM series WHERE id = %s", (series_id,)) + result = db.cursor.fetchone() + + if result is None: + # if not, create task to scrap series + + series_manager.add_series_to_db( + Series( + id=series_id, + title="", + year=0, + other_year=0, + poster_uri="", + community_rate=0, + ) + ) + + task_manager = TasksManager() + task_manager.new_task("scrap_series", series_id) + + # check it is not already in watched + db.cursor.execute( + f"SELECT * FROM watched_series WHERE id_filmweb = %s AND series_id = %s", + (id_filmweb, series_id), + ) + + result = db.cursor.fetchone() + + if result is not None: + print("Already watched") + return "Already watched" + + # add to watched + db.cursor.execute( + f"INSERT INTO watched_series (id_filmweb, series_id, rate, comment, favourite, unix_timestamp) VALUES (%s, %s, %s, %s, %s, %s)", + (id_filmweb, series_id, rate, comment, favorite, unix_timestamp), + ) + + db.connection.commit() + db.connection.close() + + if without_discord is True: + return True + + task_manager = TasksManager() + task_manager.new_task("send_discord", f"{id_filmweb},{series_id}") + + return True + def add_watched_movie( self, id_filmweb: str, movie_id: int, rate: int, comment: str, - favourite: bool, + favorite: bool, unix_timestamp: int, without_discord: bool, ): @@ -96,7 +161,7 @@ def add_watched_movie( # add to watched db.cursor.execute( f"INSERT INTO watched_movies (id_filmweb, movie_id, rate, comment, favourite, unix_timestamp) VALUES (%s, %s, %s, %s, %s, %s)", - (id_filmweb, movie_id, rate, comment, favourite, unix_timestamp), + (id_filmweb, movie_id, rate, comment, favorite, unix_timestamp), ) db.connection.commit() db.connection.close() @@ -119,7 +184,7 @@ def get_all_watched_movies(self, id_filmweb: str): if result is None: return None - + if len(result) == 0: return None From 7fa93c4e914b2b8dd80127e494980bdf33e2c03b Mon Sep 17 00:00:00 2001 From: Jakub Sucheta Date: Thu, 14 Dec 2023 20:00:33 +0100 Subject: [PATCH 2/8] series scrapers works --- FILMAN-CRAWLER/main.py | 16 +++++++++------ FILMAN-CRAWLER/tasks/scrap_series.py | 21 ++++++-------------- FILMAN-DISCORD/endpoints/info.py | 2 +- FILMAN-SERVER/main.py | 29 +++++++++++++++++++++++++--- FILMAN-SERVER/series.py | 2 ++ FILMAN-SERVER/tasks.py | 11 +++++++++++ FILMAN-SERVER/users.py | 4 ++++ 7 files changed, 60 insertions(+), 25 deletions(-) diff --git a/FILMAN-CRAWLER/main.py b/FILMAN-CRAWLER/main.py index 12dbfb5..a3b6a30 100644 --- a/FILMAN-CRAWLER/main.py +++ b/FILMAN-CRAWLER/main.py @@ -7,6 +7,7 @@ from fake_useragent import UserAgent from tasks.scrap_movie import Scraper as movie_scrapper +from tasks.scrap_series import Scraper as series_scrapper from tasks.scrap_user_watched_movies import Scraper as user_watched_movies_scrapper from tasks.scrap_user_watched_series import Scraper as user_watched_series_scrapper @@ -92,6 +93,7 @@ def fetch_tasks_from_endpoint(): allowed_tasks = [ "scrap_movie", + "scrap_series", "check_user_new_movies", "check_user_new_series", ] @@ -149,6 +151,14 @@ def main(): ) executor.submit(m_scraper.scrap, task) + if task.type == "scrap_series": + s_scraper = series_scrapper( + headers=HEADERS, + series_id=task.job, + endpoint_url=CORE_ENDPOINT, + ) + executor.submit(s_scraper.scrap, task) + if task.type == "check_user_new_movies": m_scraper = user_watched_movies_scrapper( headers=HEADERS, @@ -158,10 +168,6 @@ def main(): executor.submit(m_scraper.scrap, task) if task.type == "check_user_new_series": - logging.info("check_user_new_series") - logging.info("check_user_new_series") - logging.info("check_user_new_series") - logging.info("check_user_new_series") s_scraper = user_watched_series_scrapper( headers=HEADERS, series_id=task.job, @@ -169,8 +175,6 @@ def main(): ) executor.submit(s_scraper.scrap, task) - - else: logging.info("No tasks found") wait_time = min(wait_time * 2, max_wait) diff --git a/FILMAN-CRAWLER/tasks/scrap_series.py b/FILMAN-CRAWLER/tasks/scrap_series.py index f1bca38..a4f90c5 100644 --- a/FILMAN-CRAWLER/tasks/scrap_series.py +++ b/FILMAN-CRAWLER/tasks/scrap_series.py @@ -55,16 +55,19 @@ def scrap(self, task): title = info_data.get("title", None) year = int(info_data.get("year", None)) - other_year = int(info_data.get("otherYear", None)) + other_year = int(info_data.get("otherYear", 0)) poster_url = info_data.get("posterPath", "https://vectorified.com/images/no-data-icon-23.png") community_rate = rating_data.get("rate", None) if title is None or year is None or poster_url is None: return False + + return self.update_data( + self.series_id, title, year, other_year, poster_url, community_rate, task.id_task + ) - - def update_data(self, series_id, title, year, other_year ,poster_url, community_rate, id_task): + def update_data(self, series_id, title, year, other_year, poster_url, community_rate, id_task): r = requests.post( f"{self.endpoint_url}/series/update", headers=self.headers, @@ -78,18 +81,6 @@ def update_data(self, series_id, title, year, other_year ,poster_url, community_ }, ) - # r = requests.post( - # f"{self.endpoint_url}/movie/update", - # headers=self.headers, - # json={ - # "id": int(movie_id), - # "title": str(title), - # "year": int(year), - # "poster_uri": str(poster_url), - # "community_rate": float(community_rate), - # }, - # ) - if r.status_code != 200: logging.error(f"Error updating series data: HTTP {r.status_code}") logging.error(r.text) diff --git a/FILMAN-DISCORD/endpoints/info.py b/FILMAN-DISCORD/endpoints/info.py index acc3663..1a6dda2 100644 --- a/FILMAN-DISCORD/endpoints/info.py +++ b/FILMAN-DISCORD/endpoints/info.py @@ -16,7 +16,7 @@ async def info_command(ctx: lightbulb.SlashContext) -> None: embed.add_field( name="Wersja i ostatnia aktualizacja", - value="`1.0.1` - `2023-12-13`", + value="`1.0.2` - `2023-12-16`", ) embed.add_field( diff --git a/FILMAN-SERVER/main.py b/FILMAN-SERVER/main.py index 4143895..423d543 100644 --- a/FILMAN-SERVER/main.py +++ b/FILMAN-SERVER/main.py @@ -231,15 +231,29 @@ class SeriesUpdateIn(BaseModel): id: int title: str year: int - other_year: Optional[int] = None + other_year: Optional[int] poster_uri: str community_rate: float -@app.get("/series/update") +@app.post("/series/update") async def update_series(series_update_in: SeriesUpdateIn): series_manager = SeriesManager() - # tu skonczyles + result = series_manager.add_series_to_db( + SeriesManagerSeries( + id=series_update_in.id, + title=series_update_in.title, + year=series_update_in.year, + other_year=series_update_in.other_year, + poster_uri=series_update_in.poster_uri, + community_rate=series_update_in.community_rate, + ) + ) + + if result is True: + return {"message": "OK"} + else: + raise HTTPException(status_code=500, detail=result) ################################################## @@ -333,6 +347,15 @@ async def scrap_all_movies(): return {"message": "OK"} else: raise HTTPException(status_code=500, detail=result) + +@app.get("/task/scrap/all/series") +async def scrap_all_series(): + tasks_manager = TasksManager() + result = tasks_manager.add_scrap_series_task() + if result is True: + return {"message": "OK"} + else: + raise HTTPException(status_code=500, detail=result) @app.get("/tasks/scrap/all/series") async def scrap_all_series(): diff --git a/FILMAN-SERVER/series.py b/FILMAN-SERVER/series.py index 67a1179..0d1aaef 100644 --- a/FILMAN-SERVER/series.py +++ b/FILMAN-SERVER/series.py @@ -28,6 +28,8 @@ def add_series_to_db(self, series): result = db.cursor.fetchone() + series.other_year = 0 if series.other_year is None else series.other_year + if result is None: db.cursor.execute( f"INSERT INTO series (id, updated_unixtime, title, year, other_year, poster_uri, community_rate) VALUES (%s, %s, %s, %s, %s, %s, %s)", diff --git a/FILMAN-SERVER/tasks.py b/FILMAN-SERVER/tasks.py index c99ba83..a70aadc 100644 --- a/FILMAN-SERVER/tasks.py +++ b/FILMAN-SERVER/tasks.py @@ -127,6 +127,17 @@ def add_scrap_movies_task(self): self.new_task("scrap_movie", movie[0]) return True + + def add_scrap_series_task(self): + db = Database() + + db.cursor.execute("SELECT id FROM series") + result = db.cursor.fetchall() + + for series in result: + self.new_task("scrap_series", series[0]) + + return True def new_task(self, type: str, job: str): # status: waiting, in_progress, done, failed diff --git a/FILMAN-SERVER/users.py b/FILMAN-SERVER/users.py index 03cfd00..619c036 100644 --- a/FILMAN-SERVER/users.py +++ b/FILMAN-SERVER/users.py @@ -79,6 +79,10 @@ def create_user(self, id_filmweb: str, id_discord: int): "check_user_new_movies", f"{id_filmweb}", ) + tasks_manager.new_task( + "check_user_new_series", + f"{id_filmweb}", + ) return True From 4ea562dd957ff8c912efbb4809d602c372dd151a Mon Sep 17 00:00:00 2001 From: Jakub Sucheta Date: Tue, 19 Dec 2023 21:19:21 +0100 Subject: [PATCH 3/8] moving to sql alchemy --- FILMAN-SERVER/database/crud.py | 152 ++++++ FILMAN-SERVER/database/db.py | 14 + FILMAN-SERVER/database/models.py | 114 +++++ FILMAN-SERVER/database/schemas.py | 155 +++++++ FILMAN-SERVER/db.py | 25 - FILMAN-SERVER/main.py | 462 ++----------------- FILMAN-SERVER/{ => old_way}/discord_m.py | 0 FILMAN-SERVER/{ => old_way}/hahaha.txt | 0 FILMAN-SERVER/{ => old_way}/movie.py | 0 FILMAN-SERVER/{ => old_way}/requirements.txt | Bin FILMAN-SERVER/{ => old_way}/series.py | 0 FILMAN-SERVER/{ => old_way}/tasks.py | 0 FILMAN-SERVER/{ => old_way}/users.py | 0 FILMAN-SERVER/{ => old_way}/utils.py | 0 FILMAN-SERVER/{ => old_way}/watched.py | 0 FILMAN-SERVER/routes/discord.py | 31 ++ FILMAN-SERVER/routes/filmweb.py | 31 ++ FILMAN-SERVER/routes/users.py | 99 ++++ 18 files changed, 625 insertions(+), 458 deletions(-) create mode 100644 FILMAN-SERVER/database/crud.py create mode 100644 FILMAN-SERVER/database/db.py create mode 100644 FILMAN-SERVER/database/models.py create mode 100644 FILMAN-SERVER/database/schemas.py delete mode 100644 FILMAN-SERVER/db.py rename FILMAN-SERVER/{ => old_way}/discord_m.py (100%) rename FILMAN-SERVER/{ => old_way}/hahaha.txt (100%) rename FILMAN-SERVER/{ => old_way}/movie.py (100%) rename FILMAN-SERVER/{ => old_way}/requirements.txt (100%) rename FILMAN-SERVER/{ => old_way}/series.py (100%) rename FILMAN-SERVER/{ => old_way}/tasks.py (100%) rename FILMAN-SERVER/{ => old_way}/users.py (100%) rename FILMAN-SERVER/{ => old_way}/utils.py (100%) rename FILMAN-SERVER/{ => old_way}/watched.py (100%) create mode 100644 FILMAN-SERVER/routes/discord.py create mode 100644 FILMAN-SERVER/routes/filmweb.py create mode 100644 FILMAN-SERVER/routes/users.py diff --git a/FILMAN-SERVER/database/crud.py b/FILMAN-SERVER/database/crud.py new file mode 100644 index 0000000..00ef09a --- /dev/null +++ b/FILMAN-SERVER/database/crud.py @@ -0,0 +1,152 @@ +from sqlalchemy.orm import Session + +from . import models, schemas + +# +# USERS +# + + +def get_user( + db: Session, id: int | None, filmweb_id: str | None, discord_id: int | None +): + if id: + return db.query(models.User).filter(models.User.id == id).first() + elif filmweb_id: + return ( + db.query(models.User).filter(models.User.filmweb_id == filmweb_id).first() + ) + elif discord_id: + return ( + db.query(models.User).filter(models.User.discord_id == discord_id).first() + ) + else: + return None + + +def create_user(db: Session, user: schemas.UserCreate): + db_user = models.User(**user.model_dump()) + db.add(db_user) + db.commit() + db.refresh(db_user) + return db_user + + +def get_user_destinations(db: Session, user_id: int): + return db.query(models.DiscordDestinations).filter_by(user_id=user_id).all() + + +def get_user_destination(db: Session, user_id: int, discord_guild_id: int): + return ( + db.query(models.DiscordDestinations) + .filter_by(user_id=user_id, discord_guild_id=discord_guild_id) + .first() + ) + + +def set_user_destination(db: Session, user_id: int, discord_guild_id: int): + db_dest = ( + db.query(models.DiscordDestinations) + .filter_by(user_id=user_id, discord_guild_id=discord_guild_id) + .first() + ) + + if db_dest is None: + db_dest = models.DiscordDestinations( + user_id=user_id, discord_guild_id=discord_guild_id + ) + db.add(db_dest) + else: + db_dest.discord_guild_id = discord_guild_id + + db.commit() + db.refresh(db_dest) + + return db_dest + + +def delete_user_destination(db: Session, user_id: int, discord_guild_id: int): + db_dest = ( + db.query(models.DiscordDestinations) + .filter_by(user_id=user_id, discord_guild_id=discord_guild_id) + .first() + ) + + if db_dest is None: + raise Exception("User not in guild") + + db.delete(db_dest) + db.commit() + + return db_dest + + +def delete_user_destitations(db: Session, user_id: int): + destinations = db.query(models.DiscordDestinations).filter( + models.DiscordDestinations.user_id == user_id + ) + + destinations.delete() + + db.commit() + + +# +# DISCORD +# + + +def get_guild(db: Session, discord_guild_id: int): + return ( + db.query(models.DiscordGuilds) + .filter(models.DiscordGuilds.discord_guild_id == discord_guild_id) + .first() + ) + + +def set_guild(db: Session, guild: schemas.DiscordGuildsCreate): + db_guild = get_guild(db, guild.discord_guild_id) + if db_guild is None: + return create_guild(db, guild) + db_guild.discord_channel_id = guild.discord_channel_id + db.commit() + db.refresh(db_guild) + return db_guild + + +def create_guild(db: Session, guild: schemas.DiscordGuildsCreate): + db_guild = models.DiscordGuilds(**guild.model_dump()) + db.add(db_guild) + db.commit() + db.refresh(db_guild) + return db_guild + + +# +# FILMWEB MOVIES +# + + +def get_movie_filmweb_id(db: Session, id: int): + return db.query(models.FilmWebMovie).filter(models.FilmWebMovie.id == id).first() + + +def insert_movie(db: Session, movie: schemas.FilmWebMovie): + db_movie = models.FilmWebMovie(**movie.model_dump()) + db.add(db_movie) + db.commit() + db.refresh(db_movie) + return db_movie + + +def update_movie(db: Session, movie: schemas.FilmWebMovie): + db_movie = get_movie_filmweb_id(db, movie.id) + if db_movie is None: + return insert_movie(db, movie) + db_movie.title = movie.title + db_movie.year = movie.year + db_movie.poster_url = movie.poster_url + db_movie.community_rate = movie.community_rate + db.commit() + db.refresh(db_movie) + return db_movie diff --git a/FILMAN-SERVER/database/db.py b/FILMAN-SERVER/database/db.py new file mode 100644 index 0000000..1b521d6 --- /dev/null +++ b/FILMAN-SERVER/database/db.py @@ -0,0 +1,14 @@ +import os + +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + +SQLALCHEMY_DATABASE_URL = os.environ.get( + "SQLALCHEMY_DATABASE_URL", "mysql://root@localhost/filmweb_test3" +) + +engine = create_engine(SQLALCHEMY_DATABASE_URL) +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + +Base = declarative_base() diff --git a/FILMAN-SERVER/database/models.py b/FILMAN-SERVER/database/models.py new file mode 100644 index 0000000..f9f8fb9 --- /dev/null +++ b/FILMAN-SERVER/database/models.py @@ -0,0 +1,114 @@ +from sqlalchemy import ( + Boolean, + Column, + ForeignKey, + Integer, + String, + VARCHAR, + SmallInteger, + DateTime, + Float, + BIGINT, +) +from sqlalchemy.orm import relationship + +from .db import Base + + +class User(Base): + __tablename__ = "users" + + id = Column(Integer, primary_key=True, index=True, autoincrement=True) + filmweb_id = Column(String(128), index=True, unique=True) + discord_id = Column(BIGINT, index=True, unique=True) + + filmweb_watched_movies = relationship( + "FilmWebUserWatchedMovie", backref="user", cascade="all, delete-orphan" + ) + + +class UserPreferences(Base): + __tablename__ = "user_preferences" + + id = Column(Integer, ForeignKey("users.id"), primary_key=True, index=True) + discord_color = Column(VARCHAR(length=6)) + + +# +# DISCORD +# + + +class DiscordGuilds(Base): + __tablename__ = "discord_guilds" + + id = Column(Integer, primary_key=True, index=True, autoincrement=True) + discord_guild_id = Column(BIGINT, index=True, unique=True) + discord_channel_id = Column(BIGINT, index=True, unique=True) + + +class DiscordDestinations(Base): + __tablename__ = "discord_destinations" + + user_id = Column(Integer, ForeignKey("users.id"), primary_key=True, index=True) + discord_guild_id = Column(BIGINT, ForeignKey("discord_guilds.discord_guild_id")) + + user = relationship("User", backref="discord_destinations") + + +# +# FILMWEB +# + + +class __FilmwebMedia(Base): + __abstract__ = True + + id = Column(Integer, primary_key=True, index=True) + title = Column(String(128)) + year = Column(SmallInteger) + poster_url = Column(String(128)) + community_rate = Column(Float) + + +class FilmWebMovie(__FilmwebMedia): + __tablename__ = "filmweb_movies" + + +class FilmWebSeries(__FilmwebMedia): + __tablename__ = "filmweb_series" + + +class __FilmwebWatched(Base): + __abstract__ = True + + date = Column(DateTime) + rate = Column(SmallInteger) + comment = Column(String(256)) + favorite = Column(Boolean) + + +class FilmWebUserWatchedMovie(__FilmwebWatched): + __tablename__ = "filmweb_user_watched_movies" + + id_media = Column( + Integer, ForeignKey("filmweb_movies.id"), primary_key=True, index=True + ) + id_filmweb = Column( + String(128), ForeignKey("users.filmweb_id"), primary_key=True, index=True + ) + + movie = relationship("FilmWebMovie", backref="filmweb_user_watched_movies") + + +class FilmWebUserWatchedSeries(__FilmwebWatched): + __tablename__ = "filmweb_user_watched_series" + + id_media = Column( + Integer, ForeignKey("filmweb_series.id"), primary_key=True, index=True + ) + id_filmweb = Column( + String(128), ForeignKey("users.filmweb_id"), primary_key=True, index=True + ) + + series = relationship("FilmWebSeries", backref="filmweb_user_watched_series") diff --git a/FILMAN-SERVER/database/schemas.py b/FILMAN-SERVER/database/schemas.py new file mode 100644 index 0000000..010eef9 --- /dev/null +++ b/FILMAN-SERVER/database/schemas.py @@ -0,0 +1,155 @@ +from pydantic import BaseModel +from datetime import datetime + +# +# USER +# + + +class User(BaseModel): + id: int + filmweb_id: str + discord_id: int + + class Config: + orm_mode = True + + +class UserCreate(BaseModel): + filmweb_id: str + discord_id: int + + class Config: + orm_mode = True + + +class UserPreferences(BaseModel): + id: int + discord_color: str + + class Config: + orm_mode = True + + +class UserPreferencesCreate(BaseModel): + discord_color: str + + class Config: + orm_mode = True + + +# +# DISCORD +# + + +class DiscordGuilds(BaseModel): + id: int + discord_guild_id: int + discord_channel_id: int + + class Config: + orm_mode = True + + +class DiscordGuildsCreate(BaseModel): + discord_guild_id: int + discord_channel_id: int + + class Config: + orm_mode = True + + +class DiscordDestinations(BaseModel): + user_id: int + discord_guild_id: int + + class Config: + orm_mode = True + + +class DiscordDestinationsCreate(BaseModel): + user_id: int + discord_guild_id: int + + class Config: + orm_mode = True + + +# +# FILMWEB +# + + +class FilmWebMovie(BaseModel): + id: int + title: str + year: int + poster_url: str + community_rate: float + + class Config: + orm_mode = True + + +class FilmWebMovieCreate(BaseModel): + id: int + title: str | None + year: int | None + poster_url: str | None + community_rate: float | None + + class Config: + orm_mode = True + + +class FilmWebUserWatchedMovie(BaseModel): + movie: FilmWebMovie + id_filmweb: str + + date: datetime + rate: int | None + comment: str | None + favorite: bool + + class Config: + orm_mode = True + + +class FilmWebUserWatchedMovieCreate(BaseModel): + id_media: int + id_filmweb: str + + date: datetime + rate: int | None + comment: str | None + favorite: bool + + class Config: + orm_mode = True + + +class FilmwebUserWatchedSeries(BaseModel): + series: FilmWebMovie + id_filmweb: str + + date: datetime + rate: int | None + comment: str | None + favorite: bool + + class Config: + orm_mode = True + + +class FilmwebUserWatchedSeriesCreate(BaseModel): + id_media: int + id_filmweb: str + + date: datetime + rate: int | None + comment: str | None + favorite: bool + + class Config: + orm_mode = True diff --git a/FILMAN-SERVER/db.py b/FILMAN-SERVER/db.py deleted file mode 100644 index fa437be..0000000 --- a/FILMAN-SERVER/db.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import mysql.connector - -# MYSQL_HOST = "127.0.0.1" -# MYSQL_USER = "root" -# MYSQL_PASSWORD = "" -# MYSQL_DATABASE = "filmweb_test" - -MYSQL_HOST = os.environ.get("MYSQL_HOST", "127.0.0.1") -MYSQL_USER = os.environ.get("MYSQL_USER", "root") -MYSQL_PASSWORD = os.environ.get("MYSQL_PASSWORD", "") -MYSQL_DATABASE = os.environ.get("MYSQL_DATABASE", "filmweb_test2") - -class Database: - def __init__(self): - self.connection = mysql.connector.connect( - host=MYSQL_HOST, - user=MYSQL_USER, - password=MYSQL_PASSWORD, - database=MYSQL_DATABASE, - ) - self.cursor = self.connection.cursor() - - def __del__(self): - self.connection.close() diff --git a/FILMAN-SERVER/main.py b/FILMAN-SERVER/main.py index 423d543..1380e69 100644 --- a/FILMAN-SERVER/main.py +++ b/FILMAN-SERVER/main.py @@ -1,448 +1,44 @@ -import threading import uvicorn -from fastapi import FastAPI, HTTPException, Request -from fastapi.middleware.cors import CORSMiddleware +import sentry_sdk +from sentry_sdk.integrations.starlette import StarletteIntegration +from sentry_sdk.integrations.fastapi import FastApiIntegration -from pydantic import BaseModel -from typing import List, Optional +from typing import Optional, List, Dict, Any -from movie import Movie as MovieManagerMovie -from movie import MovieManager +from fastapi import Depends, FastAPI, HTTPException +from sqlalchemy.orm import Session +from sqlalchemy.exc import IntegrityError -from series import Series as SeriesManagerSeries -from series import SeriesManager +from database import crud, models, schemas +from database.db import SessionLocal, engine -from watched import WatchedManager -from users import UserManager -from discord_m import DiscordManager -from tasks import TasksManager +from routes import users +from routes import discord +from routes import filmweb +models.Base.metadata.create_all(bind=engine) -app = FastAPI() - -################################################## -# MODELS -################################################## - - -class Movie(BaseModel): - movie_id: str - rate: str - comment: Optional[str] = None - favourite: bool - unix_timestamp: str - - -class MoviesIn(BaseModel): - id_filmweb: str - movies: List[Movie] - without_discord: Optional[bool] = False - - -class UserIn(BaseModel): - id_filmweb: str - id_discord: Optional[int] = None - - -class MovieIn(BaseModel): - id_filmweb: str - movie_id: int - rate: int - comment: str - favourite: bool - unix_timestamp: int - - -class MovieUpdateIn(BaseModel): - id: int - title: str - year: int - poster_uri: str - community_rate: float - - -class DiscordUserIdIn(BaseModel): - id_discord: int - - -class TaskUpdate(BaseModel): - type: str - status: str - - -class TaskUpdateIn(BaseModel): - id_task: int - status: str - - -class GuildIn(BaseModel): - id_discord: int - id_guild: int - - -class GuildStopIn(BaseModel): - id_discord: int - guild_id: int - - -class GuildConfigureIn(BaseModel): - guild_id: int - channel_id: int - - -################################################## -# USER -################################################## - - -@app.get("/user/watched/film/all") -async def get_all_movies(id_filmweb: str): - watched_manager = WatchedManager() - result = watched_manager.get_all_watched_movies(id_filmweb) - if result is None: - raise HTTPException(status_code=500, detail="No movies found") - else: - return result - - -class MovieWatched(BaseModel): - id_watched: Optional[int] = None - id_filmweb: str - movie_id: int - rate: int - comment: Optional[str] = None - favorite: bool = False - unix_timestamp: int - - -class MovieWatchedList(BaseModel): - id_filmweb: str - movies: list[MovieWatched] - without_discord: Optional[bool] = False - - -@app.post("/user/watched/film/add_many") -async def add_many_movies(movies_in: MovieWatchedList): - watched_manager = WatchedManager() - - for movie in movies_in.movies: - result = watched_manager.add_watched_movie( - movies_in.id_filmweb, - movie.movie_id, - movie.rate, - movie.comment, - movie.favorite, - movie.unix_timestamp, - movies_in.without_discord, - ) - - -class SeriesWatched(BaseModel): - id_watched: Optional[int] = None - id_filmweb: str - series_id: int - rate: int - comment: Optional[str] = None - favorite: bool = False - unix_timestamp: int - - -class SeriesWatchedList(BaseModel): - id_filmweb: str - series: list[SeriesWatched] - without_discord: Optional[bool] = False - - -@app.post("/user/watched/series/add_many") -async def add_many_series(series_in: SeriesWatchedList): - watched_manager = WatchedManager() - - for series in series_in.series: - result = watched_manager.add_watched_series( - series_in.id_filmweb, - series.series_id, - series.rate, - series.comment, - series.favorite, - series.unix_timestamp, - series_in.without_discord, - ) - - -@app.post("/user/create") -async def create_user(user_in: UserIn): - user_manager = UserManager() - result = user_manager.create_user(user_in.id_filmweb, user_in.id_discord) - if result is True: - return {"message": "OK"} - - if result == "User does not exist on filmweb": - raise HTTPException(status_code=404, detail=result) - - if result == "User already exists": - raise HTTPException(status_code=409, detail=result) - - raise HTTPException(status_code=500, detail=result) +sentry_sdk.init( + dsn="https://e90f28bf688cd7f8a1f8dcbc3586d359@o4506423653105664.ingest.sentry.io/4506423687774208", + enable_tracing=True, + integrations=[ + StarletteIntegration(transaction_style="endpoint"), + FastApiIntegration(transaction_style="endpoint"), + ], +) +app = FastAPI() -@app.post("/user/watched/film/add") -async def add_movie(movie_in: MovieIn): - watched_manager = WatchedManager() - result = watched_manager.add_watched_movie( - movie_in.id_filmweb, - movie_in.movie_id, - movie_in.rate, - movie_in.comment, - movie_in.favourite, - movie_in.unix_timestamp, - ) - - if result is True: - return {"message": "OK"} - else: - raise HTTPException(status_code=500, detail=result) - - -@app.get("/user/watched/film/get") -async def get_movie(id_filmweb: str, movie_id: int): - watched_manager = WatchedManager() - result = watched_manager.get_watched_movie_with_rates(id_filmweb, movie_id) - if result is None: - raise HTTPException(status_code=500, detail="No movies found") - else: - return result - - -@app.get("/user/discord/destinations/get") -async def get_user_destinations(id_discord: int): - discord_manager = DiscordManager() - result = discord_manager.get_user_notification_destinations_by_id(id_discord) - if result is None: - raise HTTPException(status_code=500, detail="No destinations found") - else: - return result - -################################################## -# SERIES -################################################## - -class SeriesUpdateIn(BaseModel): - id: int - title: str - year: int - other_year: Optional[int] - poster_uri: str - community_rate: float - -@app.post("/series/update") -async def update_series(series_update_in: SeriesUpdateIn): - series_manager = SeriesManager() - - result = series_manager.add_series_to_db( - SeriesManagerSeries( - id=series_update_in.id, - title=series_update_in.title, - year=series_update_in.year, - other_year=series_update_in.other_year, - poster_uri=series_update_in.poster_uri, - community_rate=series_update_in.community_rate, - ) - ) - - if result is True: - return {"message": "OK"} - else: - raise HTTPException(status_code=500, detail=result) - - -################################################## -# MOVIE -################################################## - - -@app.post("/movie/update") -async def update_movie(movie_update_in: MovieUpdateIn): - movie_manager = MovieManager() - - result = movie_manager.add_movie_to_db( - MovieManagerMovie( - id=movie_update_in.id, - title=movie_update_in.title, - year=movie_update_in.year, - poster_uri=movie_update_in.poster_uri, - community_rate=movie_update_in.community_rate, - ) - ) - - if result is True: - return {"message": "OK"} - else: - raise HTTPException(status_code=500, detail=result) - - -# @app.get("/movie/get - -################################################## -# TASKS -################################################## - - -class PostGetTasks(BaseModel): - status: str - types: list[str] - - -@app.post("/tasks/get") -async def post_get_tasks(post_get_tasks: PostGetTasks): - # if post_get_tasks.types is not None: - # types = ujson.dumps(types) - - tasks_manager = TasksManager() - result = tasks_manager.get_and_update_tasks( - post_get_tasks.types, post_get_tasks.status - ) - if result is None: - raise HTTPException(status_code=500, detail="No tasks found") - else: - return result - - -# tak to sa dwie inne funkcje -@app.get("/task/get") -async def get_task(type: str = None): - tasks_manager = TasksManager() - result = tasks_manager.get_task_by_type(type) - if result is None: - raise HTTPException(status_code=500, detail="No tasks found") - else: - return result - - -@app.get("/task/update") -async def update_task(id_task: int, status: str): - tasks_manager = TasksManager() - result = tasks_manager.update_task_status(id_task, status) - if result is True: - return {"message": "OK"} - else: - raise HTTPException(status_code=500, detail=result) - - -@app.get("/task/scrap/all/users") -async def scrap_all_users(): - tasks_manager = TasksManager() - result = tasks_manager.add_scrap_users_task() - if result is True: - return {"message": "OK"} - else: - raise HTTPException(status_code=500, detail=result) - - -@app.get("/task/scrap/all/movies") -async def scrap_all_movies(): - tasks_manager = TasksManager() - result = tasks_manager.add_scrap_movies_task() - if result is True: - return {"message": "OK"} - else: - raise HTTPException(status_code=500, detail=result) - -@app.get("/task/scrap/all/series") -async def scrap_all_series(): - tasks_manager = TasksManager() - result = tasks_manager.add_scrap_series_task() - if result is True: - return {"message": "OK"} - else: - raise HTTPException(status_code=500, detail=result) - -@app.get("/tasks/scrap/all/series") -async def scrap_all_series(): - tasks_manager = TasksManager() - result = tasks_manager.add_scrap_series_task() - if result is True: - return {"message": "OK"} - else: - raise HTTPException(status_code=500, detail=result) - -@app.get("/tasks/update/stuck") -async def update_stuck_tasks(): - tasks_manager = TasksManager() - result = tasks_manager.update_stuck_tasks() - if result is True: - return {"message": "OK"} - else: - raise HTTPException(status_code=500, detail=result) - - -@app.get("/tasks/delete/old") -async def delete_old_tasks(): - tasks_manager = TasksManager() - result = tasks_manager.delete_old_tasks() - if result is True: - return {"message": "OK"} - else: - raise HTTPException(status_code=500, detail=result) - - -################################################## -# DISCORD -################################################## - - -@app.post("/discord/user/stop") -async def stop_user_notifications(guild_stop_in: GuildStopIn): - discord_manager = DiscordManager() - result = discord_manager.delete_user_from_guild( - guild_stop_in.id_discord, guild_stop_in.guild_id - ) - if result is True: - return {"message": "OK"} - else: - raise HTTPException(status_code=500, detail=result) - - -@app.post("/discord/user/cancel") -async def cancel_user_notifications(discord_user_id: DiscordUserIdIn): - discord_manager = DiscordManager() - result = discord_manager.delete_user_from_all_destinations( - discord_user_id.id_discord - ) - if result is True: - return {"message": "OK"} - else: - raise HTTPException(status_code=500, detail=result) - - -@app.post("/discord/user/add") -async def add_user_to_guild(guild_in: GuildIn): - discord_manager = DiscordManager() - result = discord_manager.add_user_to_guild(guild_in.id_discord, guild_in.id_guild) - if result is True: - return {"message": "OK"} - - if result == "User not found in database": - raise HTTPException(status_code=404, detail=result) - - if result == "Server not configured": - raise HTTPException(status_code=405, detail=result) - - if result is False: - raise HTTPException(status_code=500, detail=result) +app.include_router(users.users_router) +app.include_router(discord.discord_router) +app.include_router(filmweb.filmweb_router) -@app.post("/discord/configure/guild") -async def configure_guild(guild_configure_in: GuildConfigureIn): - discord_manager = DiscordManager() - result = discord_manager.configure_guild( - guild_configure_in.guild_id, guild_configure_in.channel_id - ) - if result is True: - return {"message": "OK"} - else: - raise HTTPException(status_code=500, detail=result) +@app.get("/sentry-debug") +async def trigger_error(): + division_by_zero = 1 / 0 if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=8000, log_level="debug") + uvicorn.run(app, host="127.0.0.1", port=8000) diff --git a/FILMAN-SERVER/discord_m.py b/FILMAN-SERVER/old_way/discord_m.py similarity index 100% rename from FILMAN-SERVER/discord_m.py rename to FILMAN-SERVER/old_way/discord_m.py diff --git a/FILMAN-SERVER/hahaha.txt b/FILMAN-SERVER/old_way/hahaha.txt similarity index 100% rename from FILMAN-SERVER/hahaha.txt rename to FILMAN-SERVER/old_way/hahaha.txt diff --git a/FILMAN-SERVER/movie.py b/FILMAN-SERVER/old_way/movie.py similarity index 100% rename from FILMAN-SERVER/movie.py rename to FILMAN-SERVER/old_way/movie.py diff --git a/FILMAN-SERVER/requirements.txt b/FILMAN-SERVER/old_way/requirements.txt similarity index 100% rename from FILMAN-SERVER/requirements.txt rename to FILMAN-SERVER/old_way/requirements.txt diff --git a/FILMAN-SERVER/series.py b/FILMAN-SERVER/old_way/series.py similarity index 100% rename from FILMAN-SERVER/series.py rename to FILMAN-SERVER/old_way/series.py diff --git a/FILMAN-SERVER/tasks.py b/FILMAN-SERVER/old_way/tasks.py similarity index 100% rename from FILMAN-SERVER/tasks.py rename to FILMAN-SERVER/old_way/tasks.py diff --git a/FILMAN-SERVER/users.py b/FILMAN-SERVER/old_way/users.py similarity index 100% rename from FILMAN-SERVER/users.py rename to FILMAN-SERVER/old_way/users.py diff --git a/FILMAN-SERVER/utils.py b/FILMAN-SERVER/old_way/utils.py similarity index 100% rename from FILMAN-SERVER/utils.py rename to FILMAN-SERVER/old_way/utils.py diff --git a/FILMAN-SERVER/watched.py b/FILMAN-SERVER/old_way/watched.py similarity index 100% rename from FILMAN-SERVER/watched.py rename to FILMAN-SERVER/old_way/watched.py diff --git a/FILMAN-SERVER/routes/discord.py b/FILMAN-SERVER/routes/discord.py new file mode 100644 index 0000000..26ba8b0 --- /dev/null +++ b/FILMAN-SERVER/routes/discord.py @@ -0,0 +1,31 @@ +from fastapi import APIRouter, Depends, FastAPI, HTTPException +from fastapi.responses import JSONResponse + +from sqlalchemy.orm import Session +from sqlalchemy.exc import IntegrityError + +from database import crud, models, schemas +from database.db import SessionLocal, engine + +from typing import Optional, List, Dict, Any + +discord_router = APIRouter(prefix="/discord", tags=["discord"]) + + +def get_db(): + db = SessionLocal() + try: + yield db + finally: + db.close() + + +@discord_router.post("/configure/guild", response_model=schemas.DiscordGuilds) +async def configure_guild( + guild: schemas.DiscordGuildsCreate, db: Session = Depends(get_db) +): + try: + db_guild = crud.set_guild(db, guild) + return db_guild + except IntegrityError: + raise HTTPException(status_code=400, detail="Guild already exists") diff --git a/FILMAN-SERVER/routes/filmweb.py b/FILMAN-SERVER/routes/filmweb.py new file mode 100644 index 0000000..0dbdfb1 --- /dev/null +++ b/FILMAN-SERVER/routes/filmweb.py @@ -0,0 +1,31 @@ +from fastapi import APIRouter, Depends, FastAPI, HTTPException +from fastapi.responses import JSONResponse + +from sqlalchemy.orm import Session +from sqlalchemy.exc import IntegrityError + +from database import crud, models, schemas +from database.db import SessionLocal, engine + +from typing import Optional, List, Dict, Any + +filmweb_router = APIRouter(prefix="/filmweb", tags=["filmweb"]) + + +def get_db(): + db = SessionLocal() + try: + yield db + finally: + db.close() + + +@filmweb_router.get("/get/movie", response_model=schemas.FilmWebMovie) +async def get_movie( + id: Optional[int] = None, + db: Session = Depends(get_db), +): + movie = crud.get_movie_filmweb_id(db, id) + if movie is None: + raise HTTPException(status_code=404, detail="Movie not found") + return movie diff --git a/FILMAN-SERVER/routes/users.py b/FILMAN-SERVER/routes/users.py new file mode 100644 index 0000000..e72b275 --- /dev/null +++ b/FILMAN-SERVER/routes/users.py @@ -0,0 +1,99 @@ +from fastapi import APIRouter, Depends, FastAPI, HTTPException +from fastapi.responses import JSONResponse + +from sqlalchemy.orm import Session +from sqlalchemy.exc import IntegrityError + +from database import crud, models, schemas +from database.db import SessionLocal, engine + +from typing import Optional, List, Dict, Any + +users_router = APIRouter(prefix="/users", tags=["users"]) + + +def get_db(): + db = SessionLocal() + try: + yield db + finally: + db.close() + + +@users_router.post("/create", response_model=schemas.User) +async def create_user(user: schemas.UserCreate, db: Session = Depends(get_db)): + try: + db_user = crud.create_user(db, user) + return db_user + except IntegrityError: + raise HTTPException(status_code=400, detail="User already exists") + + +@users_router.get("/get", response_model=schemas.User) +async def get_user( + id: Optional[int] = None, + filmweb_id: Optional[str] = None, + discord_id: Optional[int] = None, + db: Session = Depends(get_db), +): + user = crud.get_user(db, id, filmweb_id, discord_id) + if user is None: + raise HTTPException(status_code=404, detail="User not found") + return user + + +# +# discord guild actions +# + + +@users_router.get("/add_to_guild", response_model=schemas.DiscordDestinations) +async def add_to_guild( + user_id: int, + discord_guild_id: int, + db: Session = Depends(get_db), +): + guild = crud.get_guild(db, discord_guild_id) + if guild is None: + raise HTTPException(status_code=404, detail="Guild not found") + + discord_destination = crud.set_discord_destination(db, user_id, discord_guild_id) + + return discord_destination + + +@users_router.get("/remove_from_guild", response_model=schemas.DiscordDestinations) +async def remove_from_guild( + user_id: int, + discord_guild_id: int, + db: Session = Depends(get_db), +): + guild = crud.get_guild(db, discord_guild_id) + if guild is None: + raise HTTPException(status_code=404, detail="Guild not found") + + discord_destination = crud.get_user_destinations(db, user_id) + if discord_destination is None: + raise HTTPException(status_code=404, detail="User not found in any guild") + + discord_destination = crud.get_user_destination(db, user_id, discord_guild_id) + if discord_destination is None: + raise HTTPException(status_code=404, detail="User not found in this guild") + + crud.delete_user_destination(db, user_id, discord_guild_id) + + return discord_destination + + +@users_router.get("/remove_from_all_guilds", response_model=schemas.DiscordDestinations) +async def remove_from_all_guilds( + user_id: int, + db: Session = Depends(get_db), +): + discord_destination = crud.get_user_destinations(db, user_id) + if discord_destination is None: + raise HTTPException(status_code=404, detail="User not found in any guild") + + crud.delete_discord_destinations(db, user_id) + + return discord_destination From e9b86a138eebdce23a0a15f655e3e2d11d3b9c06 Mon Sep 17 00:00:00 2001 From: Jakub Sucheta Date: Wed, 20 Dec 2023 18:37:59 +0100 Subject: [PATCH 4/8] filmweb watched, working with tasks --- FILMAN-SERVER/database/crud.py | 68 ++++++++++++++++++++++-- FILMAN-SERVER/database/models.py | 17 ++++++ FILMAN-SERVER/database/schemas.py | 13 +++++ FILMAN-SERVER/main.py | 2 +- FILMAN-SERVER/routes/filmweb.py | 88 +++++++++++++++++++++++++++++++ 5 files changed, 184 insertions(+), 4 deletions(-) diff --git a/FILMAN-SERVER/database/crud.py b/FILMAN-SERVER/database/crud.py index 00ef09a..5298128 100644 --- a/FILMAN-SERVER/database/crud.py +++ b/FILMAN-SERVER/database/crud.py @@ -1,4 +1,5 @@ from sqlalchemy.orm import Session +import sqlalchemy.exc from . import models, schemas @@ -131,7 +132,7 @@ def get_movie_filmweb_id(db: Session, id: int): return db.query(models.FilmWebMovie).filter(models.FilmWebMovie.id == id).first() -def insert_movie(db: Session, movie: schemas.FilmWebMovie): +def create_filmweb_movie(db: Session, movie: schemas.FilmWebMovie): db_movie = models.FilmWebMovie(**movie.model_dump()) db.add(db_movie) db.commit() @@ -139,10 +140,10 @@ def insert_movie(db: Session, movie: schemas.FilmWebMovie): return db_movie -def update_movie(db: Session, movie: schemas.FilmWebMovie): +def update_filmweb_movie(db: Session, movie: schemas.FilmWebMovie): db_movie = get_movie_filmweb_id(db, movie.id) if db_movie is None: - return insert_movie(db, movie) + return create_filmweb_movie(db, movie) db_movie.title = movie.title db_movie.year = movie.year db_movie.poster_url = movie.poster_url @@ -150,3 +151,64 @@ def update_movie(db: Session, movie: schemas.FilmWebMovie): db.commit() db.refresh(db_movie) return db_movie + + +# +# FILMWEB WATCHED +# + +# MOVIES + + +def create_filmweb_user_watched_movie( + db: Session, user_watched_movie: schemas.FilmWebUserWatchedMovieCreate +): + watched_movie = get_movie_filmweb_id(db, user_watched_movie.id_media) + + if watched_movie is None: + watched_movie = schemas.FilmWebMovieCreate( + id=user_watched_movie.id_media, + title=None, + year=None, + poster_url=None, + community_rate=None, + ) + + create_filmweb_movie(db, watched_movie) + + db_movie_watched = ( + db.query(models.FilmWebUserWatchedMovie) + .filter( + models.FilmWebUserWatchedMovie.id_filmweb == user_watched_movie.id_filmweb, + models.FilmWebUserWatchedMovie.id_media == user_watched_movie.id_media, + ) + .first() + ) + + if db_movie_watched is not None: + return sqlalchemy.exc.IntegrityError( + "Movie already in user watched", None, None + ) + + db_movie = models.FilmWebUserWatchedMovie(**user_watched_movie.model_dump()) + db.add(db_movie) + db.commit() + db.refresh(db_movie) + return db_movie + + +def get_filmweb_user_watched_movies(db: Session, id_filmweb: str): + return ( + db.query(models.FilmWebUserWatchedMovie) + .filter(models.FilmWebUserWatchedMovie.id_filmweb == id_filmweb) + .all() + ) + + +def get_filmweb_user_watched_movies_ids(db: Session, id_filmweb: str): + result = ( + db.query(models.FilmWebUserWatchedMovie.id_media) # select only the id field + .filter(models.FilmWebUserWatchedMovie.id_filmweb == id_filmweb) + .all() + ) + return [id_media for (id_media,) in result] # extract ids from tuples diff --git a/FILMAN-SERVER/database/models.py b/FILMAN-SERVER/database/models.py index f9f8fb9..798d9e2 100644 --- a/FILMAN-SERVER/database/models.py +++ b/FILMAN-SERVER/database/models.py @@ -112,3 +112,20 @@ class FilmWebUserWatchedSeries(__FilmwebWatched): ) series = relationship("FilmWebSeries", backref="filmweb_user_watched_series") + + +# +# TASKS +# + + +class Task(Base): + __tablename__ = "tasks" + + task_id = Column(Integer, primary_key=True, index=True, autoincrement=True) + task_status = Column(String(16)) + task_type = Column(String(32)) + task_job = Column(String(256)) + task_created = Column(DateTime) + task_started = Column(DateTime) + task_finished = Column(DateTime) diff --git a/FILMAN-SERVER/database/schemas.py b/FILMAN-SERVER/database/schemas.py index 010eef9..c794959 100644 --- a/FILMAN-SERVER/database/schemas.py +++ b/FILMAN-SERVER/database/schemas.py @@ -153,3 +153,16 @@ class FilmwebUserWatchedSeriesCreate(BaseModel): class Config: orm_mode = True + +# +# TASKS +# + +class Task(BaseModel): + task_id: int + task_status: str + task_type: str + task_job: str + task_created: datetime + task_started: datetime | None + task_finished: datetime | None \ No newline at end of file diff --git a/FILMAN-SERVER/main.py b/FILMAN-SERVER/main.py index 1380e69..4ea70e9 100644 --- a/FILMAN-SERVER/main.py +++ b/FILMAN-SERVER/main.py @@ -41,4 +41,4 @@ async def trigger_error(): if __name__ == "__main__": - uvicorn.run(app, host="127.0.0.1", port=8000) + uvicorn.run(app, host="127.0.0.1", port=8001) diff --git a/FILMAN-SERVER/routes/filmweb.py b/FILMAN-SERVER/routes/filmweb.py index 0dbdfb1..ef0eb12 100644 --- a/FILMAN-SERVER/routes/filmweb.py +++ b/FILMAN-SERVER/routes/filmweb.py @@ -29,3 +29,91 @@ async def get_movie( if movie is None: raise HTTPException(status_code=404, detail="Movie not found") return movie + + +@filmweb_router.post("/add/movie", response_model=schemas.FilmWebMovie) +async def add_movie( + movie: schemas.FilmWebMovieCreate, + db: Session = Depends(get_db), +): + try: + db_movie = crud.create_filmweb_movie(db, movie) + return db_movie + except IntegrityError: + raise HTTPException(status_code=400, detail="Movie already exists") + + +@filmweb_router.post("/update/movie", response_model=schemas.FilmWebMovie) +async def update_movie( + movie: schemas.FilmWebMovie, + db: Session = Depends(get_db), +): + try: + db_movie = crud.update_filmweb_movie(db, movie) + return db_movie + except IntegrityError: + raise HTTPException(status_code=400, detail="Movie already exists") + + +@filmweb_router.post( + "/watched/movies/add", response_model=schemas.FilmWebUserWatchedMovieCreate +) +async def add_watched_movie( + user_watched_movie: schemas.FilmWebUserWatchedMovieCreate, + db: Session = Depends(get_db), +): + try: + db_movie = crud.create_filmweb_user_watched_movie(db, user_watched_movie) + + if db_movie is None or db_movie is IntegrityError: + raise HTTPException(status_code=404, detail="Movie not found") + + return db_movie + except IntegrityError: + raise HTTPException(status_code=400, detail="Movie already in user watched") + +@filmweb_router.get( + "/watched/movies/get/ids", response_model=List[int] +) +async def get_watched_movies_ids( + id: Optional[int] = None, + filmweb_id: Optional[str] = None, + discord_id: Optional[int] = None, + db: Session = Depends(get_db), +): + user = crud.get_user(db, id, filmweb_id, discord_id) + if user is None: + raise HTTPException(status_code=404, detail="User not found") + + filmweb_id = user.filmweb_id + + watched_movies = crud.get_filmweb_user_watched_movies_ids(db, filmweb_id) + + if watched_movies is None: + raise HTTPException(status_code=404, detail="User has no watched movies") + + return watched_movies + +@filmweb_router.get( + "/watched/movies/get", response_model=List[schemas.FilmWebUserWatchedMovie] +) +async def get_watched_movies( + id: Optional[int] = None, + filmweb_id: Optional[str] = None, + discord_id: Optional[int] = None, + db: Session = Depends(get_db), +): + user = crud.get_user(db, id, filmweb_id, discord_id) + if user is None: + raise HTTPException(status_code=404, detail="User not found") + + filmweb_id = user.filmweb_id + + watched_movies = crud.get_filmweb_user_watched_movies(db, filmweb_id) + + if watched_movies is None: + raise HTTPException(status_code=404, detail="User has no watched movies") + + print(watched_movies) + + return watched_movies From efbde2f5181277c14d979429305fd75d5e2fc30a Mon Sep 17 00:00:00 2001 From: Jakub Sucheta Date: Sat, 23 Dec 2023 15:16:18 +0100 Subject: [PATCH 5/8] tasks finished --- FILMAN-SERVER/database/crud.py | 63 ++++++++++++++++++++++++++++++ FILMAN-SERVER/database/models.py | 5 ++- FILMAN-SERVER/database/schemas.py | 43 +++++++++++++++++++-- FILMAN-SERVER/main.py | 5 +-- FILMAN-SERVER/routes/tasks.py | 64 +++++++++++++++++++++++++++++++ 5 files changed, 171 insertions(+), 9 deletions(-) create mode 100644 FILMAN-SERVER/routes/tasks.py diff --git a/FILMAN-SERVER/database/crud.py b/FILMAN-SERVER/database/crud.py index 5298128..db7e179 100644 --- a/FILMAN-SERVER/database/crud.py +++ b/FILMAN-SERVER/database/crud.py @@ -1,6 +1,8 @@ from sqlalchemy.orm import Session import sqlalchemy.exc +from datetime import datetime + from . import models, schemas # @@ -212,3 +214,64 @@ def get_filmweb_user_watched_movies_ids(db: Session, id_filmweb: str): .all() ) return [id_media for (id_media,) in result] # extract ids from tuples + + +# +# TASKS +# + + +def create_task(db: Session, task: schemas.TaskCreate): + db_task = models.Task(**task.model_dump()) + db.add(db_task) + db.commit() + db.refresh(db_task) + return db_task + + +def __change_task_status(db: Session, task_id: int, task_status: schemas.TaskStatus): + db_task = db.query(models.Task).filter(models.Task.task_id == task_id).first() + if db_task is None: + return None + + db_task.task_status = task_status + db_task.task_started = ( + datetime.now() if task_status == schemas.TaskStatus.RUNNING else None + ) + db_task.task_finished = ( + datetime.now() if task_status == schemas.TaskStatus.COMPLETED else None + ) + + db.commit() + db.refresh(db_task) + return db_task + + +def update_task_status(db: Session, task_id: int, task_status: schemas.TaskStatus): + return __change_task_status(db, task_id, task_status) + + +def get_task_to_do(db: Session, task_types: schemas.TaskTypes, head: bool = False): + task_to_do = ( + db.query(models.Task) + .filter( + models.Task.task_status == schemas.TaskStatus.QUEUED, + models.Task.task_type.in_(task_types), + ) + .first() + ) + + if task_to_do is None: + return None + + if head: + return task_to_do + else: + return __change_task_status(db, task_to_do.task_id, schemas.TaskStatus.RUNNING) + + +def remove_completed_tasks(db: Session): + db.query(models.Task).filter( + models.Task.task_status == schemas.TaskStatus.COMPLETED + ).delete() + db.commit() diff --git a/FILMAN-SERVER/database/models.py b/FILMAN-SERVER/database/models.py index 798d9e2..747ae8d 100644 --- a/FILMAN-SERVER/database/models.py +++ b/FILMAN-SERVER/database/models.py @@ -123,9 +123,10 @@ class Task(Base): __tablename__ = "tasks" task_id = Column(Integer, primary_key=True, index=True, autoincrement=True) - task_status = Column(String(16)) - task_type = Column(String(32)) + task_status = Column(String(32)) + task_type = Column(String(64)) task_job = Column(String(256)) task_created = Column(DateTime) task_started = Column(DateTime) task_finished = Column(DateTime) + diff --git a/FILMAN-SERVER/database/schemas.py b/FILMAN-SERVER/database/schemas.py index c794959..0d6430f 100644 --- a/FILMAN-SERVER/database/schemas.py +++ b/FILMAN-SERVER/database/schemas.py @@ -1,5 +1,8 @@ from pydantic import BaseModel from datetime import datetime +from enum import Enum + +from typing import Optional, List, Dict, Any # # USER @@ -154,15 +157,47 @@ class FilmwebUserWatchedSeriesCreate(BaseModel): class Config: orm_mode = True + # # TASKS # + +class TaskTypes(str, Enum): + SCRAP_USER = "scrap_user" + SCRAP_FILMWEB_MOVIE = "scrap_filmweb_movie" + SCRAP_FILMWEB_SERIES = "scrap_filmweb_series" + SCRAP_FILMWEB_USER_WATCHED_MOVIES = "scrap_filmweb_user_watched_movies" + SCRAP_FILMWEB_USER_WATCHED_SERIES = "scrap_filmweb_user_watched_series" + SEND_DISCORD_NOTIFICATION = "send_discord_notification" + + +class TaskStatus(str, Enum): + QUEUED = "queued" + RUNNING = "running" + COMPLETED = "completed" + ERROR = "error" + + class Task(BaseModel): task_id: int - task_status: str - task_type: str + task_status: TaskStatus + task_type: TaskTypes task_job: str task_created: datetime - task_started: datetime | None - task_finished: datetime | None \ No newline at end of file + task_started: Optional[datetime] = None + task_finished: Optional[datetime] = None + + class Config: + orm_mode = True + +class TaskCreate(BaseModel): + task_status: TaskStatus + task_type: TaskTypes + task_job: str + task_created: Optional[datetime] = None + task_started: Optional[datetime] = None + task_finished: Optional[datetime] = None + + class Config: + orm_mode = True \ No newline at end of file diff --git a/FILMAN-SERVER/main.py b/FILMAN-SERVER/main.py index 4ea70e9..ececd0c 100644 --- a/FILMAN-SERVER/main.py +++ b/FILMAN-SERVER/main.py @@ -13,9 +13,7 @@ from database import crud, models, schemas from database.db import SessionLocal, engine -from routes import users -from routes import discord -from routes import filmweb +from routes import users, discord, filmweb, tasks models.Base.metadata.create_all(bind=engine) @@ -33,6 +31,7 @@ app.include_router(users.users_router) app.include_router(discord.discord_router) app.include_router(filmweb.filmweb_router) +app.include_router(tasks.tasks_router) @app.get("/sentry-debug") diff --git a/FILMAN-SERVER/routes/tasks.py b/FILMAN-SERVER/routes/tasks.py new file mode 100644 index 0000000..c827c0a --- /dev/null +++ b/FILMAN-SERVER/routes/tasks.py @@ -0,0 +1,64 @@ +from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi.responses import JSONResponse + +from datetime import datetime + +from sqlalchemy.orm import Session +from sqlalchemy.exc import IntegrityError + +from database import crud, models, schemas +from database.db import SessionLocal, engine + +from typing import Optional, List, Dict, Any + +tasks_router = APIRouter(prefix="/tasks", tags=["tasks"]) + + +def get_db(): + db = SessionLocal() + try: + yield db + finally: + db.close() + + +@tasks_router.post("/create", response_model=schemas.Task) +def create_task(task: schemas.TaskCreate, db: Session = Depends(get_db)): + task.task_created = datetime.now() + task.task_started = None + task.task_finished = None + + db_task = crud.create_task(db, task) + return db_task + + +@tasks_router.head("/get/task/to_do") +def get_task_to_do_head( + task_types: List[schemas.TaskTypes] = Query(...), db: Session = Depends(get_db) +): + db_task = crud.get_task_to_do(db, task_types, head=True) + if db_task is None: + raise HTTPException(status_code=404, detail="Task not found") + return JSONResponse(content={"task_id": db_task.task_id}) + + +@tasks_router.get("/get/task/to_do", response_model=schemas.Task) +def get_task_to_do( + task_types: List[schemas.TaskTypes] = Query(...), db: Session = Depends(get_db) +): + db_task = crud.get_task_to_do(db, task_types, head=False) + if db_task is None: + raise HTTPException(status_code=404, detail="Task not found") + return db_task + + +@tasks_router.get( + "/update/task/status/{task_id}/{task_status}", response_model=schemas.Task +) +def update_task_status( + task_id: int, task_status: schemas.TaskStatus, db: Session = Depends(get_db) +): + db_task = crud.update_task_status(db, task_id, task_status) + if db_task is None: + raise HTTPException(status_code=404, detail="Task not found") + return db_task From 4351b422675577d0e073a32c1cc5af2651ee29a8 Mon Sep 17 00:00:00 2001 From: suchencjusz Date: Sun, 24 Dec 2023 12:56:53 +0100 Subject: [PATCH 6/8] Create blacklint.yml --- .github/workflows/blacklint.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/workflows/blacklint.yml diff --git a/.github/workflows/blacklint.yml b/.github/workflows/blacklint.yml new file mode 100644 index 0000000..a3bae27 --- /dev/null +++ b/.github/workflows/blacklint.yml @@ -0,0 +1,14 @@ +name: 🎞 Lint + +on: [push, pull_request] + +jobs: + python-black: + name: Python Black + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Python Black + uses: cytopia/docker-black@0.8 + with: + path: '.' From a6a6eae4cf9bd704bb2ec97268061e110ef9154b Mon Sep 17 00:00:00 2001 From: Jakub Sucheta Date: Mon, 25 Dec 2023 19:32:29 +0100 Subject: [PATCH 7/8] tasks done, films i think too, series not tested --- FILMAN-CRAWLER/main.py | 190 ++++++------ FILMAN-CRAWLER/tasks/scrap_movie.py | 80 +++-- FILMAN-CRAWLER/tasks/scrap_series.py | 32 +- .../tasks/scrap_user_watched_movies.py | 28 +- FILMAN-CRAWLER/tasks/utils.py | 84 ++++++ FILMAN-SERVER/database/crud.py | 25 ++ FILMAN-SERVER/database/models.py | 2 + FILMAN-SERVER/database/schemas.py | 63 +++- FILMAN-SERVER/old_way/discord_m.py | 192 ------------ FILMAN-SERVER/old_way/movie.py | 83 ------ FILMAN-SERVER/old_way/series.py | 89 ------ FILMAN-SERVER/old_way/tasks.py | 280 ------------------ FILMAN-SERVER/old_way/users.py | 115 ------- FILMAN-SERVER/old_way/utils.py | 2 - FILMAN-SERVER/old_way/watched.py | 224 -------------- FILMAN-SERVER/routes/filmweb.py | 44 +++ main.py | 30 +- old_way/discord_m.py | 192 ++++++++++++ {FILMAN-SERVER/old_way => old_way}/hahaha.txt | 0 old_way/movie.py | 83 ++++++ .../old_way => old_way}/requirements.txt | Bin old_way/series.py | 89 ++++++ old_way/tasks.py | 280 ++++++++++++++++++ old_way/users.py | 115 +++++++ old_way/utils.py | 2 + old_way/watched.py | 224 ++++++++++++++ 26 files changed, 1358 insertions(+), 1190 deletions(-) create mode 100644 FILMAN-CRAWLER/tasks/utils.py delete mode 100644 FILMAN-SERVER/old_way/discord_m.py delete mode 100644 FILMAN-SERVER/old_way/movie.py delete mode 100644 FILMAN-SERVER/old_way/series.py delete mode 100644 FILMAN-SERVER/old_way/tasks.py delete mode 100644 FILMAN-SERVER/old_way/users.py delete mode 100644 FILMAN-SERVER/old_way/utils.py delete mode 100644 FILMAN-SERVER/old_way/watched.py create mode 100644 old_way/discord_m.py rename {FILMAN-SERVER/old_way => old_way}/hahaha.txt (100%) create mode 100644 old_way/movie.py rename {FILMAN-SERVER/old_way => old_way}/requirements.txt (100%) create mode 100644 old_way/series.py create mode 100644 old_way/tasks.py create mode 100644 old_way/users.py create mode 100644 old_way/utils.py create mode 100644 old_way/watched.py diff --git a/FILMAN-CRAWLER/main.py b/FILMAN-CRAWLER/main.py index a3b6a30..969dc03 100644 --- a/FILMAN-CRAWLER/main.py +++ b/FILMAN-CRAWLER/main.py @@ -6,6 +6,12 @@ from fake_useragent import UserAgent +from enum import Enum +from pydantic import BaseModel +from datetime import datetime +from typing import Optional, List, Dict, Any + + from tasks.scrap_movie import Scraper as movie_scrapper from tasks.scrap_series import Scraper as series_scrapper from tasks.scrap_user_watched_movies import Scraper as user_watched_movies_scrapper @@ -41,8 +47,7 @@ logger.addHandler(c_handler) logger.addHandler(f_handler) -CORE_ENDPOINT = os.environ.get("CORE_ENDPOINT", "http://localhost:8000") -# CORE_ENDPOINT = "http://localhost:8000" +CORE_ENDPOINT = os.environ.get("CORE_ENDPOINT", "http://localhost:8001") HEADERS = { "User-Agent": UserAgent().random, @@ -65,122 +70,121 @@ "TE": "trailers", } -# status: waiting, in_progress, done, failed -# type: scrap_movie, scrape_series, -send_discord-, check_user_new_movies, check_user_new_series, + +class TaskTypes(str, Enum): + SCRAP_USER = "scrap_user" + SCRAP_FILMWEB_MOVIE = "scrap_filmweb_movie" + SCRAP_FILMWEB_SERIES = "scrap_filmweb_series" + SCRAP_FILMWEB_USER_WATCHED_MOVIES = "scrap_filmweb_user_watched_movies" + SCRAP_FILMWEB_USER_WATCHED_SERIES = "scrap_filmweb_user_watched_series" + SEND_DISCORD_NOTIFICATION = "send_discord_notification" + + +class TaskStatus(str, Enum): + QUEUED = "queued" + RUNNING = "running" + COMPLETED = "completed" + ERROR = "error" -class Task: - def __init__( - self, id_task, status, type, job, unix_timestamp, unix_timestamp_last_update - ): - self.id_task = id_task - self.status = status - self.type = type - self.job = job - self.unix_timestamp = unix_timestamp - self.unix_timestamp_last_update = unix_timestamp_last_update +class Task(BaseModel): + task_id: int + task_status: TaskStatus + task_type: TaskTypes + task_job: str + task_created: datetime + task_started: Optional[datetime] = None + task_finished: Optional[datetime] = None - def __str__(self): - return ( - f"{self.id_task} {self.status} {self.type} {self.job} {self.unix_timestamp}" + +ALLOWED_TASKS = [ + TaskTypes.SCRAP_FILMWEB_MOVIE, + TaskTypes.SCRAP_FILMWEB_SERIES, + TaskTypes.SCRAP_FILMWEB_USER_WATCHED_MOVIES, + TaskTypes.SCRAP_FILMWEB_USER_WATCHED_SERIES, +] + +TASK_TYPES = [task for task in ALLOWED_TASKS] + + +def check_there_are_any_tasks(): + try: + r = requests.head( + f"{CORE_ENDPOINT}/tasks/get/task/to_do", + params={"task_types": TASK_TYPES}, + headers=HEADERS, ) + if r.status_code != 200: + return False -def fetch_tasks_from_endpoint(): - # r = requests.get( - # f'{CORE_ENDPOINT}/tasks/get?status=waiting&types=["scrap_movie","check_user_new_movies"]' - # ) + return True - allowed_tasks = [ - "scrap_movie", - "scrap_series", - "check_user_new_movies", - "check_user_new_series", - ] + except Exception as e: + logging.error(f"Error checking tasks: {e}") + return False +def get_task_to_do() -> Task: try: - r = requests.post( - f"{CORE_ENDPOINT}/tasks/get", - json={ - "status": "waiting", - "types": allowed_tasks, - }, + r = requests.get( + f"{CORE_ENDPOINT}/tasks/get/task/to_do", + params={"task_types": TASK_TYPES}, + headers=HEADERS, ) + + if r.status_code != 200: + return None + + return Task(**r.json()) + except Exception as e: - logging.error(f"Error fetching tasks: {e}") + logging.error(f"Error getting task to do: {e}") return None - if r.status_code != 200: - logging.error(f"Error fetching tasks: HTTP {r.status_code}") - return None - return ujson.loads(r.text) +def do_task(task: Task): + if task.task_type == TaskTypes.SCRAP_FILMWEB_MOVIE: + scraper = movie_scrapper(headers=HEADERS, endpoint_url=CORE_ENDPOINT, movie_id=task.task_job) + scraper.scrap(task) + + elif task.task_type == TaskTypes.SCRAP_FILMWEB_SERIES: + scraper = series_scrapper(headers=HEADERS, endpoint_url=CORE_ENDPOINT, series_id=task.task_job) + scraper.scrap(task) + + elif task.task_type == TaskTypes.SCRAP_FILMWEB_USER_WATCHED_MOVIES: + scraper = user_watched_movies_scrapper(headers=HEADERS, endpoint_url=CORE_ENDPOINT, user_id=task.task_job) + scraper.scrap(task) + elif task.task_type == TaskTypes.SCRAP_FILMWEB_USER_WATCHED_SERIES: + scraper = user_watched_series_scrapper(headers=HEADERS, endpoint_url=CORE_ENDPOINT, user_id=task.task_job) + scraper.scrap(task) + + else: + logging.error(f"Unknown task type: {task.task_type}") + + + def main(): logging.info("Program started") - min_wait = 1 # Minimum wait time in seconds + min_wait = 3 # Minimum wait time in seconds max_wait = 60 # Maximum wait time in seconds wait_time = min_wait with ThreadPoolExecutor() as executor: while True: logging.info("Fetching tasks from endpoint") - tasks = fetch_tasks_from_endpoint() - - if tasks: - logging.info(f"Found {len(tasks)} tasks") - wait_time = min_wait - - for task in tasks: - task = Task( - task["id_task"], - task["status"], - task["type"], - task["job"], - task["unix_timestamp"], - task["unix_timestamp_last_update"], - ) - - if task.type == "scrap_movie": - m_scraper = movie_scrapper( - headers=HEADERS, - movie_id=task.job, - endpoint_url=CORE_ENDPOINT, - ) - executor.submit(m_scraper.scrap, task) - - if task.type == "scrap_series": - s_scraper = series_scrapper( - headers=HEADERS, - series_id=task.job, - endpoint_url=CORE_ENDPOINT, - ) - executor.submit(s_scraper.scrap, task) - - if task.type == "check_user_new_movies": - m_scraper = user_watched_movies_scrapper( - headers=HEADERS, - movie_id=task.job, - endpoint_url=CORE_ENDPOINT, - ) - executor.submit(m_scraper.scrap, task) - - if task.type == "check_user_new_series": - s_scraper = user_watched_series_scrapper( - headers=HEADERS, - series_id=task.job, - endpoint_url=CORE_ENDPOINT, - ) - executor.submit(s_scraper.scrap, task) - - else: - logging.info("No tasks found") - wait_time = min(wait_time * 2, max_wait) - - logging.info(f"Waiting for {wait_time} seconds") - time.sleep(wait_time) + + if check_there_are_any_tasks(): + task = get_task_to_do() + + if task is not None: + executor.submit(do_task, task) + wait_time = min_wait + else: + logging.info("No tasks found") + wait_time = min_wait if __name__ == "__main__": diff --git a/FILMAN-CRAWLER/tasks/scrap_movie.py b/FILMAN-CRAWLER/tasks/scrap_movie.py index f57797d..3e73ef2 100644 --- a/FILMAN-CRAWLER/tasks/scrap_movie.py +++ b/FILMAN-CRAWLER/tasks/scrap_movie.py @@ -2,6 +2,8 @@ import requests import ujson +from .utils import Task, TaskTypes, TaskStatus +from .utils import Tasks, FilmWeb logging.basicConfig( format="%(asctime)s %(levelname)-8s %(message)s", @@ -10,23 +12,6 @@ ) -class Task: - def __init__( - self, id_task, status, type, job, unix_timestamp, unix_timestamp_last_update - ): - self.id_task = id_task - self.status = status - self.type = type - self.job = job - self.unix_timestamp = unix_timestamp - self.unix_timestamp_last_update = unix_timestamp_last_update - - def __str__(self): - return ( - f"{self.id_task} {self.status} {self.type} {self.job} {self.unix_timestamp}" - ) - - class Scraper: def __init__(self, headers=None, movie_id=None, endpoint_url=None): self.headers = headers @@ -40,9 +25,9 @@ def fetch(self, url): return None return response.text - def scrap(self, task): - info_url = f"https://www.filmweb.pl/api/v1/title/{task.job}/info" - rating_url = f"https://www.filmweb.pl/api/v1/film/{task.job}/rating" + def scrap(self, task: Task): + info_url = f"https://www.filmweb.pl/api/v1/title/{task.task_job}/info" + rating_url = f"https://www.filmweb.pl/api/v1/film/{task.task_job}/rating" info_data = self.fetch(info_url) rating_data = self.fetch(rating_url) @@ -55,40 +40,41 @@ def scrap(self, task): title = info_data.get("title", None) year = int(info_data.get("year", None)) - poster_url = info_data.get("posterPath", "https://vectorified.com/images/no-data-icon-23.png") + poster_url = info_data.get( + "posterPath", "https://vectorified.com/images/no-data-icon-23.png" + ) community_rate = rating_data.get("rate", None) if title is None or year is None or poster_url is None: return False - return self.update_data( - self.movie_id, title, year, poster_url, community_rate, task.id_task - ) - - def update_data(self, movie_id, title, year, poster_url, community_rate, id_task): - r = requests.post( - f"{self.endpoint_url}/movie/update", - headers=self.headers, - json={ - "id": int(movie_id), - "title": str(title), - "year": int(year), - "poster_uri": str(poster_url), - "community_rate": float(community_rate), - }, - ) - - if r.status_code != 200: - logging.error(f"Error updating movie data: HTTP {r.status_code}") - logging.error(r.text) - return False - - r = requests.get( - f"{self.endpoint_url}/task/update?id_task={id_task}&status=done", - headers=self.headers, + update = self.update_data( + self.movie_id, title, year, poster_url, community_rate, task.task_id ) - if r.status_code != 200: + if update is True: + logging.info(f"Updated movie {title} ({year})") + + return update + + def update_data(self, movie_id, title, year, poster_url, community_rate, task_id): + try: + filmweb = FilmWeb(self.headers, self.endpoint_url) + filmweb.update_movie( + FilmWeb.FilmWebMovie( + id=movie_id, + title=title, + year=year, + poster_url=poster_url, + community_rate=community_rate, + ) + ) + + tasks = Tasks(self.headers, self.endpoint_url) + tasks.update_task_status(task_id, TaskStatus.COMPLETED) + + except Exception as e: + logging.error(f"Error updating movie data: {e}") return False return True diff --git a/FILMAN-CRAWLER/tasks/scrap_series.py b/FILMAN-CRAWLER/tasks/scrap_series.py index a4f90c5..2c0652f 100644 --- a/FILMAN-CRAWLER/tasks/scrap_series.py +++ b/FILMAN-CRAWLER/tasks/scrap_series.py @@ -68,28 +68,28 @@ def scrap(self, task): def update_data(self, series_id, title, year, other_year, poster_url, community_rate, id_task): - r = requests.post( - f"{self.endpoint_url}/series/update", - headers=self.headers, - json={ - "id": int(series_id), - "title": str(title), - "year": int(year), - "other_year": int(other_year), - "poster_uri": str(poster_url), - "community_rate": float(community_rate), - }, - ) + # r = requests.post( + # f"{self.endpoint_url}/series/update", + # headers=self.headers, + # json={ + # "id": int(series_id), + # "title": str(title), + # "year": int(year), + # "other_year": int(other_year), + # "poster_uri": str(poster_url), + # "community_rate": float(community_rate), + # }, + # ) if r.status_code != 200: logging.error(f"Error updating series data: HTTP {r.status_code}") logging.error(r.text) return False - r = requests.get( - f"{self.endpoint_url}/task/update?id_task={id_task}&status=done", - headers=self.headers, - ) + # r = requests.get( + # f"{self.endpoint_url}/task/update?id_task={id_task}&status=done", + # headers=self.headers, + # ) if r.status_code != 200: return False diff --git a/FILMAN-CRAWLER/tasks/scrap_user_watched_movies.py b/FILMAN-CRAWLER/tasks/scrap_user_watched_movies.py index d059b88..cc45829 100644 --- a/FILMAN-CRAWLER/tasks/scrap_user_watched_movies.py +++ b/FILMAN-CRAWLER/tasks/scrap_user_watched_movies.py @@ -73,9 +73,9 @@ def fetch(self, url): def scrap(self, task): last_100_watched = f"https://www.filmweb.pl/api/v1/user/{task.job}/vote/film" - user_already_watched = ( - f"{self.endpoint_url}/user/watched/film/all?id_filmweb={task.job}" - ) + # user_already_watched = ( + # f"{self.endpoint_url}/user/watched/film/all?id_filmweb={task.job}" + # ) last_100_watched_data = self.fetch(last_100_watched) user_already_watched_data = self.fetch(user_already_watched) @@ -148,14 +148,14 @@ def update_data( if len(movies_watched) == 0: return "No new movies" - r = requests.post( - f"{self.endpoint_url}/user/watched/film/add_many", - json={ - "id_filmweb": filmweb_id, - "movies": [movie.to_dict() for movie in movies_watched], - "without_discord": without_discord, - }, - ) + # r = requests.post( + # f"{self.endpoint_url}/user/watched/film/add_many", + # json={ + # "id_filmweb": filmweb_id, + # "movies": [movie.to_dict() for movie in movies_watched], + # "without_discord": without_discord, + # }, + # ) if r.status_code != 200: logging.error(f"Error adding movies: HTTP {r.status_code}") @@ -170,9 +170,9 @@ def update_data( return True def update_task_status_done(self, id_task: int): - r = requests.get( - f"{self.endpoint_url}/task/update?id_task={id_task}&status=done" - ) + # r = requests.get( + # f"{self.endpoint_url}/task/update?id_task={id_task}&status=done" + # ) if r.status_code != 200: return False diff --git a/FILMAN-CRAWLER/tasks/utils.py b/FILMAN-CRAWLER/tasks/utils.py new file mode 100644 index 0000000..0c0f764 --- /dev/null +++ b/FILMAN-CRAWLER/tasks/utils.py @@ -0,0 +1,84 @@ +import logging +import requests +import ujson + +from enum import Enum +from pydantic import BaseModel +from datetime import datetime +from typing import Optional, List, Dict, Any + + +class TaskTypes(str, Enum): + SCRAP_USER = "scrap_user" + SCRAP_FILMWEB_MOVIE = "scrap_filmweb_movie" + SCRAP_FILMWEB_SERIES = "scrap_filmweb_series" + SCRAP_FILMWEB_USER_WATCHED_MOVIES = "scrap_filmweb_user_watched_movies" + SCRAP_FILMWEB_USER_WATCHED_SERIES = "scrap_filmweb_user_watched_series" + SEND_DISCORD_NOTIFICATION = "send_discord_notification" + + +class TaskStatus(str, Enum): + QUEUED = "queued" + RUNNING = "running" + COMPLETED = "completed" + ERROR = "error" + + +class Task(BaseModel): + task_id: int + task_status: TaskStatus + task_type: TaskTypes + task_job: str + task_created: datetime + task_started: Optional[datetime] = None + task_finished: Optional[datetime] = None + + +class Updaters: + def __init__(self, headers, endpoint_url): + self.headers = headers + self.endpoint_url = endpoint_url + + +class Tasks(Updaters): + def update_task_status(self, task_id: int, task_status: TaskStatus): + r = requests.get( + f"{self.endpoint_url}/tasks/update/task/status/{task_id}/{task_status.value}", + headers=self.headers, + ) + + if r.status_code != 200: + logging.error(f"Error updating task status: HTTP {r.status_code}") + logging.error(r.text) + return False + + return True + + +class FilmWeb(Updaters): + class FilmWebMovie(BaseModel): + id: int + title: str + year: int + poster_url: str + community_rate: float + + def update_movie(self, movie: FilmWebMovie): + r = requests.post( + f"{self.endpoint_url}/filmweb/update/movie", + headers=self.headers, + json={ + "id": int(movie.id), + "title": str(movie.title), + "year": int(movie.year), + "poster_url": str(movie.poster_url), + "community_rate": float(movie.community_rate), + }, + ) + + if r.status_code != 200: + logging.error(f"Error updating movie data: HTTP {r.status_code}") + logging.error(r.text) + return False + + return True diff --git a/FILMAN-SERVER/database/crud.py b/FILMAN-SERVER/database/crud.py index db7e179..17b7909 100644 --- a/FILMAN-SERVER/database/crud.py +++ b/FILMAN-SERVER/database/crud.py @@ -154,7 +154,32 @@ def update_filmweb_movie(db: Session, movie: schemas.FilmWebMovie): db.refresh(db_movie) return db_movie +# +# FILMWEB SERIES +# + +def get_series_filmweb_id(db: Session, id: int): + return db.query(models.FilmWebSeries).filter(models.FilmWebSeries.id == id).first() +def create_filmweb_series(db: Session, series: schemas.FilmWebSeries): + db_series = models.FilmWebSeries(**series.model_dump()) + db.add(db_series) + db.commit() + db.refresh(db_series) + return db_series + +def update_filmweb_series(db: Session, series: schemas.FilmWebSeries): + db_series = get_series_filmweb_id(db, series.id) + if db_series is None: + return create_filmweb_series(db, series) + db_series.title = series.title + db_series.year = series.year + db_series.poster_url = series.poster_url + db_series.community_rate = series.community_rate + db_series.other_year = series.other_year + db.commit() + db.refresh(db_series) + return db_series # # FILMWEB WATCHED # diff --git a/FILMAN-SERVER/database/models.py b/FILMAN-SERVER/database/models.py index 747ae8d..bf71a64 100644 --- a/FILMAN-SERVER/database/models.py +++ b/FILMAN-SERVER/database/models.py @@ -78,6 +78,8 @@ class FilmWebMovie(__FilmwebMedia): class FilmWebSeries(__FilmwebMedia): __tablename__ = "filmweb_series" + other_year = Column(SmallInteger) + class __FilmwebWatched(Base): __abstract__ = True diff --git a/FILMAN-SERVER/database/schemas.py b/FILMAN-SERVER/database/schemas.py index 0d6430f..096773c 100644 --- a/FILMAN-SERVER/database/schemas.py +++ b/FILMAN-SERVER/database/schemas.py @@ -83,6 +83,8 @@ class Config: # FILMWEB # +# MOVIES + class FilmWebMovie(BaseModel): id: int @@ -132,21 +134,35 @@ class Config: orm_mode = True -class FilmwebUserWatchedSeries(BaseModel): - series: FilmWebMovie - id_filmweb: str +# SERIES - date: datetime - rate: int | None - comment: str | None - favorite: bool + +class FilmWebSeries(BaseModel): + id: int + title: str + year: int + other_year: int + poster_url: str + community_rate: float class Config: orm_mode = True -class FilmwebUserWatchedSeriesCreate(BaseModel): - id_media: int +class FilmWebSeriesCreate(BaseModel): + id: int + title: str | None + year: int | None + other_year: int | None + poster_url: str | None + community_rate: float | None + + class Config: + orm_mode = True + + +class FilmWebUserWatchedSeries(BaseModel): + series: FilmWebSeries id_filmweb: str date: datetime @@ -158,6 +174,32 @@ class Config: orm_mode = True +# class FilmwebUserWatchedSeries(BaseModel): +# series: FilmWebMovie +# id_filmweb: str + +# date: datetime +# rate: int | None +# comment: str | None +# favorite: bool + +# class Config: +# orm_mode = True + + +# class FilmwebUserWatchedSeriesCreate(BaseModel): +# id_media: int +# id_filmweb: str + +# date: datetime +# rate: int | None +# comment: str | None +# favorite: bool + +# class Config: +# orm_mode = True + + # # TASKS # @@ -191,6 +233,7 @@ class Task(BaseModel): class Config: orm_mode = True + class TaskCreate(BaseModel): task_status: TaskStatus task_type: TaskTypes @@ -200,4 +243,4 @@ class TaskCreate(BaseModel): task_finished: Optional[datetime] = None class Config: - orm_mode = True \ No newline at end of file + orm_mode = True diff --git a/FILMAN-SERVER/old_way/discord_m.py b/FILMAN-SERVER/old_way/discord_m.py deleted file mode 100644 index af7ae97..0000000 --- a/FILMAN-SERVER/old_way/discord_m.py +++ /dev/null @@ -1,192 +0,0 @@ -from db import Database - - -class DiscordUser: - def __init__(self, id_filmweb, id_discord, discord_color): - self.id_filmweb = id_filmweb - self.id_discord = id_discord - self.discord_color = discord_color - - def __str__(self): - return f"{self.id_filmweb} {self.id_discord} {self.discord_color}" - - -class DiscordGuild: - def __init__(self, id, guild_id, channel_id): - self.id = id - self.guild_id = guild_id - self.channel_id = channel_id - - def __str__(self): - return f"{self.id} {self.guild_id} {self.channel_id}" - - -class DiscordManager: - def __init__(self) -> None: - pass - - def check_user_is_in_db(self, id_discord: int): - db = Database() - - db.cursor.execute( - f"SELECT * FROM users WHERE id_discord = %s", - (id_discord,), - ) - - result = db.cursor.fetchone() - - db.connection.close() - - return result - - # def get_user_notification_destinations_by_id(self, id_discord: int): - # db = Database() - - # db.cursor.execute( - # f"SELECT discord_guilds.guild_id, discord_guilds.channel_id FROM discord_guilds, discord_destinations WHERE discord_guilds.guild_id = discord_destinations.guild_id AND discord_destinations.id_filmweb = (SELECT id_filmweb FROM users WHERE id_discord = %s)", - # (id_discord,), - # ) - - # result = db.cursor.fetchall() - - # db.connection.close() - - # return result - - def get_user_notification_destinations(self, id_filmweb: str): - db = Database() - - db.cursor.execute( - f"SELECT discord_guilds.guild_id, discord_guilds.channel_id FROM discord_guilds, discord_destinations WHERE discord_guilds.guild_id = discord_destinations.guild_id AND discord_destinations.id_filmweb = %s", - (id_filmweb,), - ) - - result = db.cursor.fetchall() - - db.connection.close() - - return result - - def get_user_guilds(self, id_filmweb: str): - db = Database() - - db.cursor.execute( - f"SELECT * FROM discord_destinations WHERE id_filmweb = %s", - (id_filmweb,), - ) - - result = db.cursor.fetchall() - - db.connection.close() - - return result - - def delete_user_from_db(self, id_discord: int): - db = Database() - - db.cursor.execute( - f"DELETE FROM users WHERE id_discord = %s", - (id_discord,), - ) - - db.connection.commit() - db.connection.close() - - return True - - def delete_user_from_all_guilds(self, id_filmweb: str): - db = Database() - - db.cursor.execute( - f"DELETE FROM discord_destinations WHERE id_filmweb = %s", - (id_filmweb,), - ) - - db.connection.commit() - db.connection.close() - - return True - - def delete_user_from_guild(self, id_discord: int, guild_id: int): - db = Database() - - db.cursor.execute( - "DELETE FROM discord_destinations WHERE id_filmweb = (SELECT id_filmweb FROM users WHERE id_discord = %s) AND guild_id = %s", - (id_discord, guild_id), - ) - - db.connection.commit() - db.connection.close() - - return True - - def delete_user_from_all_destinations(self, id_discord: int): - db = Database() - - db.cursor.execute( - "DELETE FROM discord_destinations WHERE id_filmweb = (SELECT id_filmweb FROM users WHERE id_discord = %s)", - (id_discord,), - ) - - db.connection.commit() - db.connection.close() - - self.delete_user_from_db(id_discord) - - return True - - def add_user_to_guild(self, id_discord: int, guild_id: int): - if self.check_user_is_in_db(id_discord) is None: - return "User not found in database" - - id_discord = int(id_discord) - guild_id = int(guild_id) - - db = Database() - - db.cursor.execute( - f"SELECT * FROM discord_destinations WHERE id_filmweb = (SELECT id_filmweb FROM users WHERE id_discord = %s) AND guild_id = %s", - (id_discord, guild_id), - ) - - result = db.cursor.fetchone() - - if result is None: - try: - db.cursor.execute( - f"INSERT INTO discord_destinations (id_filmweb, guild_id) VALUES ((SELECT id_filmweb FROM users WHERE id_discord = %s), %s)", - (id_discord, guild_id), - ) - db.connection.commit() - except Exception: - return "Server not configured" - - db.connection.close() - - return True - - def configure_guild(self, guild_id: int, channel_id: int): - db = Database() - - db.cursor.execute( - f"SELECT * FROM discord_guilds WHERE guild_id = %s", - (guild_id,), - ) - - result = db.cursor.fetchone() - - if result is None: - db.cursor.execute( - f"INSERT INTO discord_guilds (guild_id, channel_id) VALUES (%s, %s)", - (guild_id, channel_id), - ) - else: - db.cursor.execute( - f"UPDATE discord_guilds SET channel_id = %s WHERE guild_id = %s", - (channel_id, guild_id), - ) - - db.connection.commit() - db.connection.close() - - return True diff --git a/FILMAN-SERVER/old_way/movie.py b/FILMAN-SERVER/old_way/movie.py deleted file mode 100644 index 9a10f7b..0000000 --- a/FILMAN-SERVER/old_way/movie.py +++ /dev/null @@ -1,83 +0,0 @@ -import time - -from db import Database -from tasks import TasksManager - - -class Movie: - def __init__(self, **kwargs): - self.id = kwargs.get("id") - self.title = kwargs.get("title") - self.year = kwargs.get("year") - self.poster_uri = kwargs.get("poster_uri") - self.community_rate = kwargs.get("community_rate") - - def __str__(self): - return f"{self.title} ({self.year})" - - -class MovieManager: - def __init__(self): - pass - - def add_movie_to_db(self, movie): - db = Database() - - db.cursor.execute(f"SELECT * FROM movies WHERE id = %s", (movie.id,)) - - result = db.cursor.fetchone() - - if result is None: - db.cursor.execute( - f"INSERT INTO movies (id, updated_unixtime, title, year, poster_uri, community_rate) VALUES (%s, %s, %s, %s, %s, %s)", - ( - movie.id, - int(time.time()), - movie.title, - movie.year, - movie.poster_uri, - movie.community_rate, - ), - ) - - db.connection.commit() - db.connection.close() - - return True - - db.cursor.execute( - f"UPDATE movies SET updated_unixtime = %s, title = %s, year = %s, poster_uri = %s, community_rate = %s WHERE id = %s", - ( - int(time.time()), - movie.title, - movie.year, - movie.poster_uri, - movie.community_rate, - movie.id, - ), - ) - - db.connection.commit() - db.connection.close() - - return True - - def get_movie_by_id(self, id): - db = Database() - db.cursor.execute(f"SELECT * FROM movies WHERE id = {id}") - result = db.cursor.fetchone() - db.connection.close() - - if result is None: - task_manager = TasksManager() - task_manager.new_task("scrap_movie", id) - - return None - else: - return Movie( - id=result[0], - title=result[2], - year=result[3], - poster_uri=result[4], - community_rate=result[5], - ) diff --git a/FILMAN-SERVER/old_way/series.py b/FILMAN-SERVER/old_way/series.py deleted file mode 100644 index 0d1aaef..0000000 --- a/FILMAN-SERVER/old_way/series.py +++ /dev/null @@ -1,89 +0,0 @@ -import time - -from db import Database -from tasks import TasksManager - - -class Series: - def __init__(self, **kwargs): - self.id = kwargs.get("id") - self.title = kwargs.get("title") - self.year = kwargs.get("year") - self.other_year = kwargs.get("other_year") - self.poster_uri = kwargs.get("poster_uri") - self.community_rate = kwargs.get("community_rate") - - def __str__(self): - return f"{self.title} ({self.year})" - - -class SeriesManager: - def __init__(self): - pass - - def add_series_to_db(self, series): - db = Database() - - db.cursor.execute(f"SELECT * FROM series WHERE id = %s", (series.id,)) - - result = db.cursor.fetchone() - - series.other_year = 0 if series.other_year is None else series.other_year - - if result is None: - db.cursor.execute( - f"INSERT INTO series (id, updated_unixtime, title, year, other_year, poster_uri, community_rate) VALUES (%s, %s, %s, %s, %s, %s, %s)", - ( - series.id, - int(time.time()), - series.title, - series.year, - series.other_year, - series.poster_uri, - series.community_rate, - ), - ) - - db.connection.commit() - db.connection.close() - - return True - - db.cursor.execute( - f"UPDATE series SET updated_unixtime = %s, title = %s, year = %s, other_year = %s, poster_uri = %s, community_rate = %s WHERE id = %s", - ( - int(time.time()), - series.title, - series.year, - series.other_year, - series.poster_uri, - series.community_rate, - series.id, - ), - ) - - db.connection.commit() - db.connection.close() - - return True - - def get_series_by_id(self, id): - db = Database() - db.cursor.execute(f"SELECT * FROM series WHERE id = {id}") - - result = db.cursor.fetchone() - - if result is None: - task_manager = TasksManager() - task_manager.new_task("scrap_series", id) - - return None - else: - return Series( - id=result[0], - title=result[3], - year=result[4], - other_year=result[5], - poster_uri=result[6], - community_rate=result[7], - ) diff --git a/FILMAN-SERVER/old_way/tasks.py b/FILMAN-SERVER/old_way/tasks.py deleted file mode 100644 index a70aadc..0000000 --- a/FILMAN-SERVER/old_way/tasks.py +++ /dev/null @@ -1,280 +0,0 @@ -import time -from datetime import datetime, timedelta - -from users import UserManager -from db import Database - - -class Task: - def __init__( - self, id_task, status, type, job, unix_timestamp, unix_timestamp_last_update - ): - self.id_task = id_task - self.status = status - self.type = type - self.job = job - self.unix_timestamp = unix_timestamp - self.unix_timestamp_last_update = unix_timestamp_last_update - - def __str__(self): - return ( - f"{self.id_task} {self.status} {self.type} {self.job} {self.unix_timestamp}" - ) - - -# status: waiting, in_progress, done, failed -# type: scrap_movie, scrape_series, send_discord, check_user_new_movies, check_user_new_series - - -class TasksManager: - def __init__(self): - pass - - def update_stuck_tasks(self): - db = Database() - - # Get the current time - now = datetime.now() - - # Calculate the threshold for 'stuck' tasks - threshold = now - timedelta(minutes=5) # adjust as needed - - # Convert the threshold to a Unix timestamp - threshold_timestamp = int(threshold.timestamp()) - - # Get all tasks with a status of 'in_progress' and a unix_timestamp_last_update older than the threshold - db.cursor.execute( - f"UPDATE tasks SET status = %s WHERE status = %s AND unix_timestamp_last_update < %s", - ("waiting", "in_progress", threshold_timestamp), - ) - - db.connection.commit() - db.connection.close() - - return True - - def delete_old_tasks(self): - db = Database() - - # Get the current time - now = datetime.now() - - # Calculate the threshold for 'stuck' tasks - threshold = now - timedelta(minutes=30) - - # Convert the threshold to a Unix timestamp - threshold_timestamp = int(threshold.timestamp()) - - # Get all tasks with a status of 'in_progress' and a unix_timestamp_last_update older than the threshold - db.cursor.execute( - f"DELETE FROM tasks WHERE unix_timestamp_last_update < %s", - (threshold_timestamp,), - ) - db.connection.commit() - db.connection.close() - - return True - - def update_task_status(self, id_task: int, status: str): - db = Database() - - if status == "done": - self.delete_task(id_task) - - db.cursor.execute( - f"UPDATE tasks SET status = %s WHERE id_task = %s", - (status, id_task), - ) - - db.connection.commit() - db.connection.close() - - return True - - def delete_task(self, id_task: int): - db = Database() - - db.cursor.execute( - f"DELETE FROM tasks WHERE id_task = %s", - (id_task,), - ) - - db.connection.commit() - db.connection.close() - - return True - - def add_scrap_users_task(self): - db = Database() - - user_manager = UserManager() - - all_users = user_manager.get_all_users_from_db() - - for user in all_users: - self.new_task("check_user_new_movies", user.id_filmweb) - self.new_task("check_user_new_series", user.id_filmweb) - - return True - - def add_scrap_movies_task(self): - db = Database() - - db.cursor.execute("SELECT id FROM movies") - result = db.cursor.fetchall() - - for movie in result: - self.new_task("scrap_movie", movie[0]) - - return True - - def add_scrap_series_task(self): - db = Database() - - db.cursor.execute("SELECT id FROM series") - result = db.cursor.fetchall() - - for series in result: - self.new_task("scrap_series", series[0]) - - return True - - def new_task(self, type: str, job: str): - # status: waiting, in_progress, done, failed - # type: scrap_movie, scrape_series, send_discord, check_user_new_movies, check_user_new_series, - - db = Database() - - unix_timestamp = int(time.time()) - - db.cursor.execute( - f"INSERT INTO tasks (status, type, job, unix_timestamp, unix_timestamp_last_update) VALUES (%s, %s, %s, %s, %s)", - ("waiting", type, job, unix_timestamp, unix_timestamp), - ) - - db.connection.commit() - db.connection.close() - - return True - - def get_and_update_tasks(self, types: list = None, status: str = "waiting"): - db = Database() - - types_placeholder = ", ".join(["%s"] * len(types)) - - db.cursor.execute( - f"SELECT * FROM tasks WHERE type IN ({types_placeholder}) AND status = %s ORDER BY id_task ASC LIMIT 5", - (*types, status), - ) - - result = db.cursor.fetchall() - - if result is None: - return None - - tasks = [] - - for task in result: - tasks.append( - Task( - id_task=task[0], - status=task[1], - type=task[2], - job=task[3], - unix_timestamp=task[4], - unix_timestamp_last_update=task[5], - ) - ) - - for task in tasks: - db.cursor.execute( - f"UPDATE tasks SET status = %s WHERE id_task = %s", - ("in_progress", task.id_task), - ) - - db.connection.commit() - db.connection.close() - - return tasks - - def get_tasks_count_by_type_and_status(self, type: str, status: str): - db = Database() - - db.cursor.execute( - f"SELECT COUNT(*) FROM tasks WHERE type = %s AND status = %s", - (type, status), - ) - - result = db.cursor.fetchone() - - db.connection.commit() - db.connection.close() - - return result[0] - - def get_task_by_type(self, type: str): - db = Database() - - if type is None: - db.cursor.execute( - f"SELECT * FROM tasks WHERE status = %s ORDER BY id_task ASC LIMIT 1", - ("waiting",), - ) - else: - db.cursor.execute( - f"SELECT * FROM tasks WHERE type = %s AND status = %s ORDER BY id_task ASC LIMIT 1", - (type, "waiting"), - ) - - result = db.cursor.fetchone() - - if result is None: - return None - - task = Task( - id_task=result[0], - status=result[1], - type=result[2], - job=result[3], - unix_timestamp=result[4], - unix_timestamp_last_update=result[5], - ) - - return task - - # db.cursor.execute( - # f"UPDATE tasks SET status = %s WHERE id_task = %s", - # ("in_progress", task.id_task), - # ) - - # db.connection.commit() - # db.connection.close() - - def reset_stuck_tasks(): - db = Database() - - # Get the current time - now = datetime.now() - - # Calculate the threshold for 'stuck' tasks - threshold = now - timedelta(minutes=5) # adjust as needed - - # Convert the threshold to a Unix timestamp - threshold_timestamp = int(threshold.timestamp()) - - # Get all tasks with a status of 'in_progress' and a unix_timestamp_last_update older than the threshold - db.cursor.execute( - f"SELECT * FROM tasks WHERE status = 'in_progress' AND unix_timestamp_last_update < %s", - (threshold_timestamp,), - ) - tasks = db.cursor.fetchall() - - for task in tasks: - # Update the status of the task back to 'in_progress' - db.cursor.execute( - f"UPDATE tasks SET status = 'in_progress' WHERE id_task = %s", - (task["id_task"],), - ) - - db.connection.commit() - db.connection.close() diff --git a/FILMAN-SERVER/old_way/users.py b/FILMAN-SERVER/old_way/users.py deleted file mode 100644 index 619c036..0000000 --- a/FILMAN-SERVER/old_way/users.py +++ /dev/null @@ -1,115 +0,0 @@ -from mysql.connector.errors import IntegrityError -import requests - -from fake_useragent import UserAgent - -from db import Database - - -class User: - def __init__(self, id_filmweb, id_discord, discord_color): - self.id_filmweb = id_filmweb - self.id_discord = id_discord - self.discord_color = discord_color - - def __str__(self): - return f"{self.id_filmweb} {self.id_discord} {self.discord_color}" - - -class UserManager: - def __init__(self): - pass - - def get_all_users_from_db(self): - db = Database() - db.cursor.execute(f"SELECT * FROM users") - result = db.cursor.fetchall() - db.connection.close() - - if result is None: - return None - - return list( - map( - lambda x: User( - id_filmweb=x[0], - id_discord=x[1], - discord_color=x[2], - ), - result, - ) - ) - - def check_user_has_filmweb_account(self, id_filmweb: str): - headers = { - "User-Agent": UserAgent().random, - } - - response = requests.head( - f"https://www.filmweb.pl/api/v1/user/{id_filmweb}/preview", headers=headers - ) - - if response.status_code == 200: - return True - - return False - - def create_user(self, id_filmweb: str, id_discord: int): - from tasks import TasksManager - - db = Database() - - if self.check_user_has_filmweb_account(id_filmweb) is False: - return "User does not exist on filmweb" - - try: - db.cursor.execute( - f"INSERT INTO users (id_filmweb, id_discord) VALUES (%s, %s)", - (id_filmweb, id_discord), - ) - - db.connection.commit() - except IntegrityError: - return "User already exists" - - db.connection.close() - - tasks_manager = TasksManager() - tasks_manager.new_task( - "check_user_new_movies", - f"{id_filmweb}", - ) - tasks_manager.new_task( - "check_user_new_series", - f"{id_filmweb}", - ) - - return True - - def delete_user(self, id_filmweb: str): - db = Database() - - db.cursor.execute( - f"DELETE FROM users WHERE id_filmweb = %s", - (id_filmweb,), - ) - - db.connection.commit() - db.connection.close() - - return True - - def get_user_by_filmweb_id(self, id_filmweb: str): - db = Database() - db.cursor.execute(f"SELECT * FROM users WHERE id_filmweb = %s", (id_filmweb,)) - result = db.cursor.fetchone() - db.connection.close() - - if result is None: - return None - - return User( - id_filmweb=result[0], - id_discord=result[1], - discord_color=result[2], - ) diff --git a/FILMAN-SERVER/old_way/utils.py b/FILMAN-SERVER/old_way/utils.py deleted file mode 100644 index 00a6b04..0000000 --- a/FILMAN-SERVER/old_way/utils.py +++ /dev/null @@ -1,2 +0,0 @@ -def cut_unix_timestamp_miliseconds(unix_timestamp: int) -> int: - return int(unix_timestamp / 1000) \ No newline at end of file diff --git a/FILMAN-SERVER/old_way/watched.py b/FILMAN-SERVER/old_way/watched.py deleted file mode 100644 index e7f91de..0000000 --- a/FILMAN-SERVER/old_way/watched.py +++ /dev/null @@ -1,224 +0,0 @@ -from db import Database -from movie import Movie, MovieManager -from series import Series, SeriesManager -from tasks import TasksManager -from users import UserManager -from discord_m import DiscordManager - - -class Watched: - def __init__( - self, id_watched, id_filmweb, movie_id, rate, comment, favorite, unix_timestamp - ): - self.id_watched = id_watched - self.id_filmweb = id_filmweb - self.movie_id = movie_id - self.rate = rate - self.comment = comment - self.favorite = favorite - self.unix_timestamp = unix_timestamp - - def __str__(self): - return f"{self.id_watched} {self.id_filmweb} {self.movie_id} {self.rate} {self.comment} {self.favourite} {self.unix_timestamp}" - - -class WatchedManager: - def __init__(self): - pass - - def update_movie_data( - self, - movie_id: int, - title: str, - year: int, - poster_path: str, - community_rate: float, - ): - movie_manager = MovieManager() - - movie = movie_manager.get_movie_by_id(movie_id) - - if movie is None: - return False - - movie.title = title - movie.year = year - movie.poster_path = poster_path - movie.community_rate = community_rate - - movie_manager.update_movie(movie) - - return True - - def add_watched_series( - self, - id_filmweb: str, - series_id: int, - rate: int, - comment: str, - favorite: bool, - unix_timestamp: int, - without_discord: bool, - ): - db = Database() - series_manager = SeriesManager() - - # check series is in db - db.cursor.execute(f"SELECT * FROM series WHERE id = %s", (series_id,)) - result = db.cursor.fetchone() - - if result is None: - # if not, create task to scrap series - - series_manager.add_series_to_db( - Series( - id=series_id, - title="", - year=0, - other_year=0, - poster_uri="", - community_rate=0, - ) - ) - - task_manager = TasksManager() - task_manager.new_task("scrap_series", series_id) - - # check it is not already in watched - db.cursor.execute( - f"SELECT * FROM watched_series WHERE id_filmweb = %s AND series_id = %s", - (id_filmweb, series_id), - ) - - result = db.cursor.fetchone() - - if result is not None: - print("Already watched") - return "Already watched" - - # add to watched - db.cursor.execute( - f"INSERT INTO watched_series (id_filmweb, series_id, rate, comment, favourite, unix_timestamp) VALUES (%s, %s, %s, %s, %s, %s)", - (id_filmweb, series_id, rate, comment, favorite, unix_timestamp), - ) - - db.connection.commit() - db.connection.close() - - if without_discord is True: - return True - - task_manager = TasksManager() - task_manager.new_task("send_discord", f"{id_filmweb},{series_id}") - - return True - - def add_watched_movie( - self, - id_filmweb: str, - movie_id: int, - rate: int, - comment: str, - favorite: bool, - unix_timestamp: int, - without_discord: bool, - ): - db = Database() - movie_manager = MovieManager() - - # check movie is in db - db.cursor.execute(f"SELECT * FROM movies WHERE id = %s", (movie_id,)) - result = db.cursor.fetchone() - - if result is None: - # if not, create task to scrap movie - - movie_manager.add_movie_to_db( - Movie( - id=movie_id, - title="", - year=0, - poster_uri="", - community_rate=0, - ) - ) - - task_manager = TasksManager() - task_manager.new_task("scrap_movie", movie_id) - - # check it is not already in watched - db.cursor.execute( - f"SELECT * FROM watched_movies WHERE id_filmweb = %s AND movie_id = %s", - (id_filmweb, movie_id), - ) - - result = db.cursor.fetchone() - - if result is not None: - print("Already watched") - return "Already watched" - - # add to watched - db.cursor.execute( - f"INSERT INTO watched_movies (id_filmweb, movie_id, rate, comment, favourite, unix_timestamp) VALUES (%s, %s, %s, %s, %s, %s)", - (id_filmweb, movie_id, rate, comment, favorite, unix_timestamp), - ) - db.connection.commit() - db.connection.close() - - if without_discord is True: - return True - - task_manager = TasksManager() - task_manager.new_task("send_discord", f"{id_filmweb},{movie_id}") - - return True - - def get_all_watched_movies(self, id_filmweb: str): - db = Database() - db.cursor.execute( - f"SELECT * FROM watched_movies WHERE id_filmweb = %s", (id_filmweb,) - ) - result = db.cursor.fetchall() - db.connection.close() - - if result is None: - return None - - if len(result) == 0: - return None - - watched_movies = [] - - for row in result: - watched_movies.append( - Watched(row[0], row[1], row[2], row[3], row[4], row[5], row[6]) - ) - - return watched_movies - - def get_watched_movie_with_rates(self, id_filmweb: str, movie_id: int): - db = Database() - db.cursor.execute( - f"SELECT * FROM watched_movies WHERE id_filmweb = %s AND movie_id = %s", - (id_filmweb, movie_id), - ) - - result = db.cursor.fetchone() - - w = Watched( - result[0], result[1], result[2], result[3], result[4], result[5], result[6] - ) - - db.connection.close() - - movie_manager = MovieManager() - user_manager = UserManager() - discord_manager = DiscordManager() - - return ( - w, - movie_manager.get_movie_by_id(movie_id), - user_manager.get_user_by_filmweb_id(id_filmweb), - discord_manager.get_user_notification_destinations(id_filmweb), - ) diff --git a/FILMAN-SERVER/routes/filmweb.py b/FILMAN-SERVER/routes/filmweb.py index ef0eb12..eded24b 100644 --- a/FILMAN-SERVER/routes/filmweb.py +++ b/FILMAN-SERVER/routes/filmweb.py @@ -19,6 +19,10 @@ def get_db(): finally: db.close() +# +# MOVIES +# + @filmweb_router.get("/get/movie", response_model=schemas.FilmWebMovie) async def get_movie( @@ -54,6 +58,10 @@ async def update_movie( except IntegrityError: raise HTTPException(status_code=400, detail="Movie already exists") +# +# MOVIES WATCHED +# + @filmweb_router.post( "/watched/movies/add", response_model=schemas.FilmWebUserWatchedMovieCreate @@ -117,3 +125,39 @@ async def get_watched_movies( print(watched_movies) return watched_movies + +# +# SERIES +# + +@filmweb_router.get("/get/series", response_model=schemas.FilmWebSeries) +async def get_series( + id: Optional[int] = None, + db: Session = Depends(get_db), +): + series = crud.get_series_filmweb_id(db, id) + if series is None: + raise HTTPException(status_code=404, detail="Series not found") + return series + +@filmweb_router.post("/add/series", response_model=schemas.FilmWebSeries) +async def add_series( + series: schemas.FilmWebSeriesCreate, + db: Session = Depends(get_db), +): + try: + db_series = crud.create_filmweb_series(db, series) + return db_series + except IntegrityError: + raise HTTPException(status_code=400, detail="Series already exists") + +@filmweb_router.post("/update/series", response_model=schemas.FilmWebSeries) +async def update_series( + series: schemas.FilmWebSeries, + db: Session = Depends(get_db), +): + try: + db_series = crud.update_filmweb_series(db, series) + return db_series + except IntegrityError: + raise HTTPException(status_code=400, detail="Series already exists") \ No newline at end of file diff --git a/main.py b/main.py index 2e076d0..6df3cdc 100644 --- a/main.py +++ b/main.py @@ -1,29 +1,9 @@ import requests -from fake_useragent import UserAgent - -HEADERS = { - "User-Agent": UserAgent().random, - "x-locale": "pl_PL", - "Host": "www.filmweb.pl", - "Accept": "*/*", - "Accept-Language": "en-US,en;q=0.5", - "Accept-Encoding": "gzip, deflate, br", - "Cache-Control": "no-cache", - "Pragma": "no-cache", - "If-Modified-Since": "0", - "Origin": "https://www.filmweb.pl", - "DNT": "1", - "Connection": "keep-alive", - "Upgrade-Insecure-Requests": "1", - "Sec-Fetch-Dest": "empty", - "Sec-Fetch-Mode": "cors", - "Sec-Fetch-Site": "same-origin", - "Sec-GPC": "1", - "TE": "trailers", +url = "http://localhost:8001/filmweb/add/movie" # replace with your actual server URL +data = { + "id": 123 # replace with your actual movie ID } +response = requests.post(url, json=data) -r = requests.get("https://www.filmweb.pl/api/v1/film/222032/info", headers=HEADERS) - -print(r.text) -print(r.status_code) \ No newline at end of file +print(response.json()) \ No newline at end of file diff --git a/old_way/discord_m.py b/old_way/discord_m.py new file mode 100644 index 0000000..7be3f01 --- /dev/null +++ b/old_way/discord_m.py @@ -0,0 +1,192 @@ +# from db import Database + + +# class DiscordUser: +# def __init__(self, id_filmweb, id_discord, discord_color): +# self.id_filmweb = id_filmweb +# self.id_discord = id_discord +# self.discord_color = discord_color + +# def __str__(self): +# return f"{self.id_filmweb} {self.id_discord} {self.discord_color}" + + +# class DiscordGuild: +# def __init__(self, id, guild_id, channel_id): +# self.id = id +# self.guild_id = guild_id +# self.channel_id = channel_id + +# def __str__(self): +# return f"{self.id} {self.guild_id} {self.channel_id}" + + +# class DiscordManager: +# def __init__(self) -> None: +# pass + +# def check_user_is_in_db(self, id_discord: int): +# db = Database() + +# db.cursor.execute( +# f"SELECT * FROM users WHERE id_discord = %s", +# (id_discord,), +# ) + +# result = db.cursor.fetchone() + +# db.connection.close() + +# return result + +# # def get_user_notification_destinations_by_id(self, id_discord: int): +# # db = Database() + +# # db.cursor.execute( +# # f"SELECT discord_guilds.guild_id, discord_guilds.channel_id FROM discord_guilds, discord_destinations WHERE discord_guilds.guild_id = discord_destinations.guild_id AND discord_destinations.id_filmweb = (SELECT id_filmweb FROM users WHERE id_discord = %s)", +# # (id_discord,), +# # ) + +# # result = db.cursor.fetchall() + +# # db.connection.close() + +# # return result + +# def get_user_notification_destinations(self, id_filmweb: str): +# db = Database() + +# db.cursor.execute( +# f"SELECT discord_guilds.guild_id, discord_guilds.channel_id FROM discord_guilds, discord_destinations WHERE discord_guilds.guild_id = discord_destinations.guild_id AND discord_destinations.id_filmweb = %s", +# (id_filmweb,), +# ) + +# result = db.cursor.fetchall() + +# db.connection.close() + +# return result + +# def get_user_guilds(self, id_filmweb: str): +# db = Database() + +# db.cursor.execute( +# f"SELECT * FROM discord_destinations WHERE id_filmweb = %s", +# (id_filmweb,), +# ) + +# result = db.cursor.fetchall() + +# db.connection.close() + +# return result + +# def delete_user_from_db(self, id_discord: int): +# db = Database() + +# db.cursor.execute( +# f"DELETE FROM users WHERE id_discord = %s", +# (id_discord,), +# ) + +# db.connection.commit() +# db.connection.close() + +# return True + +# def delete_user_from_all_guilds(self, id_filmweb: str): +# db = Database() + +# db.cursor.execute( +# f"DELETE FROM discord_destinations WHERE id_filmweb = %s", +# (id_filmweb,), +# ) + +# db.connection.commit() +# db.connection.close() + +# return True + +# def delete_user_from_guild(self, id_discord: int, guild_id: int): +# db = Database() + +# db.cursor.execute( +# "DELETE FROM discord_destinations WHERE id_filmweb = (SELECT id_filmweb FROM users WHERE id_discord = %s) AND guild_id = %s", +# (id_discord, guild_id), +# ) + +# db.connection.commit() +# db.connection.close() + +# return True + +# def delete_user_from_all_destinations(self, id_discord: int): +# db = Database() + +# db.cursor.execute( +# "DELETE FROM discord_destinations WHERE id_filmweb = (SELECT id_filmweb FROM users WHERE id_discord = %s)", +# (id_discord,), +# ) + +# db.connection.commit() +# db.connection.close() + +# self.delete_user_from_db(id_discord) + +# return True + +# def add_user_to_guild(self, id_discord: int, guild_id: int): +# if self.check_user_is_in_db(id_discord) is None: +# return "User not found in database" + +# id_discord = int(id_discord) +# guild_id = int(guild_id) + +# db = Database() + +# db.cursor.execute( +# f"SELECT * FROM discord_destinations WHERE id_filmweb = (SELECT id_filmweb FROM users WHERE id_discord = %s) AND guild_id = %s", +# (id_discord, guild_id), +# ) + +# result = db.cursor.fetchone() + +# if result is None: +# try: +# db.cursor.execute( +# f"INSERT INTO discord_destinations (id_filmweb, guild_id) VALUES ((SELECT id_filmweb FROM users WHERE id_discord = %s), %s)", +# (id_discord, guild_id), +# ) +# db.connection.commit() +# except Exception: +# return "Server not configured" + +# db.connection.close() + +# return True + +# def configure_guild(self, guild_id: int, channel_id: int): +# db = Database() + +# db.cursor.execute( +# f"SELECT * FROM discord_guilds WHERE guild_id = %s", +# (guild_id,), +# ) + +# result = db.cursor.fetchone() + +# if result is None: +# db.cursor.execute( +# f"INSERT INTO discord_guilds (guild_id, channel_id) VALUES (%s, %s)", +# (guild_id, channel_id), +# ) +# else: +# db.cursor.execute( +# f"UPDATE discord_guilds SET channel_id = %s WHERE guild_id = %s", +# (channel_id, guild_id), +# ) + +# db.connection.commit() +# db.connection.close() + +# return True diff --git a/FILMAN-SERVER/old_way/hahaha.txt b/old_way/hahaha.txt similarity index 100% rename from FILMAN-SERVER/old_way/hahaha.txt rename to old_way/hahaha.txt diff --git a/old_way/movie.py b/old_way/movie.py new file mode 100644 index 0000000..2760e86 --- /dev/null +++ b/old_way/movie.py @@ -0,0 +1,83 @@ +# import time + +# from db import Database +# from tasks import TasksManager + + +# class Movie: +# def __init__(self, **kwargs): +# self.id = kwargs.get("id") +# self.title = kwargs.get("title") +# self.year = kwargs.get("year") +# self.poster_uri = kwargs.get("poster_uri") +# self.community_rate = kwargs.get("community_rate") + +# def __str__(self): +# return f"{self.title} ({self.year})" + + +# class MovieManager: +# def __init__(self): +# pass + +# def add_movie_to_db(self, movie): +# db = Database() + +# db.cursor.execute(f"SELECT * FROM movies WHERE id = %s", (movie.id,)) + +# result = db.cursor.fetchone() + +# if result is None: +# db.cursor.execute( +# f"INSERT INTO movies (id, updated_unixtime, title, year, poster_uri, community_rate) VALUES (%s, %s, %s, %s, %s, %s)", +# ( +# movie.id, +# int(time.time()), +# movie.title, +# movie.year, +# movie.poster_uri, +# movie.community_rate, +# ), +# ) + +# db.connection.commit() +# db.connection.close() + +# return True + +# db.cursor.execute( +# f"UPDATE movies SET updated_unixtime = %s, title = %s, year = %s, poster_uri = %s, community_rate = %s WHERE id = %s", +# ( +# int(time.time()), +# movie.title, +# movie.year, +# movie.poster_uri, +# movie.community_rate, +# movie.id, +# ), +# ) + +# db.connection.commit() +# db.connection.close() + +# return True + +# def get_movie_by_id(self, id): +# db = Database() +# db.cursor.execute(f"SELECT * FROM movies WHERE id = {id}") +# result = db.cursor.fetchone() +# db.connection.close() + +# if result is None: +# task_manager = TasksManager() +# task_manager.new_task("scrap_movie", id) + +# return None +# else: +# return Movie( +# id=result[0], +# title=result[2], +# year=result[3], +# poster_uri=result[4], +# community_rate=result[5], +# ) diff --git a/FILMAN-SERVER/old_way/requirements.txt b/old_way/requirements.txt similarity index 100% rename from FILMAN-SERVER/old_way/requirements.txt rename to old_way/requirements.txt diff --git a/old_way/series.py b/old_way/series.py new file mode 100644 index 0000000..70cbae5 --- /dev/null +++ b/old_way/series.py @@ -0,0 +1,89 @@ +# import time + +# from db import Database +# from tasks import TasksManager + + +# class Series: +# def __init__(self, **kwargs): +# self.id = kwargs.get("id") +# self.title = kwargs.get("title") +# self.year = kwargs.get("year") +# self.other_year = kwargs.get("other_year") +# self.poster_uri = kwargs.get("poster_uri") +# self.community_rate = kwargs.get("community_rate") + +# def __str__(self): +# return f"{self.title} ({self.year})" + + +# class SeriesManager: +# def __init__(self): +# pass + +# def add_series_to_db(self, series): +# db = Database() + +# db.cursor.execute(f"SELECT * FROM series WHERE id = %s", (series.id,)) + +# result = db.cursor.fetchone() + +# series.other_year = 0 if series.other_year is None else series.other_year + +# if result is None: +# db.cursor.execute( +# f"INSERT INTO series (id, updated_unixtime, title, year, other_year, poster_uri, community_rate) VALUES (%s, %s, %s, %s, %s, %s, %s)", +# ( +# series.id, +# int(time.time()), +# series.title, +# series.year, +# series.other_year, +# series.poster_uri, +# series.community_rate, +# ), +# ) + +# db.connection.commit() +# db.connection.close() + +# return True + +# db.cursor.execute( +# f"UPDATE series SET updated_unixtime = %s, title = %s, year = %s, other_year = %s, poster_uri = %s, community_rate = %s WHERE id = %s", +# ( +# int(time.time()), +# series.title, +# series.year, +# series.other_year, +# series.poster_uri, +# series.community_rate, +# series.id, +# ), +# ) + +# db.connection.commit() +# db.connection.close() + +# return True + +# def get_series_by_id(self, id): +# db = Database() +# db.cursor.execute(f"SELECT * FROM series WHERE id = {id}") + +# result = db.cursor.fetchone() + +# if result is None: +# task_manager = TasksManager() +# task_manager.new_task("scrap_series", id) + +# return None +# else: +# return Series( +# id=result[0], +# title=result[3], +# year=result[4], +# other_year=result[5], +# poster_uri=result[6], +# community_rate=result[7], +# ) diff --git a/old_way/tasks.py b/old_way/tasks.py new file mode 100644 index 0000000..9177ebf --- /dev/null +++ b/old_way/tasks.py @@ -0,0 +1,280 @@ +# import time +# from datetime import datetime, timedelta + +# from users import UserManager +# from db import Database + + +# class Task: +# def __init__( +# self, id_task, status, type, job, unix_timestamp, unix_timestamp_last_update +# ): +# self.id_task = id_task +# self.status = status +# self.type = type +# self.job = job +# self.unix_timestamp = unix_timestamp +# self.unix_timestamp_last_update = unix_timestamp_last_update + +# def __str__(self): +# return ( +# f"{self.id_task} {self.status} {self.type} {self.job} {self.unix_timestamp}" +# ) + + +# # status: waiting, in_progress, done, failed +# # type: scrap_movie, scrape_series, send_discord, check_user_new_movies, check_user_new_series + + +# class TasksManager: +# def __init__(self): +# pass + +# def update_stuck_tasks(self): +# db = Database() + +# # Get the current time +# now = datetime.now() + +# # Calculate the threshold for 'stuck' tasks +# threshold = now - timedelta(minutes=5) # adjust as needed + +# # Convert the threshold to a Unix timestamp +# threshold_timestamp = int(threshold.timestamp()) + +# # Get all tasks with a status of 'in_progress' and a unix_timestamp_last_update older than the threshold +# db.cursor.execute( +# f"UPDATE tasks SET status = %s WHERE status = %s AND unix_timestamp_last_update < %s", +# ("waiting", "in_progress", threshold_timestamp), +# ) + +# db.connection.commit() +# db.connection.close() + +# return True + +# def delete_old_tasks(self): +# db = Database() + +# # Get the current time +# now = datetime.now() + +# # Calculate the threshold for 'stuck' tasks +# threshold = now - timedelta(minutes=30) + +# # Convert the threshold to a Unix timestamp +# threshold_timestamp = int(threshold.timestamp()) + +# # Get all tasks with a status of 'in_progress' and a unix_timestamp_last_update older than the threshold +# db.cursor.execute( +# f"DELETE FROM tasks WHERE unix_timestamp_last_update < %s", +# (threshold_timestamp,), +# ) +# db.connection.commit() +# db.connection.close() + +# return True + +# def update_task_status(self, id_task: int, status: str): +# db = Database() + +# if status == "done": +# self.delete_task(id_task) + +# db.cursor.execute( +# f"UPDATE tasks SET status = %s WHERE id_task = %s", +# (status, id_task), +# ) + +# db.connection.commit() +# db.connection.close() + +# return True + +# def delete_task(self, id_task: int): +# db = Database() + +# db.cursor.execute( +# f"DELETE FROM tasks WHERE id_task = %s", +# (id_task,), +# ) + +# db.connection.commit() +# db.connection.close() + +# return True + +# def add_scrap_users_task(self): +# db = Database() + +# user_manager = UserManager() + +# all_users = user_manager.get_all_users_from_db() + +# for user in all_users: +# self.new_task("check_user_new_movies", user.id_filmweb) +# self.new_task("check_user_new_series", user.id_filmweb) + +# return True + +# def add_scrap_movies_task(self): +# db = Database() + +# db.cursor.execute("SELECT id FROM movies") +# result = db.cursor.fetchall() + +# for movie in result: +# self.new_task("scrap_movie", movie[0]) + +# return True + +# def add_scrap_series_task(self): +# db = Database() + +# db.cursor.execute("SELECT id FROM series") +# result = db.cursor.fetchall() + +# for series in result: +# self.new_task("scrap_series", series[0]) + +# return True + +# def new_task(self, type: str, job: str): +# # status: waiting, in_progress, done, failed +# # type: scrap_movie, scrape_series, send_discord, check_user_new_movies, check_user_new_series, + +# db = Database() + +# unix_timestamp = int(time.time()) + +# db.cursor.execute( +# f"INSERT INTO tasks (status, type, job, unix_timestamp, unix_timestamp_last_update) VALUES (%s, %s, %s, %s, %s)", +# ("waiting", type, job, unix_timestamp, unix_timestamp), +# ) + +# db.connection.commit() +# db.connection.close() + +# return True + +# def get_and_update_tasks(self, types: list = None, status: str = "waiting"): +# db = Database() + +# types_placeholder = ", ".join(["%s"] * len(types)) + +# db.cursor.execute( +# f"SELECT * FROM tasks WHERE type IN ({types_placeholder}) AND status = %s ORDER BY id_task ASC LIMIT 5", +# (*types, status), +# ) + +# result = db.cursor.fetchall() + +# if result is None: +# return None + +# tasks = [] + +# for task in result: +# tasks.append( +# Task( +# id_task=task[0], +# status=task[1], +# type=task[2], +# job=task[3], +# unix_timestamp=task[4], +# unix_timestamp_last_update=task[5], +# ) +# ) + +# for task in tasks: +# db.cursor.execute( +# f"UPDATE tasks SET status = %s WHERE id_task = %s", +# ("in_progress", task.id_task), +# ) + +# db.connection.commit() +# db.connection.close() + +# return tasks + +# def get_tasks_count_by_type_and_status(self, type: str, status: str): +# db = Database() + +# db.cursor.execute( +# f"SELECT COUNT(*) FROM tasks WHERE type = %s AND status = %s", +# (type, status), +# ) + +# result = db.cursor.fetchone() + +# db.connection.commit() +# db.connection.close() + +# return result[0] + +# def get_task_by_type(self, type: str): +# db = Database() + +# if type is None: +# db.cursor.execute( +# f"SELECT * FROM tasks WHERE status = %s ORDER BY id_task ASC LIMIT 1", +# ("waiting",), +# ) +# else: +# db.cursor.execute( +# f"SELECT * FROM tasks WHERE type = %s AND status = %s ORDER BY id_task ASC LIMIT 1", +# (type, "waiting"), +# ) + +# result = db.cursor.fetchone() + +# if result is None: +# return None + +# task = Task( +# id_task=result[0], +# status=result[1], +# type=result[2], +# job=result[3], +# unix_timestamp=result[4], +# unix_timestamp_last_update=result[5], +# ) + +# return task + +# # db.cursor.execute( +# # f"UPDATE tasks SET status = %s WHERE id_task = %s", +# # ("in_progress", task.id_task), +# # ) + +# # db.connection.commit() +# # db.connection.close() + +# def reset_stuck_tasks(): +# db = Database() + +# # Get the current time +# now = datetime.now() + +# # Calculate the threshold for 'stuck' tasks +# threshold = now - timedelta(minutes=5) # adjust as needed + +# # Convert the threshold to a Unix timestamp +# threshold_timestamp = int(threshold.timestamp()) + +# # Get all tasks with a status of 'in_progress' and a unix_timestamp_last_update older than the threshold +# db.cursor.execute( +# f"SELECT * FROM tasks WHERE status = 'in_progress' AND unix_timestamp_last_update < %s", +# (threshold_timestamp,), +# ) +# tasks = db.cursor.fetchall() + +# for task in tasks: +# # Update the status of the task back to 'in_progress' +# db.cursor.execute( +# f"UPDATE tasks SET status = 'in_progress' WHERE id_task = %s", +# (task["id_task"],), +# ) + +# db.connection.commit() +# db.connection.close() diff --git a/old_way/users.py b/old_way/users.py new file mode 100644 index 0000000..b53b59b --- /dev/null +++ b/old_way/users.py @@ -0,0 +1,115 @@ +# from mysql.connector.errors import IntegrityError +# import requests + +# from fake_useragent import UserAgent + +# from db import Database + + +# class User: +# def __init__(self, id_filmweb, id_discord, discord_color): +# self.id_filmweb = id_filmweb +# self.id_discord = id_discord +# self.discord_color = discord_color + +# def __str__(self): +# return f"{self.id_filmweb} {self.id_discord} {self.discord_color}" + + +# class UserManager: +# def __init__(self): +# pass + +# def get_all_users_from_db(self): +# db = Database() +# db.cursor.execute(f"SELECT * FROM users") +# result = db.cursor.fetchall() +# db.connection.close() + +# if result is None: +# return None + +# return list( +# map( +# lambda x: User( +# id_filmweb=x[0], +# id_discord=x[1], +# discord_color=x[2], +# ), +# result, +# ) +# ) + +# def check_user_has_filmweb_account(self, id_filmweb: str): +# headers = { +# "User-Agent": UserAgent().random, +# } + +# response = requests.head( +# f"https://www.filmweb.pl/api/v1/user/{id_filmweb}/preview", headers=headers +# ) + +# if response.status_code == 200: +# return True + +# return False + +# def create_user(self, id_filmweb: str, id_discord: int): +# from tasks import TasksManager + +# db = Database() + +# if self.check_user_has_filmweb_account(id_filmweb) is False: +# return "User does not exist on filmweb" + +# try: +# db.cursor.execute( +# f"INSERT INTO users (id_filmweb, id_discord) VALUES (%s, %s)", +# (id_filmweb, id_discord), +# ) + +# db.connection.commit() +# except IntegrityError: +# return "User already exists" + +# db.connection.close() + +# tasks_manager = TasksManager() +# tasks_manager.new_task( +# "check_user_new_movies", +# f"{id_filmweb}", +# ) +# tasks_manager.new_task( +# "check_user_new_series", +# f"{id_filmweb}", +# ) + +# return True + +# def delete_user(self, id_filmweb: str): +# db = Database() + +# db.cursor.execute( +# f"DELETE FROM users WHERE id_filmweb = %s", +# (id_filmweb,), +# ) + +# db.connection.commit() +# db.connection.close() + +# return True + +# def get_user_by_filmweb_id(self, id_filmweb: str): +# db = Database() +# db.cursor.execute(f"SELECT * FROM users WHERE id_filmweb = %s", (id_filmweb,)) +# result = db.cursor.fetchone() +# db.connection.close() + +# if result is None: +# return None + +# return User( +# id_filmweb=result[0], +# id_discord=result[1], +# discord_color=result[2], +# ) diff --git a/old_way/utils.py b/old_way/utils.py new file mode 100644 index 0000000..2c87f80 --- /dev/null +++ b/old_way/utils.py @@ -0,0 +1,2 @@ +# def cut_unix_timestamp_miliseconds(unix_timestamp: int) -> int: +# return int(unix_timestamp / 1000) \ No newline at end of file diff --git a/old_way/watched.py b/old_way/watched.py new file mode 100644 index 0000000..c65e1d5 --- /dev/null +++ b/old_way/watched.py @@ -0,0 +1,224 @@ +# from db import Database +# from movie import Movie, MovieManager +# from series import Series, SeriesManager +# from tasks import TasksManager +# from users import UserManager +# from discord_m import DiscordManager + + +# class Watched: +# def __init__( +# self, id_watched, id_filmweb, movie_id, rate, comment, favorite, unix_timestamp +# ): +# self.id_watched = id_watched +# self.id_filmweb = id_filmweb +# self.movie_id = movie_id +# self.rate = rate +# self.comment = comment +# self.favorite = favorite +# self.unix_timestamp = unix_timestamp + +# def __str__(self): +# return f"{self.id_watched} {self.id_filmweb} {self.movie_id} {self.rate} {self.comment} {self.favourite} {self.unix_timestamp}" + + +# class WatchedManager: +# def __init__(self): +# pass + +# def update_movie_data( +# self, +# movie_id: int, +# title: str, +# year: int, +# poster_path: str, +# community_rate: float, +# ): +# movie_manager = MovieManager() + +# movie = movie_manager.get_movie_by_id(movie_id) + +# if movie is None: +# return False + +# movie.title = title +# movie.year = year +# movie.poster_path = poster_path +# movie.community_rate = community_rate + +# movie_manager.update_movie(movie) + +# return True + +# def add_watched_series( +# self, +# id_filmweb: str, +# series_id: int, +# rate: int, +# comment: str, +# favorite: bool, +# unix_timestamp: int, +# without_discord: bool, +# ): +# db = Database() +# series_manager = SeriesManager() + +# # check series is in db +# db.cursor.execute(f"SELECT * FROM series WHERE id = %s", (series_id,)) +# result = db.cursor.fetchone() + +# if result is None: +# # if not, create task to scrap series + +# series_manager.add_series_to_db( +# Series( +# id=series_id, +# title="", +# year=0, +# other_year=0, +# poster_uri="", +# community_rate=0, +# ) +# ) + +# task_manager = TasksManager() +# task_manager.new_task("scrap_series", series_id) + +# # check it is not already in watched +# db.cursor.execute( +# f"SELECT * FROM watched_series WHERE id_filmweb = %s AND series_id = %s", +# (id_filmweb, series_id), +# ) + +# result = db.cursor.fetchone() + +# if result is not None: +# print("Already watched") +# return "Already watched" + +# # add to watched +# db.cursor.execute( +# f"INSERT INTO watched_series (id_filmweb, series_id, rate, comment, favourite, unix_timestamp) VALUES (%s, %s, %s, %s, %s, %s)", +# (id_filmweb, series_id, rate, comment, favorite, unix_timestamp), +# ) + +# db.connection.commit() +# db.connection.close() + +# if without_discord is True: +# return True + +# task_manager = TasksManager() +# task_manager.new_task("send_discord", f"{id_filmweb},{series_id}") + +# return True + +# def add_watched_movie( +# self, +# id_filmweb: str, +# movie_id: int, +# rate: int, +# comment: str, +# favorite: bool, +# unix_timestamp: int, +# without_discord: bool, +# ): +# db = Database() +# movie_manager = MovieManager() + +# # check movie is in db +# db.cursor.execute(f"SELECT * FROM movies WHERE id = %s", (movie_id,)) +# result = db.cursor.fetchone() + +# if result is None: +# # if not, create task to scrap movie + +# movie_manager.add_movie_to_db( +# Movie( +# id=movie_id, +# title="", +# year=0, +# poster_uri="", +# community_rate=0, +# ) +# ) + +# task_manager = TasksManager() +# task_manager.new_task("scrap_movie", movie_id) + +# # check it is not already in watched +# db.cursor.execute( +# f"SELECT * FROM watched_movies WHERE id_filmweb = %s AND movie_id = %s", +# (id_filmweb, movie_id), +# ) + +# result = db.cursor.fetchone() + +# if result is not None: +# print("Already watched") +# return "Already watched" + +# # add to watched +# db.cursor.execute( +# f"INSERT INTO watched_movies (id_filmweb, movie_id, rate, comment, favourite, unix_timestamp) VALUES (%s, %s, %s, %s, %s, %s)", +# (id_filmweb, movie_id, rate, comment, favorite, unix_timestamp), +# ) +# db.connection.commit() +# db.connection.close() + +# if without_discord is True: +# return True + +# task_manager = TasksManager() +# task_manager.new_task("send_discord", f"{id_filmweb},{movie_id}") + +# return True + +# def get_all_watched_movies(self, id_filmweb: str): +# db = Database() +# db.cursor.execute( +# f"SELECT * FROM watched_movies WHERE id_filmweb = %s", (id_filmweb,) +# ) +# result = db.cursor.fetchall() +# db.connection.close() + +# if result is None: +# return None + +# if len(result) == 0: +# return None + +# watched_movies = [] + +# for row in result: +# watched_movies.append( +# Watched(row[0], row[1], row[2], row[3], row[4], row[5], row[6]) +# ) + +# return watched_movies + +# def get_watched_movie_with_rates(self, id_filmweb: str, movie_id: int): +# db = Database() +# db.cursor.execute( +# f"SELECT * FROM watched_movies WHERE id_filmweb = %s AND movie_id = %s", +# (id_filmweb, movie_id), +# ) + +# result = db.cursor.fetchone() + +# w = Watched( +# result[0], result[1], result[2], result[3], result[4], result[5], result[6] +# ) + +# db.connection.close() + +# movie_manager = MovieManager() +# user_manager = UserManager() +# discord_manager = DiscordManager() + +# return ( +# w, +# movie_manager.get_movie_by_id(movie_id), +# user_manager.get_user_by_filmweb_id(id_filmweb), +# discord_manager.get_user_notification_destinations(id_filmweb), +# ) From d1e71971098e561cc6230becaca939f3dda5e41c Mon Sep 17 00:00:00 2001 From: Jakub Sucheta Date: Fri, 5 Jan 2024 15:15:45 +0100 Subject: [PATCH 8/8] api finished (no user prefs for now) --- FILMAN-CRAWLER/Dockerfile | 3 + FILMAN-CRAWLER/main.py | 3 +- FILMAN-CRAWLER/requirements.txt | 47 --------------- FILMAN-CRAWLER/tasks/scrap_series.py | 84 ++++++++++++--------------- FILMAN-CRAWLER/tasks/utils.py | 30 ++++++++++ FILMAN-DISCORD/main.py | 2 +- FILMAN-DISCORD/requirements.txt | Bin 1842 -> 0 bytes FILMAN-SERVER/Dockerfile | 3 + FILMAN-SERVER/database/crud.py | 14 +++-- FILMAN-SERVER/database/db.py | 8 ++- FILMAN-SERVER/main.py | 2 +- docker-compose.yml | 29 ++++----- requirements.txt | Bin 1842 -> 1922 bytes 13 files changed, 103 insertions(+), 122 deletions(-) delete mode 100644 FILMAN-CRAWLER/requirements.txt delete mode 100644 FILMAN-DISCORD/requirements.txt diff --git a/FILMAN-CRAWLER/Dockerfile b/FILMAN-CRAWLER/Dockerfile index 4410457..7cf3e4e 100644 --- a/FILMAN-CRAWLER/Dockerfile +++ b/FILMAN-CRAWLER/Dockerfile @@ -1,5 +1,8 @@ FROM python:3.11-slim-bullseye +RUN apt update +RUN apt install -y pkg-config default-libmysqlclient-dev build-essential + WORKDIR /FILMAN-CRAWLER COPY requirements.txt requirements.txt diff --git a/FILMAN-CRAWLER/main.py b/FILMAN-CRAWLER/main.py index 969dc03..3f72771 100644 --- a/FILMAN-CRAWLER/main.py +++ b/FILMAN-CRAWLER/main.py @@ -168,7 +168,7 @@ def do_task(task: Task): def main(): logging.info("Program started") - min_wait = 3 # Minimum wait time in seconds + min_wait = 2 # Minimum wait time in seconds max_wait = 60 # Maximum wait time in seconds wait_time = min_wait @@ -186,6 +186,7 @@ def main(): logging.info("No tasks found") wait_time = min_wait + time.sleep(wait_time) if __name__ == "__main__": main() diff --git a/FILMAN-CRAWLER/requirements.txt b/FILMAN-CRAWLER/requirements.txt deleted file mode 100644 index 6fe7ebf..0000000 --- a/FILMAN-CRAWLER/requirements.txt +++ /dev/null @@ -1,47 +0,0 @@ -aiohttp==3.9.1 -aiosignal==1.3.1 -annotated-types==0.6.0 -anyio==3.7.1 -attrs==23.1.0 -certifi==2023.11.17 -charset-normalizer==3.3.2 -click==8.1.7 -colorama==0.4.6 -colorlog==6.8.0 -discord-py-interactions==5.11.0 -discord-typings==0.7.0 -discord.py==2.3.2 -emoji==2.9.0 -fake-useragent==1.4.0 -fastapi==0.104.1 -frozenlist==1.4.0 -greenlet==3.0.2 -h11==0.14.0 -hikari==2.0.0.dev122 -hikari-lightbulb==2.3.4 -idna==3.6 -multidict==6.0.4 -mypy-extensions==1.0.0 -mysql-connector-python==8.2.0 -protobuf==4.21.12 -psutil==5.9.6 -pydantic==2.5.2 -pydantic-settings==2.1.0 -pydantic_core==2.14.5 -python-crontab==3.0.0 -python-dateutil==2.8.2 -python-dotenv==1.0.0 -python-multipart==0.0.6 -requests==2.31.0 -schedule==1.2.1 -six==1.16.0 -sniffio==1.3.0 -SQLAlchemy==1.4.50 -starlette==0.27.0 -tomli==2.0.1 -typing-inspect==0.9.0 -typing_extensions==4.8.0 -ujson==5.8.0 -urllib3==2.1.0 -uvicorn==0.24.0.post1 -yarl==1.9.4 diff --git a/FILMAN-CRAWLER/tasks/scrap_series.py b/FILMAN-CRAWLER/tasks/scrap_series.py index 2c0652f..d1863fe 100644 --- a/FILMAN-CRAWLER/tasks/scrap_series.py +++ b/FILMAN-CRAWLER/tasks/scrap_series.py @@ -2,6 +2,8 @@ import requests import ujson +from .utils import Task, TaskTypes, TaskStatus +from .utils import Tasks, FilmWeb logging.basicConfig( format="%(asctime)s %(levelname)-8s %(message)s", @@ -10,23 +12,6 @@ ) -class Task: - def __init__( - self, id_task, status, type, job, unix_timestamp, unix_timestamp_last_update - ): - self.id_task = id_task - self.status = status - self.type = type - self.job = job - self.unix_timestamp = unix_timestamp - self.unix_timestamp_last_update = unix_timestamp_last_update - - def __str__(self): - return ( - f"{self.id_task} {self.status} {self.type} {self.job} {self.unix_timestamp}" - ) - - class Scraper: def __init__(self, headers=None, series_id=None, endpoint_url=None): self.headers = headers @@ -41,8 +26,8 @@ def fetch(self, url): return response.text def scrap(self, task): - info_url = f"https://www.filmweb.pl/api/v1/title/{task.job}/info" - rating_url = f"https://www.filmweb.pl/api/v1/film/{task.job}/rating" + info_url = f"https://www.filmweb.pl/api/v1/title/{task.task_job}/info" + rating_url = f"https://www.filmweb.pl/api/v1/film/{task.task_job}/rating" info_data = self.fetch(info_url) rating_data = self.fetch(rating_url) @@ -56,42 +41,45 @@ def scrap(self, task): title = info_data.get("title", None) year = int(info_data.get("year", None)) other_year = int(info_data.get("otherYear", 0)) - poster_url = info_data.get("posterPath", "https://vectorified.com/images/no-data-icon-23.png") + poster_url = info_data.get( + "posterPath", "https://vectorified.com/images/no-data-icon-23.png" + ) community_rate = rating_data.get("rate", None) if title is None or year is None or poster_url is None: return False - + return self.update_data( - self.series_id, title, year, other_year, poster_url, community_rate, task.id_task + self.series_id, + title, + year, + other_year, + poster_url, + community_rate, + task.task_id, ) - - def update_data(self, series_id, title, year, other_year, poster_url, community_rate, id_task): - # r = requests.post( - # f"{self.endpoint_url}/series/update", - # headers=self.headers, - # json={ - # "id": int(series_id), - # "title": str(title), - # "year": int(year), - # "other_year": int(other_year), - # "poster_uri": str(poster_url), - # "community_rate": float(community_rate), - # }, - # ) - - if r.status_code != 200: - logging.error(f"Error updating series data: HTTP {r.status_code}") - logging.error(r.text) - return False - - # r = requests.get( - # f"{self.endpoint_url}/task/update?id_task={id_task}&status=done", - # headers=self.headers, - # ) - - if r.status_code != 200: + def update_data( + self, series_id, title, year, other_year, poster_url, community_rate, task_id + ): + try: + filmweb = FilmWeb(self.headers, self.endpoint_url) + filmweb.update_series( + FilmWeb.FilmWebSeries( + id=series_id, + title=title, + year=year, + other_year=other_year, + poster_url=poster_url, + community_rate=community_rate, + ) + ) + + tasks = Tasks(self.headers, self.endpoint_url) + tasks.update_task_status(task_id, TaskStatus.COMPLETED) + + except Exception as e: + logging.error(f"Error updating series data: {e}") return False return True diff --git a/FILMAN-CRAWLER/tasks/utils.py b/FILMAN-CRAWLER/tasks/utils.py index 0c0f764..7c5f2cd 100644 --- a/FILMAN-CRAWLER/tasks/utils.py +++ b/FILMAN-CRAWLER/tasks/utils.py @@ -63,6 +63,15 @@ class FilmWebMovie(BaseModel): poster_url: str community_rate: float + class FilmWebSeries(BaseModel): + id: int + title: str + year: int + other_year: int + poster_url: str + community_rate: float + + def update_movie(self, movie: FilmWebMovie): r = requests.post( f"{self.endpoint_url}/filmweb/update/movie", @@ -82,3 +91,24 @@ def update_movie(self, movie: FilmWebMovie): return False return True + + def update_series(self, series: FilmWebSeries): + r = requests.post( + f"{self.endpoint_url}/filmweb/update/series", + headers=self.headers, + json={ + "id": int(series.id), + "title": str(series.title), + "year": int(series.year), + "other_year": int(series.other_year), + "poster_url": str(series.poster_url), + "community_rate": float(series.community_rate), + }, + ) + + if r.status_code != 200: + logging.error(f"Error updating series data: HTTP {r.status_code}") + logging.error(r.text) + return False + + return True \ No newline at end of file diff --git a/FILMAN-DISCORD/main.py b/FILMAN-DISCORD/main.py index 739413f..8ba2bea 100644 --- a/FILMAN-DISCORD/main.py +++ b/FILMAN-DISCORD/main.py @@ -9,7 +9,7 @@ # bot = lightbulb.BotApp( -# "MTE4MjM3MTY1ODM0NzA2NTM5NA.Gr7OHu.Y3PW4Dl98nLZAzAw1eJfvAzpushE3GXyM0qAX8", +# "MTE4NDUzMTQxMjY2MjEwNDA5NQ.GDmZof.QH06crcIcS3vdiFeH5JhLkkCv-pz2GcccB8360", # intents=hikari.Intents.ALL, # banner=None, # ) diff --git a/FILMAN-DISCORD/requirements.txt b/FILMAN-DISCORD/requirements.txt deleted file mode 100644 index 1bc4c391905e52afe124936491f119904f1445d2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1842 zcmZ{lL2uen5QO(!sXxUic0$q~dg#5Esy`qy*cb}f1Vfbc$G4qt*AJVD3cW;O=grRS z?%VwNUE0PvTiZ#WgMHEG-e&e$C%2FG!6M!Z)mW>)c8|l?Y^B5PLYxVOs@$GD$5Nh3=c$Fy zT;I|5?qFlToF`?+Nip2oiFavR=lg=a7 zrx+6u3@fSHN?m4WWeAmxexn&Q+Ni;Z?iDGODcGn{t|*w8mzA7$-h);Zr6RYO8lh^> zv(wK&&?)#r^;&$nYls}9inUm`(r}zO0>i1q_Nsfug=NHMuBY+$u#n%KEiM%y_k;I& zBM#mI?)f&lzjr*{qs#P>cuak|x%XE=&ue=Vf-3U1#(qbqQ#HZ&_97jbPYv&^c5Wsp z=I11}+$S58R_H`84qoSzqIg+3f*KWaPcNd12W2=*%SKt?VB+Y0xMD&pd8OO90zn2) z>Hak$6CeEoKTaig#wq@n_1WFg@5G<`yv)7cpp#4L!1SP}9J-UHBDtQT(N3MG4$%^o zV#8gtSK^Moytqy(l}I@T8Oc-A}5)Mpz3`3CFC1A#ciDIYqs%=bbs>`@lrs!v(ZodW;j@$FB4b^+^;bQ`)F|?y>A@{_u iC+@5_&;0Uk;;n?Y*ct4_&N~!d-o2mW|2lNX+y4s=wGq+) diff --git a/FILMAN-SERVER/Dockerfile b/FILMAN-SERVER/Dockerfile index 8936bbe..4a87b4d 100644 --- a/FILMAN-SERVER/Dockerfile +++ b/FILMAN-SERVER/Dockerfile @@ -1,5 +1,8 @@ FROM python:3.11-slim-bullseye +RUN apt update +RUN apt install -y pkg-config default-libmysqlclient-dev build-essential + WORKDIR /FILMAN-SERVER COPY requirements.txt requirements.txt diff --git a/FILMAN-SERVER/database/crud.py b/FILMAN-SERVER/database/crud.py index 17b7909..66bd4e7 100644 --- a/FILMAN-SERVER/database/crud.py +++ b/FILMAN-SERVER/database/crud.py @@ -260,12 +260,14 @@ def __change_task_status(db: Session, task_id: int, task_status: schemas.TaskSta return None db_task.task_status = task_status - db_task.task_started = ( - datetime.now() if task_status == schemas.TaskStatus.RUNNING else None - ) - db_task.task_finished = ( - datetime.now() if task_status == schemas.TaskStatus.COMPLETED else None - ) + + task_started = datetime.now() if task_status == schemas.TaskStatus.RUNNING else None + if task_started is not None: + db_task.task_started = task_started + + task_finished = datetime.now() if task_status == schemas.TaskStatus.COMPLETED else None + if task_finished is not None: + db_task.task_finished = task_finished db.commit() db.refresh(db_task) diff --git a/FILMAN-SERVER/database/db.py b/FILMAN-SERVER/database/db.py index 1b521d6..858d251 100644 --- a/FILMAN-SERVER/database/db.py +++ b/FILMAN-SERVER/database/db.py @@ -5,9 +5,15 @@ from sqlalchemy.orm import sessionmaker SQLALCHEMY_DATABASE_URL = os.environ.get( - "SQLALCHEMY_DATABASE_URL", "mysql://root@localhost/filmweb_test3" + "SQLALCHEMY_DATABASE_URL", None ) +# SQLALCHEMY_DATABASE_URL = os.environ.get( +# "SQLALCHEMY_DATABASE_URL", "mysql://root@localhost/filmweb_test3" +# ) + +print(SQLALCHEMY_DATABASE_URL) + engine = create_engine(SQLALCHEMY_DATABASE_URL) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) diff --git a/FILMAN-SERVER/main.py b/FILMAN-SERVER/main.py index ececd0c..0fe1550 100644 --- a/FILMAN-SERVER/main.py +++ b/FILMAN-SERVER/main.py @@ -40,4 +40,4 @@ async def trigger_error(): if __name__ == "__main__": - uvicorn.run(app, host="127.0.0.1", port=8001) + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/docker-compose.yml b/docker-compose.yml index 021370a..64096a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,25 +10,20 @@ services: context: . dockerfile: FILMAN-SERVER/Dockerfile environment: - - MYSQL_HOST=host.docker.internal - # - MYSQL_USER=root - # - MYSQL_PASSWORD= - - MYSQL_DATABASE=filmweb_test2 + - SQLALCHEMY_DATABASE_URL=mysql://y168:1B00_3267ae@mysql.mikr.us/db_y168 restart: unless-stopped - # extra_hosts: - # - "host.docker.internal" - filman-discord: - depends_on: - - filman-server - container_name: filman-discord - build: - # context: /FILMAN-DISCORD - context: . - dockerfile: FILMAN-DISCORD/Dockerfile - environment: - - DISCORD_TOKEN=MTE4NDUzMTQxMjY2MjEwNDA5NQ.GDmZof.QH06crcIcS3vdiFeH5JhLkkCv-pz2GcccB8360 - restart: unless-stopped + # filman-discord: + # depends_on: + # - filman-server + # container_name: filman-discord + # build: + # # context: /FILMAN-DISCORD + # context: . + # dockerfile: FILMAN-DISCORD/Dockerfile + # environment: + # - DISCORD_TOKEN=MTE4NDUzMTQxMjY2MjEwNDA5NQ.GDmZof.QH06crcIcS3vdiFeH5JhLkkCv-pz2GcccB8360 + # restart: unless-stopped filman-crawler: depends_on: diff --git a/requirements.txt b/requirements.txt index 1bc4c391905e52afe124936491f119904f1445d2..42abe4145391615ac8a2b1a9d3e14ae46700000c 100644 GIT binary patch delta 83 zcmdnQ*Tlc!9TLnT8oLm@*BLoyI&GNdx(F_bXa0-+Iu9vB;LzR7%%Q5GUm1XQjI W)R4lE4OV0bRAkIxxp_Y8WJUlxff989 delta 17 ZcmZqT-^91!9`j~4mNSf-f3bEj0suY32B-i4