Merge pull request #12 from suchencjusz/dev

tasks test

FILMAN-CRAWLER/main.py

@@ -50,6 +50,9 @@
     "Accept": "*/*",
     "Accept-Language": "en-US,en;q=0.5",
     "Accept-Encoding": "gzip, deflate, br",
+    "Cache-Control": "no-cache",
+    "Pragma": "no-cache",
+    "If-Modified-Since": "0",
     "Origin": "https://www.filmweb.pl",
     "DNT": "1",
     "Connection": "keep-alive",

FILMAN-CRAWLER/requirements.txt

@@ -14,6 +14,7 @@ discord.py==2.3.2
 emoji==2.9.0
 fake-useragent==1.4.0
 fastapi==0.104.1
+fastapi-restful==0.5.0
 fastapi-utils==0.2.1
 frozenlist==1.4.0
 greenlet==3.0.2
@@ -22,16 +23,25 @@ hikari==2.0.0.dev122
 hikari-lightbulb==2.3.4
 idna==3.6
 multidict==6.0.4
+mypy-extensions==1.0.0
 mysql-connector-python==8.2.0
 protobuf==4.21.12
-pydantic==1.10.13
+psutil==5.9.6
+pydantic==2.5.2
+pydantic-settings==2.1.0
 pydantic_core==2.14.5
+python-crontab==3.0.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
 python-multipart==0.0.6
 requests==2.31.0
+schedule==1.2.1
+six==1.16.0
 sniffio==1.3.0
 SQLAlchemy==1.4.50
 starlette==0.27.0
 tomli==2.0.1
+typing-inspect==0.9.0
 typing_extensions==4.8.0
 ujson==5.8.0
 urllib3==2.1.0

FILMAN-DISCORD/endpoints/configure.py

@@ -7,7 +7,7 @@
 
 
 @configure_plugin.command
-@lightbulb.command("configure", "Configure the bot.")
+@lightbulb.command("configure", "Skonfiguruj bota")
 @lightbulb.implements(lightbulb.SlashCommandGroup)
 async def configure_group(_: lightbulb.SlashContext) -> None:
     pass

FILMAN-DISCORD/endpoints/help.py

@@ -32,7 +32,7 @@ async def configure_subcommand(ctx: lightbulb.SlashContext) -> None:
 
 
 @help_group.child
-@lightbulb.command("tracker", "powiadomienia i śledzenie użytkownika")
+@lightbulb.command("tracker", "monitoruje konto filmweb")
 @lightbulb.implements(lightbulb.SlashSubCommand)
 async def tracker_subcommand(ctx: lightbulb.SlashContext) -> None:
     embed = hikari.Embed(

docker-compose-prod.yml → FILMAN-SERVER/docker-compose-prod.yml

@@ -6,9 +6,9 @@ services:
     image: suchencjusz/filman-server:main
     ports:
       - "8000:8000"
-    # build:
-    #   context: /FILMAN-SERVER
-    #   dockerfile: Dockerfile
+    build:
+      context: /FILMAN-SERVER
+      dockerfile: Dockerfile
     environment:
       - MYSQL_HOST=mysql.mikr.us
       - MYSQL_USER=t345
@@ -23,9 +23,9 @@ services:
       - filman-server
     container_name: filman-discord
     image: suchencjusz/filman-discord:main
-    # build:
-    #   context: /FILMAN-DISCORD
-    #   dockerfile: Dockerfile
+    build:
+      context: /FILMAN-DISCORD
+      dockerfile: Dockerfile
     environment:
       - DISCORD_TOKEN=MTE4MjM3MTY1ODM0NzA2NTM5NA.Gr7OHu.Y3PW4Dl98nLZAzAw1eJfvAzpushE3GXyM0qAX8
     restart: unless-stopped
@@ -35,9 +35,9 @@ services:
       - filman-server
     container_name: filman-crawler
     image: suchencjusz/filman-crawler:main
-    # build:
-    #   context: /FILMAN-CRAWLER
-    #   dockerfile: Dockerfile
+    build:
+      context: /FILMAN-CRAWLER
+      dockerfile: Dockerfile
     environment:
       - CORE_ENDPOINT=http://filman-server:8000
     restart: unless-stopped

FILMAN-SERVER/main.py

@@ -1,11 +1,9 @@
+import threading
 import uvicorn
 
 from fastapi import FastAPI, HTTPException, Request
 from fastapi.middleware.cors import CORSMiddleware
 
-from fastapi_utils.session import FastAPISessionMaker
-from fastapi_utils.tasks import repeat_every
-
 from pydantic import BaseModel
 from typing import List, Optional
 
@@ -16,8 +14,13 @@
 from discord_m import DiscordManager
 from tasks import TasksManager
 
+
 app = FastAPI()
 
+##################################################
+# MODELS
+##################################################
+
 
 class Movie(BaseModel):
     movie_id: str
@@ -258,7 +261,6 @@ async def update_task(id_task: int, status: str):
         raise HTTPException(status_code=500, detail=result)
 
 
-@repeat_every(seconds=60 * 3)
 @app.get("/task/scrap/all/users")
 async def scrap_all_users():
     tasks_manager = TasksManager()
@@ -269,6 +271,26 @@ async def scrap_all_users():
         raise HTTPException(status_code=500, detail=result)
 
 
+@app.get("/tasks/update/stuck")
+async def update_stuck_tasks():
+    tasks_manager = TasksManager()
+    result = tasks_manager.update_stuck_tasks()
+    if result is True:
+        return {"message": "OK"}
+    else:
+        raise HTTPException(status_code=500, detail=result)
+
+
+@app.get("/tasks/delete/old")
+async def delete_old_tasks():
+    tasks_manager = TasksManager()
+    result = tasks_manager.delete_old_tasks()
+    if result is True:
+        return {"message": "OK"}
+    else:
+        raise HTTPException(status_code=500, detail=result)
+
+
 ##################################################
 # DISCORD
 ##################################################
@@ -325,4 +347,18 @@ async def configure_guild(guild_configure_in: GuildConfigureIn):
 
 
 if __name__ == "__main__":
-    uvicorn.run(app, host="0.0.0.0", port=8000, log_level="debug")
+    # uvicorn.run(app, host="0.0.0.0", port=8000, log_level="debug")
+
+    uvicorn_thread = threading.Thread(
+        target=uvicorn.run,
+        args=(app,),
+        kwargs={"host": "0.0.0.0", "port": 8000, "log_level": "debug"},
+    )
+
+    jobs_thread = threading.Thread(target=run_jobs_thread)
+
+    uvicorn_thread.start()
+    jobs_thread.start()
+
+    uvicorn_thread.join()
+    jobs_thread.join()

FILMAN-SERVER/tasks.py

@@ -30,6 +30,47 @@ class TasksManager:
     def __init__(self):
         pass
 
+    def update_stuck_tasks(self):
+        db = Database()
+
+        # Get the current time
+        now = datetime.now()
+
+        # Calculate the threshold for 'stuck' tasks
+        threshold = now - timedelta(minutes=5)  # adjust as needed
+
+        # Convert the threshold to a Unix timestamp
+        threshold_timestamp = int(threshold.timestamp())
+
+        # Get all tasks with a status of 'in_progress' and a unix_timestamp_last_update older than the threshold
+        db.cursor.execute(
+            f"UPDATE tasks SET status = %s WHERE status = %s AND unix_timestamp_last_update < %s",
+            ("waiting", "in_progress", threshold_timestamp),
+        )
+
+        db.connection.commit()
+        db.connection.close()
+
+    def delete_old_tasks(self):
+        db = Database()
+
+        # Get the current time
+        now = datetime.now()
+
+        # Calculate the threshold for 'stuck' tasks
+        threshold = now - timedelta(minutes=30)
+
+        # Convert the threshold to a Unix timestamp
+        threshold_timestamp = int(threshold.timestamp())
+
+        # Get all tasks with a status of 'in_progress' and a unix_timestamp_last_update older than the threshold
+        db.cursor.execute(
+            f"DELETE FROM tasks WHERE unix_timestamp_last_update < %s",
+            (threshold_timestamp,),
+        )
+        db.connection.commit()
+        db.connection.close()
+
     def update_task_status(self, id_task: int, status: str):
         db = Database()
 

data/data/global/70/a4/5b/70a45b37adf20d85be16525c9a8e8e03.json

@@ -0,0 +1 @@
+{"type":"list_page","items":[{"id":"maingrp","title":"Primary Group","regexp":"^(main)$","master":1},{"id":"allgrp","title":"All Servers","regexp":".+","master":0}]}

data/data/global/73/f2/06/73f2061c54ebbd19ba9bbddd70299297.json

@@ -0,0 +1 @@
+{"page_size":50,"first_page":0,"last_page":0,"length":1,"type":"list"}

data/data/global/7d/59/1d/7d591db2f282c22552f4d445fd236154.json

@@ -0,0 +1 @@
+{"type":"list_page","items":[{"username":"admin"}]}

data/data/global/81/f0/77/81f077cc3e3d7f58ba1f2da11fd20297.json

@@ -0,0 +1 @@
+{"type":"list_page","items":[{"id":"general","title":"General","enabled":1,"username":"admin","modified":1702405390,"created":1702405390,"description":"For events that don't fit anywhere else.","max_children":0}]}

data/data/global/83/ab/32/83ab32f287f97e49b1f77a810bfe242d.json

@@ -0,0 +1 @@
+{"type":"list_page","items":[{"id":"testplug","title":"Test Plugin","enabled":1,"command":"bin/test-plugin.js","username":"admin","modified":1702405390,"created":1702405390,"params":[{"id":"duration","type":"text","size":10,"title":"Test Duration (seconds)","value":60},{"id":"progress","type":"checkbox","title":"Report Progress","value":1},{"id":"burn","type":"checkbox","title":"Burn Memory/CPU","value":0},{"id":"action","type":"select","title":"Simulate Action","items":["Success","Failure","Crash"],"value":"Success"},{"id":"secret","type":"hidden","value":"Will not be shown in Event UI"}]},{"id":"shellplug","title":"Shell Script","enabled":1,"command":"bin/shell-plugin.js","username":"admin","modified":1702405390,"created":1702405390,"params":[{"id":"script","type":"textarea","rows":10,"title":"Script Source","value":"#!/bin/sh\n\n# Enter your shell script code here"},{"id":"annotate","type":"checkbox","title":"Add Date/Time Stamps to Log","value":0},{"id":"json","type":"checkbox","title":"Interpret JSON in Output","value":0}]},{"id":"urlplug","title":"HTTP Request","enabled":1,"command":"bin/url-plugin.js","username":"admin","modified":1702405390,"created":1702405390,"params":[{"type":"select","id":"method","title":"Method","items":["GET","HEAD","POST"],"value":"GET"},{"type":"textarea","id":"url","title":"URL","rows":3,"value":"http://"},{"type":"textarea","id":"headers","title":"Request Headers","rows":4,"value":"User-Agent: Cronicle/1.0"},{"type":"textarea","id":"data","title":"POST Data","rows":4,"value":""},{"type":"text","id":"timeout","title":"Timeout (Seconds)","size":5,"value":"30"},{"type":"checkbox","id":"follow","title":"Follow Redirects","value":0},{"type":"checkbox","id":"ssl_cert_bypass","title":"SSL Cert Bypass","value":0},{"type":"text","id":"success_match","title":"Success Match","size":20,"value":""},{"type":"text","id":"error_match","title":"Error Match","size":20,"value":""}]}]}

data/data/global/af/56/5a/af565a097a0cf8d7d5855ecc92c24c34.json

@@ -0,0 +1 @@
+{"type":"list_page","items":[{"hostname":"main","ip":"172.20.0.3"}]}

data/data/global/be/21/ad/be21ad44831ac52d7c3b3c06ec30ab02.json

@@ -0,0 +1 @@
+{"page_size":50,"first_page":0,"last_page":0,"length":3,"type":"list"}

data/data/global/bf/10/4a/bf104a19df63f5227cc0e457c353afa7.json

@@ -0,0 +1 @@
+{"page_size":50,"first_page":0,"last_page":0,"length":2,"type":"list"}

data/data/global/c7/45/9c/c7459c956e50650e77e33cd03ea3b31b.json

@@ -0,0 +1 @@
+{"page_size":50,"first_page":0,"last_page":0,"length":0,"type":"list"}

data/data/global/d5/34/10/d53410b6a012ac9db284846c1d465e2c.json

@@ -0,0 +1 @@
+{"page_size":50,"first_page":0,"last_page":0,"length":1,"type":"list"}

data/data/global/e7/3f/47/e73f47df4dd92352849bce2a5d622318.json

@@ -0,0 +1 @@
+{"page_size":100,"first_page":0,"last_page":0,"length":1,"type":"list"}

data/data/users/34/68/bc/3468bc0c4e5f6aa06c7aee62212ac18f.json

@@ -0,0 +1 @@
+{"username":"admin","password":"$2a$10$VAF.FNvz1JqhCAB5rCh9GOa965eYWH3fcgWIuQFAmsZnnVS/.ye1y","full_name":"Administrator","email":"admin@cronicle.com","active":1,"modified":1702405390,"created":1702405390,"salt":"salty","privileges":{"admin":1}}

data/logs/cronicled.pid

@@ -0,0 +1 @@
+1

discord_bot_list_commands.py

@@ -0,0 +1,42 @@
+import requests
+
+headers = {
+    'Authorization': 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0IjoxLCJpZCI6IjExODIzNzE2NTgzNDcwNjUzOTQiLCJpYXQiOjE3MDIzOTk1NjJ9.-7jLpvYAKHXagsJt5ADmGHGvt4Qtw5i87J7B_slda3w'
+}
+
+body_commands = [
+    {
+        "name": "configure channel",
+        "description": "Ustawia kanał z powiadomieniami",
+        "type": 1
+    },
+    {
+        "name": "help",
+        "description": "Pomoc",
+        "type": 1
+    },
+    {
+        "name": "tracker me",
+        "description": "Monitoruje konto filmweb",
+    },
+    {
+        "name": "tracker cancel",
+        "description": "Anuluje monitorowanie konta filmweb i usuwa dane z bazy danych",
+    },
+    {
+        "name": "tracker stop",
+        "description": "Anuluje wysyłanie powiadomień na danym serwerze",
+    },
+    {
+        "name": "tracker here",
+        "description": "Dopisuje użytkownika do listy powiadomień na danym serwerze",
+    },
+    {
+        "name": "info",
+        "description": "Informacje o bocie",
+    }
+]
+
+r = requests.post('https://discordbotlist.com/api/v1/bots/filman/commands', headers=headers, json=body_commands)
+
+print(r.status_code)