Skip to content

Commit 4298bad

Browse files
thatITfoxKerby Keith Aquinokuroji-fusky
authored
Fast api change (#26)
* Added fastAPI to the requirements and imports * Implemented fastAPI for our api's, ready for testing * Shorten the variable names for /data * Use `async def` * Shortening variable names * Added google analytics (#25) * Added `argpase` on server * Minor setup.sh tweaks * OMG A TYPO * Formatting * moving `.env.local.example` * Removing babel Co-authored-by: Kerby Keith Aquino <94678583+skepfusky@users.noreply.github.com> Co-authored-by: skepfusky <skepfoosky15@gmail.com>
1 parent 2dbb16b commit 4298bad

File tree

15 files changed

+166
-99
lines changed

15 files changed

+166
-99
lines changed

‎.gitignore

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,8 @@ venv
4141
*.key
4242
*.crt
4343

44-
housepets_db.json
44+
# Ignore local Housepets json file
45+
housepets_db.json
46+
47+
# Ignore __pycache__ stuff
48+
server/__pycache__

‎app/.babelrc

Lines changed: 0 additions & 9 deletions
This file was deleted.

‎app/.env.local.example

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
NEXT_PUBLIC_GOOGLE_ANALYTICS=

‎app/lib/ga.ts

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
declare global {
2+
interface Window {
3+
gtag: any
4+
}
5+
}
6+
7+
export const pageview = (url) => {
8+
window.gtag("config", process.env.NEXT_PUBLIC_GOOGLE_ANALYTICS, {
9+
page_path: url,
10+
})
11+
}
12+
13+
export const event = ({ action, params }) => {
14+
window.gtag("event", action, params)
15+
}

‎app/next.config.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
const nextConfig = {
33
reactStrictMode: true,
44
compress: true,
5+
compiler: {
6+
removeConsole: true,
7+
},
58
images: {
69
domains: ['www.housepetscomic.com']
710
},

‎app/package.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
"@types/react": "17.0.43",
2525
"@types/react-dom": "17.0.14",
2626
"autoprefixer": "^10.4.4",
27-
"babel-plugin-transform-remove-console": "^6.9.4",
2827
"cssnano": "^5.1.7",
2928
"eslint": "8.12.0",
3029
"eslint-config-next": "12.1.2",

‎app/src/pages/_app.tsx

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,28 @@
11
import type { AppProps } from "next/app"
2+
import { useEffect } from "react"
3+
import { useRouter } from "next/router"
24
import Layout from "../components/Layout"
35
import { config } from "@fortawesome/fontawesome-svg-core"
46
import "@fortawesome/fontawesome-svg-core/styles.css"
57
import "../styles/globals.scss"
68
import "react-notifications-component/dist/theme.css"
79

10+
import * as ga from "../../lib/ga"
11+
812
config.autoAddCss = false
913

1014
export default function SearchpetsApp({ Component, pageProps }: AppProps) {
15+
const router = useRouter()
16+
17+
useEffect(() => {
18+
const handleRouteChange = (url) => {
19+
ga.pageview(url)
20+
}
21+
22+
router.events.on("routeChangeComplete", handleRouteChange)
23+
return () => router.events.off("routeChangeComplete", handleRouteChange)
24+
}, [router.events])
25+
1126
return (
1227
<Layout>
1328
<Component {...pageProps} />

‎app/src/pages/_document.tsx

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,22 @@ export default function Document() {
88
rel="stylesheet"
99
href="https://fonts.googleapis.com/css2?family=Montserrat:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&family=Open+Sans:ital,wght@0,300;0,400;0,500;0,600;0,700;0,800;1,300;1,400;1,500;1,600;1,700;1,800&display=swap"
1010
/>
11+
<script
12+
async
13+
src={`https://www.googletagmanager.com/gtag/js?id=${process.env.NEXT_PUBLIC_GOOGLE_ANALYTICS}`}
14+
></script>
15+
<script
16+
dangerouslySetInnerHTML={{
17+
__html: `
18+
window.dataLayer = window.dataLayer || [];
19+
function gtag(){dataLayer.push(arguments);}
20+
gtag('js', new Date());
21+
gtag('config', '${process.env.NEXT_PUBLIC_GOOGLE_ANALYTICS}', {
22+
page_path: window.location.pathname,
23+
});
24+
`,
25+
}}
26+
/>
1127
</Head>
1228
<body>
1329
<Main />

‎app/yarn.lock

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -451,11 +451,6 @@ axobject-query@^2.2.0:
451451
resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-2.2.0.tgz#943d47e10c0b704aa42275e20edf3722648989be"
452452
integrity sha512-Td525n+iPOOyUQIeBfcASuG6uJsDOITl7Mds5gFyerkWiX7qhUTdYUBlSgNMyVqtSJqwpt1kXGLdUt6SykLMRA==
453453

454-
babel-plugin-transform-remove-console@^6.9.4:
455-
version "6.9.4"
456-
resolved "https://registry.yarnpkg.com/babel-plugin-transform-remove-console/-/babel-plugin-transform-remove-console-6.9.4.tgz#b980360c067384e24b357a588d807d3c83527780"
457-
integrity sha512-88blrUrMX3SPiGkT1GnvVY8E/7A+k6oj3MNvUtTIxJflFzXTw1bHkuJ/y039ouhFMp2prRn5cQGzokViYi1dsg==
458-
459454
balanced-match@^1.0.0:
460455
version "1.0.2"
461456
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"

‎package.json

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@
22
"name": "searchpets",
33
"description": "A search engine to find your favorite Housepets comic!",
44
"scripts": {
5-
"dev": "concurrently -n next,python -k \"npm run dev:next\" \"npm run server\"",
6-
"build": "concurrently -n next,python -k \"yarn --cwd ./app build\" \"yarn server\"",
7-
"start": "concurrently -n next,python -k \"npm run start:next\" \"npm run server\"",
8-
"server": "cd server && python3 app.py",
5+
"dev": "concurrently -n next,python -k \"npm run dev:next\" \"npm run server:dev\"",
6+
"build": "concurrently -n next,python -k \"yarn --cwd ./app build\" \"yarn server:dev\"",
7+
"start": "concurrently -n next,python -k \"npm run start:next\" \"npm run server:build\"",
8+
"server:dev": "cd server && python app.py",
9+
"server:build": "cd server && python app.py --build",
910
"dev:next": "yarn --cwd ./app dev",
1011
"start:next": "yarn --cwd ./app start -p 80"
1112
},

‎scripts/setup.sh

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,12 @@
22

33
yarn install
44

5-
cd app
5+
yarn --cwd ./app install
66

7-
yarn install
8-
9-
cd ..
10-
11-
cd server
7+
cd ../server
128

13-
pip install -r requirements.txt
9+
python -m pip install -r requirements.txt
1410

1511
python gen.py
1612

17-
echo Done! Start the development server by running `yarn dev`.
13+
echo "Done! Start the development server by running \"yarn dev\"."

‎server/app.py

Lines changed: 77 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,38 @@
1-
from traceback import print_tb
2-
from flask import Flask, request, jsonify
1+
from bs4 import BeautifulSoup
2+
from fastapi import FastAPI
3+
from pydantic import BaseModel
4+
import argparse
35
import json
4-
import os
56
import threading
67
import time
78
import requests as req
8-
from bs4 import BeautifulSoup
99
import re
10+
import uvicorn
11+
12+
parser = argparse.ArgumentParser(description="Searchpets FastAPI Server")
13+
parser.add_argument('-b', '--build', action="store_true",
14+
help="Run the server in production mode, for development, don't provide with any arguments and run the file as is.")
15+
args = parser.parse_args()
16+
1017

1118
with open('housepets_db.json', 'r') as housepets_db_json:
1219
housepets_db = json.load(housepets_db_json)
1320

21+
"""
22+
Update database
23+
"""
24+
25+
1426
def update_database():
1527
while True:
1628
time.sleep(600)
1729
characters_db = set(housepets_db['characters_db'])
1830
year = time.strftime("%Y")
19-
web = req.get(f"https://www.housepetscomic.com/archive/?archive_year={year}")
31+
web = req.get(
32+
f"https://www.housepetscomic.com/archive/?archive_year={year}")
2033
soup = BeautifulSoup(web.text, 'html.parser')
21-
link_tag = soup.find_all('a', {'rel':"bookmark", 'href': re.compile("^https://")})
34+
link_tag = soup.find_all(
35+
'a', {'rel': "bookmark", 'href': re.compile("^https://")})
2236

2337
comics_db = []
2438

@@ -30,61 +44,77 @@ def update_database():
3044
if "https://www.housepetscomic.com/character" in web_link_page.text:
3145
print(f'{web_link} is a real comic by rick grifin')
3246

33-
characters_in_comic = []
34-
characters_in_comic_soup = BeautifulSoup(web_link_page.text, 'html.parser')
35-
characters_in_comic_tag = characters_in_comic_soup.find_all('a', {'href': re.compile("^https://www\.housepetscomic\.com/character")})
36-
for character in characters_in_comic_tag:
37-
characters_in_comic.append(character.text.lower())
47+
chars = []
48+
chars_soup = BeautifulSoup(
49+
web_link_page.text, 'html.parser')
50+
chars_tag = chars_soup.find_all(
51+
'a', {'href': re.compile("^https://www\.housepetscomic\.com/character")})
52+
for character in chars_tag:
53+
chars.append(character.text.lower())
3854
# the code under this will find the image where there is a tittle and alt
39-
comic_image = characters_in_comic_soup.find('img', {'title': True, 'alt': True})
40-
comics_db.append({"title": characters_in_comic_soup.title.text.split(' \u2013 ')[0],
41-
"comic_link": web_link,
42-
"characters": characters_in_comic,
43-
"image": comic_image.get('src')})
44-
characters_db.update(characters_in_comic)
45-
housepets_db['characters_db'] = list(characters_db) # update the characters db
46-
housepets_db[year] = comics_db # update the comics db
55+
comic_image = chars_soup.find(
56+
'img', {'title': True, 'alt': True})
57+
comics_db.append({"title": chars_soup.title.text.split(' \u2013 ')[0],
58+
"comic_link": web_link,
59+
"characters": chars,
60+
"image": comic_image.get('src')})
61+
characters_db.update(chars)
62+
housepets_db['characters_db'] = list(
63+
characters_db) # update the characters db
64+
housepets_db[year] = comics_db # update the comics db
4765
with open('housepets_db.json', 'w') as housepets_db_json:
4866
json.dump(housepets_db, housepets_db_json)
4967
print("[*] Database updated!")
5068

5169
else:
5270
print("[*] Database is currently up to date!")
5371

72+
5473
threading.Thread(target=update_database).start()
5574

56-
app = Flask(__name__)
5775

58-
@app.route('/search', methods=['POST'])
59-
def test():
60-
print(request.json)
61-
years = request.json['year']
62-
characters = request.json['characters']
76+
class Search(BaseModel):
77+
characters: list = []
78+
year: list = []
79+
80+
81+
app = FastAPI()
82+
83+
84+
@app.post('/search')
85+
async def search(search: Search):
86+
characters = search.characters
87+
years = search.year
88+
print(characters)
89+
print(years)
6390
comics = []
6491
for year in years:
65-
# check if the year is a key in the database
66-
if year in housepets_db:
67-
print(f'[*] {year} is a valid year')
68-
for comic in housepets_db[year]:
69-
if all(character in comic['characters'] for character in characters):
70-
comics.append(comic)
71-
print(f"[*] {len(comics)} comics found")
72-
return jsonify({'comics': comics})
73-
74-
@app.route('/data', methods=['GET'])
75-
def data():
76-
housepets_db_length = 0
92+
for comic in housepets_db[year]:
93+
if all(character in comic['characters'] for character in characters):
94+
comics.append(comic)
95+
return {'comics': comics}
96+
97+
98+
@app.get('/data')
99+
async def data():
100+
comic_length = 0
77101
for year in range(2008, 2022+1):
78-
housepets_db_length += len(housepets_db[str(year)])
79-
characters_db_length = len(housepets_db['characters_db'])
80-
return jsonify({'comicCount': housepets_db_length, 'charCount': characters_db_length})
102+
comic_length += len(housepets_db[str(year)])
103+
chars_length = len(housepets_db['characters_db'])
104+
return {'comicCount': comic_length, 'charCount': chars_length}
105+
81106

82-
@app.route('/characters', methods=['GET'])
83-
def characters():
84-
return jsonify({'characters_db': housepets_db['characters_db']})
107+
@app.get('/characters')
108+
async def characters():
109+
return {'characters_db': housepets_db['characters_db']}
85110

86111
if __name__ == '__main__':
87-
from waitress import serve
88-
print("[*] Starting server...")
89-
print("[*] Server running on port 5000.")
90-
serve(app, host='localhost', port=5000)
112+
if args.build:
113+
print("[*] Server started in build/production mode")
114+
print("[*] Server running on port 5000.")
115+
uvicorn.run("app:app", host="0.0.0.0", port=5000)
116+
117+
else:
118+
print("[*] Server started in development mode")
119+
uvicorn.run("app:app", host="0.0.0.0",
120+
port=5000, reload=True, debug=True)

‎server/gen.py

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
'Chrome/45.0.2454.101 Safari/537.36'),
1515
'referer': 'https://www.housepetscomic.com'}
1616

17-
years = ['2008', "2009", "2010", "2011", "2012", "2013", "2014",
17+
years = ['2008', "2009", "2010", "2011", "2012", "2013", "2014",
1818
"2015", "2016", "2017", "2018", "2019", "2020", "2021", "2022"]
1919

2020
housepets_db = {}
@@ -24,36 +24,36 @@
2424
for year in years:
2525
housepets_db.update({year: []})
2626
print(f"Searching in year {Fore.GREEN}{Style.BRIGHT}{year}{Style.RESET_ALL}")
27-
27+
2828
web = requests.get(f"https://www.housepetscomic.com/archive/?archive_year={year}", headers=user_agent, timeout=None)
2929
soup = BeautifulSoup(web.text, 'html.parser')
3030
link_tag = soup.find_all('a', {'rel':"bookmark", 'href': re.compile("^https://")})
3131
print(f"Found {Fore.GREEN}{Style.BRIGHT}{len(link_tag)}{Style.RESET_ALL} tags!")
32-
32+
3333
for link in link_tag:
34-
web_link = link.get('href')
35-
web_link_page = requests.get(web_link, headers=user_agent, timeout=None)
36-
if "https://www.housepetscomic.com/character" in web_link_page.text:
37-
print(web_link)
34+
link = link.get('href')
35+
link_page = requests.get(link, headers=user_agent, timeout=None)
36+
if "https://www.housepetscomic.com/character" in link_page.text:
37+
print(link)
3838

3939
characters = []
40-
comic_soup = BeautifulSoup(web_link_page.text, 'html.parser')
40+
comic_soup = BeautifulSoup(link_page.text, 'html.parser')
4141
characters_tag = comic_soup.find_all('a', {'href': re.compile("^https://www\.housepetscomic\.com/character")})
4242
for character in characters_tag:
4343
characters.append(character.text.lower())
4444
characters_db.add(character.text.lower())
45-
45+
4646
comic_image = comic_soup.find('img', {'title': True, 'alt': True})
4747

4848
print(comic_image.get('src'))
4949
housepets_db[year].append({
5050
'title':comic_soup.title.text.split(' \u2013 ')[0], # The character "u\u2013" is the unicode for the dash
51-
'comic_link': web_link,
51+
'comic_link': link,
5252
'characters': characters,
5353
'image': comic_image.get('src')
54-
})
54+
})
5555
else:
56-
print(f'{Fore.BLACK}{Back.LIGHTWHITE_EX}{Style.BRIGHT}{web_link} is a guest comics{Style.RESET_ALL}')
56+
print(f'{Fore.BLACK}{Back.LIGHTWHITE_EX}{Style.BRIGHT}{link} is a guest comics{Style.RESET_ALL}')
5757

5858
housepets_db['characters_db'] = list(characters_db)
5959
print("Saving to database...")

‎server/requirements.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
Flask
21
requests
32
beautifulsoup4
43
waitress
54
colorama
65
yapf
6+
fastapi
7+
uvicorn

0 commit comments

Comments
 (0)