From 963805cdf3b383abede58a26f1f1bee2e55a091d Mon Sep 17 00:00:00 2001 From: zainuldeen <78583049+Zain-ul-din@users.noreply.github.com> Date: Sat, 27 Apr 2024 19:59:09 +0500 Subject: [PATCH] feat: added docs #1 --- .github/readme.md | 47 +++++++++++++++++-- scripts/vercel-deploy.js | 11 ++--- src/constants/env.ts | 19 ++++++++ src/constants/errors.ts | 7 +++ src/constants/week-days-name.ts | 3 ++ src/crawlers/MetaDataCrawler.ts | 8 ++++ src/index.ts | 32 ++++++++----- src/lib/TimetableRepository.ts | 25 +++++++++++ src/lib/computes.ts | 40 ++++++++++++++++- src/lib/logger.ts | 4 ++ src/lib/util.ts | 24 ++++++++++ src/lib/worker.ts | 60 +++++++++++++++++++++++-- src/local-db/cipher.ts | 17 +++++++ src/local-db/initDB.ts | 5 +++ src/local-db/paths.ts | 7 +++ src/local-db/readDB.ts | 6 +++ src/local-db/types/DBUpadateStatus.d.ts | 5 +++ src/local-db/types/DBWriteOptions.d.ts | 10 +++++ src/local-db/types/FileSchema.d.ts | 7 +++ src/local-db/util.ts | 7 +++ src/local-db/writeDB.ts | 23 ++++++++++ src/parsers/OptionsParser.ts | 9 ++++ src/parsers/TimetableParser.ts | 20 +++++++++ 23 files changed, 370 insertions(+), 26 deletions(-) diff --git a/.github/readme.md b/.github/readme.md index dfc13dbcf..d8a11a83d 100644 --- a/.github/readme.md +++ b/.github/readme.md @@ -36,9 +36,48 @@ This is how data encryption works and how you should approach it. ```md GET -> meta_data.json - | - | --> Generate hash from user input ('
'.replaceAll('/', '')) - | - | --> GET base_url/generated_hash.json => timetable +| +| --> Generate hash from user input ('
'.replaceAll('/', '')) +| +| --> GET base_url/generated_hash.json => timetable ``` +## Usage + +- **Download Source Code** + + ```bash + git clone + ``` + + > OR + + [**Download**](https://github.com/Zain-ul-din/lgu-crawler/archive/refs/heads/master.zip) + +- **Prepare Environment Variables** + + - create `.env` file in the root of the project. + + ```.env + NODE_ENV="development" # recommended if running locally + + # Note! default keys will work replace with your own keys if needed. + OPEN_DB_KEY="ae3ca465683436cbfd33e2ddd4aa0fcf9fbfcfe26d29379e111a01462f87ebeb" # Must be 32 characters + OPEN_DB_IV="0bf4254a8293e5aedcdfcb8095c08ffa" # Must be 16 characters + + # get session id from LGU timetable website + PHPSESSID="" + ``` + + - paste following code in .env with your values. + + **Appendix:** + + - [How to Get Session ID?](https://github.com/IIvexII/LGU-TimetableAPI/blob/main/docs/How_to_get_session.md) + - To generate key and iv run `openssl rand -hex ` + +- **Run** + ```bash + npx yarn + npx yarn dev + ``` diff --git a/scripts/vercel-deploy.js b/scripts/vercel-deploy.js index a1657a3d8..fae78da79 100644 --- a/scripts/vercel-deploy.js +++ b/scripts/vercel-deploy.js @@ -1,6 +1,7 @@ -const dotenv = require("dotenv") -dotenv.config() - -const url = process.env.VERCEL_DEPLOYMENT_HOOK || 'https://www.google.com' -fetch(url, { method: 'POST' }).then(res=> res.text()).then(console.log) +const dotenv = require("dotenv"); +dotenv.config(); +const url = process.env.VERCEL_DEPLOYMENT_HOOK || "https://www.google.com"; +fetch(url, {method: "POST"}) + .then((res) => res.text()) + .then(console.log); diff --git a/src/constants/env.ts b/src/constants/env.ts index b1622be4f..7d4bab7ad 100644 --- a/src/constants/env.ts +++ b/src/constants/env.ts @@ -1,11 +1,30 @@ +// ******************************************************** +// Responsible for loading env variable from .env file +// ******************************************************** + import dotenv from "dotenv"; dotenv.config(); +/** + * Environment variables + */ const ENV = { + /** + * The environment in which the application is running (e.g., "development", "production"). + */ NODE_ENV: process.env.NODE_ENV, + /** + * The encryption key used for data encryption. + */ OPEN_DB_KEY: process.env.OPEN_DB_KEY, + /** + * The initialization vector used for encryption in an open database. + */ OPEN_DB_IV: process.env.OPEN_DB_IV, + /** + * The PHP session ID used as a cookie to bypass login. + */ PHPSESSID: process.env.PHPSESSID, }; diff --git a/src/constants/errors.ts b/src/constants/errors.ts index 532d93d43..88e965130 100644 --- a/src/constants/errors.ts +++ b/src/constants/errors.ts @@ -1,6 +1,13 @@ import ENV from "./env"; +/** + * Errors designed to provide clear feedback when certain conditions are not + * met or unexpected behavior occurs. + */ const ERRORS = { + /** + * This error is thrown when the provided PHP session ID (cookie) is invalid. + */ INVALID_COOKIE: new Error(`Invalid Cookie (Php Session Id) '${ENV.PHPSESSID}'`), }; diff --git a/src/constants/week-days-name.ts b/src/constants/week-days-name.ts index 8d0023e9c..392a64bb2 100644 --- a/src/constants/week-days-name.ts +++ b/src/constants/week-days-name.ts @@ -1,3 +1,6 @@ +/** + * List of week days name starting from monday to sunday + */ const WEEK_DAYS_NAME = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]; export default WEEK_DAYS_NAME; diff --git a/src/crawlers/MetaDataCrawler.ts b/src/crawlers/MetaDataCrawler.ts index 36f3702c6..57401218f 100644 --- a/src/crawlers/MetaDataCrawler.ts +++ b/src/crawlers/MetaDataCrawler.ts @@ -9,11 +9,19 @@ interface MetaDataCrawlerParams { parser: Parser; } +/** + * Abstract type that will return from MetadataCrawler + */ export interface MetaDataCrawlerReturnType { metaData: MetaDataType; timeTableRequestPayloads: TimetableRequestPayload[]; } +/** + * MetaDataCrawler responsible for scrapping drop downs data from the targeted website. + * @description It's called metadata since metadata is `a set of data that describes and gives information about other data`. + * In this case, it is describing about the timetable data. + */ class MetaDataCrawler extends Crawler { /** * Semesters page endpoint diff --git a/src/index.ts b/src/index.ts index a999ad5f6..abc0f5027 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,17 +5,27 @@ import {logOnCrawlTimetable} from "./lib/logger"; new Worker() -.onCrawlMetaData(({metaData, timeTableRequestPayloads}) => { - if (timeTableRequestPayloads.length == 0) throw ERRORS.INVALID_COOKIE; - TimetableRepository.writeMetaData(metaData); -}) + /* + Callback invoked when metadata is available +*/ + .onCrawlMetaData(({metaData, timeTableRequestPayloads}) => { + if (timeTableRequestPayloads.length == 0) throw ERRORS.INVALID_COOKIE; + TimetableRepository.writeMetaData(metaData); + }) -.onCrawlTimetable(logOnCrawlTimetable) + /* + Callback invoked when timetable is crawled +*/ + .onCrawlTimetable(logOnCrawlTimetable) -.onFinish((allTimetables) => - TimetableRepository.writeTimetables(allTimetables) - .writeTeachersTimetable(allTimetables) - .writeRoomsTimetable(allTimetables) -) + /* + Callback invoked when all timetables are crawled +*/ + .onFinish((allTimetables) => + TimetableRepository.writeTimetables(allTimetables) + .writeTeachersTimetable(allTimetables) + .writeRoomsTimetable(allTimetables) + ) -.start(); + // start the worker 🔥 + .start(); diff --git a/src/lib/TimetableRepository.ts b/src/lib/TimetableRepository.ts index 69d63ed94..436485751 100644 --- a/src/lib/TimetableRepository.ts +++ b/src/lib/TimetableRepository.ts @@ -7,11 +7,22 @@ import {isJsonString} from "./util"; import {WEEK_DAYS_NAME} from "../constants"; import TimetableType from "../types/TimetableType"; +/** + * Repository for managing timetable data. + */ class TimetableRepository { + /** + * Writes metadata to the database. + * @param metaData The metadata to write. + */ public static writeMetaData(metaData: MetaDataType) { writeDB("meta_data", metaData, {hash: false}); } + /** + * Writes timetables to the database. + * @param timetables The timetables to write. + */ public static writeTimetables(timetables: TimetableDocType[]) { writeDB("all_timetables", timetables, {hash: false}); writeDB("timetable_paths", timetables.map((t) => t.uid).map(hashStr), {hash: false}); @@ -31,6 +42,10 @@ class TimetableRepository { return this; } + /** + * Writes teachers' timetables to the database. + * @param timetables The timetables to write. + */ public static writeTeachersTimetable(timetables: TimetableDocType[]) { const teachers = computeTeachers(timetables); writeDB("teachers", teachers, {hash: false}); @@ -51,6 +66,10 @@ class TimetableRepository { return this; } + /** + * Writes rooms' timetables to the database. + * @param timetables The timetables to write. + */ public static writeRoomsTimetable(timetables: TimetableDocType[]) { const rooms = computeRooms(timetables); writeDB("rooms", rooms, {hash: false}); @@ -71,6 +90,12 @@ class TimetableRepository { return this; } + /** + * Compares the current timetable with the content stored in the database. + * @param curr The current timetable. + * @param dbContent The content stored in the database as a string. + * @returns true if the current timetable is different from the content in the database, false otherwise. + */ private static compareDBTimetable(curr: TimetableDocType, dbContent: string) { if (!isJsonString(dbContent)) return false; const timetable = JSON.parse(dbContent) as any; diff --git a/src/lib/computes.ts b/src/lib/computes.ts index f76d75689..5509cc6b4 100644 --- a/src/lib/computes.ts +++ b/src/lib/computes.ts @@ -1,6 +1,21 @@ +// ************************************************************************************************** +// BACKGROUND: +// The Official Website maintains separate timetables for students, teachers, and rooms. +// Retrieving each of these timetables individually can be a time-consuming process. +// To tackle this issue, we have devised a strategy where we initially fetch students' timetables. +// Subsequently, we utilize this data to derive timetables for teachers and rooms. +// This approach significantly reduces the time required for data retrieval and minimizes the number +// of requests made to the official website. +// ************************************************************************************************** + import TimetableDocType from "../types/TimetableDocType"; import {WEEK_DAYS_NAME} from "../constants"; +/** + * Computes the list of unique teachers from the provided timetables. + * @param timetables The list of timetables to process. + * @returns An array containing the unique teachers. + */ export function computeTeachers(timetables: TimetableDocType[]) { const teachers = timetables .map(({timetable}) => { @@ -13,6 +28,12 @@ export function computeTeachers(timetables: TimetableDocType[]) { return Array.from(new Set(teachers)); } +/** + * Computes the timetable for the given teacher from the provided timetables. + * @param teacher The teacher for whom to compute the timetable. + * @param timetables The list of timetables to process. + * @returns The timetable for the specified teacher. + */ export function computeTeacherTimetable(teacher: string, timetables: TimetableDocType[]) { const teacherTimetable: TimetableDocType = { updatedAt: new Date(), @@ -20,10 +41,12 @@ export function computeTeacherTimetable(teacher: string, timetables: TimetableDo timetable: {}, }; + // Initialize timetable for each day of the week WEEK_DAYS_NAME.forEach((name) => { teacherTimetable.timetable[name] = []; }); + // Populate teacher's timetable timetables.forEach(({uid, timetable}) => { Object.entries(timetable).forEach(([day, lectures]) => { lectures.forEach((lecture) => { @@ -36,7 +59,7 @@ export function computeTeacherTimetable(teacher: string, timetables: TimetableDo }); }); - // sorts lectures + // Sort lectures by start time WEEK_DAYS_NAME.forEach((day) => { teacherTimetable.timetable[day] = teacherTimetable.timetable[day].sort( (lhs, rhs) => @@ -47,6 +70,11 @@ export function computeTeacherTimetable(teacher: string, timetables: TimetableDo return teacherTimetable; } +/** + * Computes the list of unique rooms from the provided timetables. + * @param timetables The list of timetables to process. + * @returns An array containing the unique rooms. + */ export function computeRooms(timetables: TimetableDocType[]) { const rooms = timetables .map(({timetable}) => { @@ -59,6 +87,12 @@ export function computeRooms(timetables: TimetableDocType[]) { return Array.from(new Set(rooms)); } +/** + * Computes the timetable for the given room from the provided timetables. + * @param roomNo The room number for which to compute the timetable. + * @param timetables The list of timetables to process. + * @returns The timetable for the specified room. + */ export function computeRoomTimetable(roomNo: string, timetables: TimetableDocType[]) { const roomTimetable: TimetableDocType = { updatedAt: new Date(), @@ -66,10 +100,12 @@ export function computeRoomTimetable(roomNo: string, timetables: TimetableDocTyp timetable: {}, }; + // Initialize timetable for each day of the week WEEK_DAYS_NAME.forEach((name) => { roomTimetable.timetable[name] = []; }); + // Populate room's timetable timetables.forEach(({uid, timetable}) => { Object.entries(timetable).forEach(([day, lectures]) => { lectures.forEach((lecture) => { @@ -82,7 +118,7 @@ export function computeRoomTimetable(roomNo: string, timetables: TimetableDocTyp }); }); - // sorts lectures + // Sort lectures by start time WEEK_DAYS_NAME.forEach((day) => { roomTimetable.timetable[day] = roomTimetable.timetable[day].sort( (lhs, rhs) => diff --git a/src/lib/logger.ts b/src/lib/logger.ts index a2fc06c92..fe826c06a 100644 --- a/src/lib/logger.ts +++ b/src/lib/logger.ts @@ -1,6 +1,10 @@ import TimetableDocType from "../types/TimetableDocType"; import pc from "picocolors"; +/** + * Wrapper around console.log to print prettier output on console + * @param timetable + */ export function logOnCrawlTimetable(timetable: TimetableDocType) { console.log(`🎯 ${pc.cyan("successfully crawled")} ${pc.magenta(timetable.uid)}`); } diff --git a/src/lib/util.ts b/src/lib/util.ts index a2d7e062a..eddd00cc7 100644 --- a/src/lib/util.ts +++ b/src/lib/util.ts @@ -1,7 +1,19 @@ +/** + * Returns a promise that resolves when all promises in the array have resolved. + * @param values An array of promises + * @returns A promise that resolves to an array of resolved values + */ export async function promisify(values: Promise[]) { return await Promise.all(values); } +/** + * Divides an array into smaller chunks based on the specified chunk size. + * @param arr The array to be divided into chunks + * @param chunks The number of chunks to create + * @returns An array of arrays, each containing a chunk of the original array + * @throws Error if number of chunks is less than or equal to 0 + */ export function chunkifyArray(arr: T[], chunks: number) { if (chunks <= 0) throw new Error("Number of chunks should be greater than 0"); @@ -13,10 +25,22 @@ export function chunkifyArray(arr: T[], chunks: number) { return res; } +/** + * Ensures that a given number falls within a specified range. + * @param num The number to clamp + * @param min The minimum value of the range + * @param max The maximum value of the range + * @returns The clamped value within the specified range + */ export function clamp(num: number, min: number, max: number) { return num <= min ? min : num >= max ? max : num; } +/** + * Checks whether a given string is a valid JSON string. + * @param str The string to be checked + * @returns true if the string is a valid JSON string, false otherwise + */ export function isJsonString(str: string) { try { JSON.parse(str); diff --git a/src/lib/worker.ts b/src/lib/worker.ts index bc7bdc89f..4d46179cc 100644 --- a/src/lib/worker.ts +++ b/src/lib/worker.ts @@ -1,3 +1,11 @@ +// *********************************************************************************** +// BACKGROUND: +// The application utilizes a cluster of worker processes to improve performance when gathering timetable data from +// an official website. Without clusters, fetching and processing data sequentially could lead to slower performance +// due to limited CPU utilization. By distributing the workload across multiple cores using clusters, +// the application can efficiently retrieve and process data in parallel, resulting in faster execution times. +// *********************************************************************************** + import os from "os"; import {chunkifyArray, clamp, promisify} from "./util"; import TimetableRequestPayload from "../types/TimetableRequestPayload"; @@ -10,34 +18,65 @@ import {EventEmitter} from "stream"; import {MetaDataCrawlerReturnType} from "../crawlers/MetaDataCrawler"; import TimetableDocType from "../types/TimetableDocType"; +/** + * Worker class manages the process of crawling and processing timetable data using + * a cluster of worker processes to improve performance. + */ class Worker { + /** The number of available CPU cores */ private availableCores: number; + /** Event emitter for handling metadata crawl events */ + private onCrawlMetaDataEvent: EventEmitter; + + /** Event emitter for handling timetable crawl events */ + private onCrawlTimetableEvent: EventEmitter; + + /** Event emitter for handling crawl completion events */ + private onCrawlFinishEvent: EventEmitter; + + /** Events */ private static EVENT_NAMES = { + /** Event emitted when metadata is crawled */ ON_CRAWL_METADATA: "meta_data", + + /** Event emitted when timetable data is crawled */ ON_CRAWL_TIMETABLE: "timetable", + + /** Event emitted when the crawl process is finished */ ON_CRAWL_FINISH: "finish", }; - private onCrawlMetaDataEvent: EventEmitter; - private onCrawlTimetableEvent: EventEmitter; - private onCrawlFinishEvent: EventEmitter; - + /** + * Registers a callback function to be invoked when metadata is crawled. + * @param cb Callback function to handle crawled metadata. + */ public onCrawlMetaData(cb: (data: MetaDataCrawlerReturnType) => void) { this.onCrawlMetaDataEvent.on(Worker.EVENT_NAMES.ON_CRAWL_METADATA, cb); return this; } + /** + * Registers a callback function to be invoked when timetable data is crawled. + * @param cb Callback function to handle crawled timetable data. + */ public onCrawlTimetable(cb: (data: TimetableDocType) => void) { this.onCrawlTimetableEvent.on(Worker.EVENT_NAMES.ON_CRAWL_TIMETABLE, cb); return this; } + /** + * Registers a callback function to be invoked when crawling finishes. + * @param cb Callback function to handle the completion of crawling. + */ public onFinish(cb: (data: TimetableDocType[]) => void) { this.onCrawlFinishEvent.on(Worker.EVENT_NAMES.ON_CRAWL_FINISH, cb); return this; } + /** + * Constructs a new Worker instance. + */ public constructor() { this.availableCores = clamp(os.cpus().length, 2, 2); this.onCrawlMetaDataEvent = new EventEmitter(); @@ -45,12 +84,18 @@ class Worker { this.onCrawlFinishEvent = new EventEmitter(); } + /** + * Starts the crawling process. + */ public start() { this.scrapMetaData(); this.registerWorkers(); return this; } + /** + * Initiates the crawling of metadata. + */ private scrapMetaData() { if (!cluster.isPrimary) return; @@ -64,6 +109,10 @@ class Worker { metaDataCrawler.crawl(); } + /** + * Spawns worker processes to crawl timetable data. + * @param payloads Array of timetable request payloads to be processed. + */ private spawnWorkers(payloads: TimetableRequestPayload[]) { chunkifyArray(payloads, this.availableCores).forEach((chunk) => { const worker = cluster.fork(); @@ -82,6 +131,9 @@ class Worker { } } + /** + * Registers worker processes to crawl timetable data. + */ private registerWorkers() { if (!cluster.isWorker) return; diff --git a/src/local-db/cipher.ts b/src/local-db/cipher.ts index 769d4ec72..9a9fa42bb 100644 --- a/src/local-db/cipher.ts +++ b/src/local-db/cipher.ts @@ -2,14 +2,21 @@ import {Encoding, createCipheriv, createDecipheriv, createHash} from "crypto"; import {ENV} from "../constants"; import assert from "assert"; +// Define the cipher algorithm and the encoding for encrypted data export const CIPHER_ALGO = "aes-256-cbc"; export const ENCRYPTED_DATA_ENCODING: Encoding = "hex"; +// Retrieves encryption credentials (key and IV) const getCredentials = () => ({ key: Buffer.from(ENV.OPEN_DB_KEY || "", "hex"), iv: Buffer.from(ENV.OPEN_DB_IV || "", "hex"), }); +/** + * Encrypts data using AES-256-CBC algorithm. + * @param data The data to encrypt. + * @returns The encrypted data. + */ export function encrypt(data: string) { const {key, iv} = getCredentials(); const cipher = createCipheriv(CIPHER_ALGO, key, iv); @@ -19,6 +26,11 @@ export function encrypt(data: string) { return crypted; } +/** + * Decrypts data using AES-256-CBC algorithm. + * @param data The encrypted data to decrypt. + * @returns The decrypted data. + */ export function decrypt(data: string) { const {key, iv} = getCredentials(); const decipher = createDecipheriv(CIPHER_ALGO, key, iv); @@ -27,6 +39,11 @@ export function decrypt(data: string) { return decrypted; } +/** + * Calculates the SHA-256 hash of a string. + * @param str The string to hash. + * @returns The SHA-256 hash of the string. + */ export function hashStr(str: string) { return createHash("sha256").update(str).digest("hex"); } diff --git a/src/local-db/initDB.ts b/src/local-db/initDB.ts index 8ba6171b8..e4738ef6e 100644 --- a/src/local-db/initDB.ts +++ b/src/local-db/initDB.ts @@ -2,6 +2,11 @@ import {existsSync, mkdirSync} from "fs"; import {DB_PATH, LOCAL_DB_PATH} from "./paths"; import {ENV} from "../constants"; +/** + * Initializes the local database directory. + * If the local database directory does not exist and the environment is development, it creates the directory. + * It also creates the main database directory if it does not exist. + */ export default function initLocalDB() { if (!existsSync(LOCAL_DB_PATH) && ENV.NODE_ENV === "development") mkdirSync(LOCAL_DB_PATH); if (!existsSync(DB_PATH)) mkdirSync(DB_PATH); diff --git a/src/local-db/paths.ts b/src/local-db/paths.ts index 031df431f..abf6e36d7 100644 --- a/src/local-db/paths.ts +++ b/src/local-db/paths.ts @@ -1,2 +1,9 @@ +/** + * Path to the main database directory used in production. + */ export const DB_PATH = process.cwd() + "/db"; + +/** + * Path to the local database directory used in development. + */ export const LOCAL_DB_PATH = process.cwd() + "/local_db"; diff --git a/src/local-db/readDB.ts b/src/local-db/readDB.ts index 46cdbc427..3b6c32bd6 100644 --- a/src/local-db/readDB.ts +++ b/src/local-db/readDB.ts @@ -3,6 +3,12 @@ import {DB_PATH} from "./paths"; import {decrypt} from "./cipher"; import FileSchema from "./types/FileSchema"; +/** + * Reads data from the database file with the given UID. + * @param uid The unique identifier of the data file to read. + * @param decrypted If true, decrypts the data before returning. + * @returns The data read from the database file, or an empty string if the file does not exist. + */ export function readDB(uid: string, decrypted: boolean = true) { const filePath = `${DB_PATH}/${uid}.json`; if (!existsSync(filePath)) return ""; diff --git a/src/local-db/types/DBUpadateStatus.d.ts b/src/local-db/types/DBUpadateStatus.d.ts index ca6f9af9b..1e9837bab 100644 --- a/src/local-db/types/DBUpadateStatus.d.ts +++ b/src/local-db/types/DBUpadateStatus.d.ts @@ -1,5 +1,10 @@ +/** + * Represents the status of a database update operation. + */ interface DBUpdateStatus { + /** The content that was updated in the database. */ content: T; + /** Indicates whether the updated content is identical or different from the previous content in the database. */ similarity: "identical" | "different"; } diff --git a/src/local-db/types/DBWriteOptions.d.ts b/src/local-db/types/DBWriteOptions.d.ts index d6fbc3a3b..ac473e16a 100644 --- a/src/local-db/types/DBWriteOptions.d.ts +++ b/src/local-db/types/DBWriteOptions.d.ts @@ -1,5 +1,15 @@ +/** + * Represents options for writing to the database. + */ interface DBWriteOptions { + /** Indicates whether to hash the UID before writing to the database. */ hash?: boolean; + /** + * A custom comparison function to determine if the current content is different from the previous content in the database. + * @param curr The current content. + * @param previous The previous content stored in the database as a string. + * @returns true if the current content is different from the previous content, false otherwise. + */ compare?: (curr: T, previous: string) => boolean; } diff --git a/src/local-db/types/FileSchema.d.ts b/src/local-db/types/FileSchema.d.ts index 3c360a9f3..8da1075f9 100644 --- a/src/local-db/types/FileSchema.d.ts +++ b/src/local-db/types/FileSchema.d.ts @@ -1,9 +1,16 @@ import {CIPHER_ALGO, ENCRYPTED_DATA_ENCODING} from "../cipher"; +/** + * Represents the schema of a JSON file. + */ interface FileSchema { + /** The date when the file was last updated. */ updated_at: Date | string; + /** The encryption algorithm used to encrypt the file data. */ algo: typeof CIPHER_ALGO; + /** The encoding used for the encrypted data. */ encoding: typeof ENCRYPTED_DATA_ENCODING; + /** The encrypted data stored in the file. */ crypted: string; } diff --git a/src/local-db/util.ts b/src/local-db/util.ts index 7303abe15..32212119f 100644 --- a/src/local-db/util.ts +++ b/src/local-db/util.ts @@ -1,5 +1,12 @@ import {readDB} from "./readDB"; +/** + * Checks if the content is the same as the data stored in the database file with the given UID. + * @param uid The unique identifier of the database file. + * @param content The content to compare with the data in the database file. + * @param compareCB An optional callback function to customize the comparison logic. + * @returns true if the content is the same as the data in the database file, false otherwise. + */ export function isSame(uid: string, content: T, compareCB?: (curr: T, previous: string) => boolean) { const fileContent = readDB(uid); const contentToStr = typeof content === "string" ? content : JSON.stringify(content); diff --git a/src/local-db/writeDB.ts b/src/local-db/writeDB.ts index 2567dbbe0..134151067 100644 --- a/src/local-db/writeDB.ts +++ b/src/local-db/writeDB.ts @@ -8,6 +8,9 @@ import DBUpdateStatus from "./types/DBUpadateStatus"; import DBWriteOptions from "./types/DBWriteOptions"; import pc from "picocolors"; +/** + * Default options for writing to the database + */ const DEFAULT_OPTIONS: DBWriteOptions = { hash: true, compare(curr, previous) { @@ -15,17 +18,37 @@ const DEFAULT_OPTIONS: DBWriteOptions = { }, }; +/** + * Writes data to the database with the given UID. + * @param uid The unique identifier of the data. + * @param content The data to write to the database. + * @param options The options for writing to the database. + * @returns The status of the database update operation. + */ export function writeDB(uid: string, content: T, options: DBWriteOptions = DEFAULT_OPTIONS) { const _options = {...DEFAULT_OPTIONS, ...options}; writeDBLocal(uid, content); return writeDBPublic(uid, content, _options); } +/** + * Writes data to the local database directory only in development. + * @param uid The unique identifier of the data. + * @param content The data to write to the local database. + */ function writeDBLocal(uid: string, content: any) { if (ENV.NODE_ENV !== "development") return; writeFileSync(`${LOCAL_DB_PATH}/${uid}.json`, JSON.stringify(content, null, 2), "utf-8"); } +/** + * Writes encrypted data to the public database directory. + * @param uid The unique identifier of the data. + * @param content The data to write to the public database. + * @param hash Indicates whether to hash the UID. + * @param compare A custom comparison function to determine if the content has changed. + * @returns The status of the database update operation. + */ function writeDBPublic(uid: string, content: T, {hash, compare}: DBWriteOptions): DBUpdateStatus { const crypted = encrypt(JSON.stringify(content)); diff --git a/src/parsers/OptionsParser.ts b/src/parsers/OptionsParser.ts index fab6f94df..92f5e2f52 100644 --- a/src/parsers/OptionsParser.ts +++ b/src/parsers/OptionsParser.ts @@ -2,7 +2,16 @@ import {load} from "cheerio"; import HTMLOptionsType from "../types/HTMLOptionsType"; import Parser from "./Parser"; +/** + * Parses dropdown options from HTML content using Cheerio. + */ class OptionsParser extends Parser { + /** + * Parses dropdown options from HTML content. + * @param html The raw HTML content to parse. + * @param selector The CSS selector to locate dropdown options. + * @returns Parsed dropdown options containing text nodes and values. + */ public parse(html: string, selector: string): HTMLOptionsType { const $ = load(html); const options = $(selector) diff --git a/src/parsers/TimetableParser.ts b/src/parsers/TimetableParser.ts index 8fed13180..06a7e480d 100644 --- a/src/parsers/TimetableParser.ts +++ b/src/parsers/TimetableParser.ts @@ -1,7 +1,16 @@ import {load, Element, CheerioAPI} from "cheerio/lib/slim"; import Parser from "./Parser"; +/** + * Parsers timetable data from HTML content. + */ class TimetableParser extends Parser { + /** + * Parses timetable data from raw HTML content. + * @param rawContent The raw HTML content to parse. + * @param selector The CSS selector to locate timetable rows. + * @returns Parsed timetable data. + */ public parse(rawContent: string, selector: string = "tr") { const $ = load(rawContent); const rows = $("tr").toArray().slice(1); @@ -15,6 +24,12 @@ class TimetableParser extends Parser { return timetable; } + /** + * Parses a single timetable row. + * @param $ The Cheerio instance. + * @param row The row element to parse. + * @returns Parsed timetable data for the row. + */ private parseRow($: CheerioAPI, row: Element) { const weekName = $(row).find("th").text(); const tdNodes = $(row).find("td").toArray(); @@ -36,6 +51,11 @@ class TimetableParser extends Parser { return {[weekName]: lectures}; } + /** + * Parses text nodes containing lecture data. + * @param nodes An array of text nodes representing lecture data. + * @returns Parsed lecture data. + */ private parseTextNodes(nodes: string[]) { const time = nodes[4].split("-"); return {