feat: added docs #1

.github/readme.md

@@ -36,9 +36,48 @@ This is how data encryption works and how you should approach it.
 
 ```md
 GET -> meta_data.json
-     |
-     | --> Generate hash from user input ('<semester> <program> <section>'.replaceAll('/', ''))
-         |
-         | --> GET base_url/generated_hash.json => timetable
+|
+| --> Generate hash from user input ('<semester> <program> <section>'.replaceAll('/', ''))
+|
+| --> GET base_url/generated_hash.json => timetable
 ```
 
+## Usage
+
+- **Download Source Code**
+
+  ```bash
+  git clone <this_repo>
+  ```
+
+  > OR
+
+  [**Download**](https://github.com/Zain-ul-din/lgu-crawler/archive/refs/heads/master.zip)
+
+- **Prepare Environment Variables**
+
+  - create `.env` file in the root of the project.
+
+  ```.env
+  NODE_ENV="development" # recommended if running locally
+
+  # Note! default keys will work replace with your own keys if needed.
+  OPEN_DB_KEY="ae3ca465683436cbfd33e2ddd4aa0fcf9fbfcfe26d29379e111a01462f87ebeb" # Must be 32 characters
+  OPEN_DB_IV="0bf4254a8293e5aedcdfcb8095c08ffa" # Must be 16 characters
+
+  # get session id from LGU timetable website
+  PHPSESSID=""
+  ```
+
+  - paste following code in .env with your values.
+
+  **Appendix:**
+
+  - [How to Get Session ID?](https://github.com/IIvexII/LGU-TimetableAPI/blob/main/docs/How_to_get_session.md)
+  - To generate key and iv run `openssl rand -hex <size 32 | 16>`
+
+- **Run**
+  ```bash
+    npx yarn
+    npx yarn dev
+  ```

scripts/vercel-deploy.js

@@ -1,6 +1,7 @@
-const dotenv = require("dotenv")
-dotenv.config()
-
-const url = process.env.VERCEL_DEPLOYMENT_HOOK || 'https://www.google.com'
-fetch(url, { method: 'POST' }).then(res=> res.text()).then(console.log)
+const dotenv = require("dotenv");
+dotenv.config();
 
+const url = process.env.VERCEL_DEPLOYMENT_HOOK || "https://www.google.com";
+fetch(url, {method: "POST"})
+  .then((res) => res.text())
+  .then(console.log);

src/constants/env.ts

@@ -1,11 +1,30 @@
+// ********************************************************
+// Responsible for loading env variable from .env file
+// ********************************************************
+
 import dotenv from "dotenv";
 
 dotenv.config();
 
+/**
+ * Environment variables
+ */
 const ENV = {
+  /**
+   * The environment in which the application is running (e.g., "development", "production").
+   */
   NODE_ENV: process.env.NODE_ENV,
+  /**
+   * The encryption key used for data encryption.
+   */
   OPEN_DB_KEY: process.env.OPEN_DB_KEY,
+  /**
+   * The initialization vector used for encryption in an open database.
+   */
   OPEN_DB_IV: process.env.OPEN_DB_IV,
+  /**
+   * The PHP session ID used as a cookie to bypass login.
+   */
   PHPSESSID: process.env.PHPSESSID,
 };
 

src/constants/errors.ts

@@ -1,6 +1,13 @@
 import ENV from "./env";
 
+/**
+ * Errors designed to provide clear feedback when certain conditions are not
+ * met or unexpected behavior occurs.
+ */
 const ERRORS = {
+  /**
+   * This error is thrown when the provided PHP session ID (cookie) is invalid.
+   */
   INVALID_COOKIE: new Error(`Invalid Cookie (Php Session Id) '${ENV.PHPSESSID}'`),
 };
 

src/constants/week-days-name.ts

@@ -1,3 +1,6 @@
+/**
+ * List of week days name starting from monday to sunday
+ */
 const WEEK_DAYS_NAME = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"];
 
 export default WEEK_DAYS_NAME;

src/crawlers/MetaDataCrawler.ts

@@ -9,11 +9,19 @@ interface MetaDataCrawlerParams {
   parser: Parser<HTMLOptionsType>;
 }
 
+/**
+ * Abstract type that will return from MetadataCrawler
+ */
 export interface MetaDataCrawlerReturnType {
   metaData: MetaDataType;
   timeTableRequestPayloads: TimetableRequestPayload[];
 }
 
+/**
+ * MetaDataCrawler responsible for scrapping drop downs data from the targeted website.
+ * @description It's called metadata since metadata is `a set of data that describes and gives information about other data`.
+ * In this case, it is describing about the timetable data.
+ */
 class MetaDataCrawler extends Crawler<MetaDataCrawlerReturnType> {
   /**
    * Semesters page endpoint

src/index.ts

@@ -5,17 +5,27 @@ import {logOnCrawlTimetable} from "./lib/logger";
 
 new Worker()
 
-.onCrawlMetaData(({metaData, timeTableRequestPayloads}) => {
-  if (timeTableRequestPayloads.length == 0) throw ERRORS.INVALID_COOKIE;
-  TimetableRepository.writeMetaData(metaData);
-})
+  /* 
+  Callback invoked when metadata is available 
+*/
+  .onCrawlMetaData(({metaData, timeTableRequestPayloads}) => {
+    if (timeTableRequestPayloads.length == 0) throw ERRORS.INVALID_COOKIE;
+    TimetableRepository.writeMetaData(metaData);
+  })
 
-.onCrawlTimetable(logOnCrawlTimetable)
+  /* 
+  Callback invoked when timetable is crawled
+*/
+  .onCrawlTimetable(logOnCrawlTimetable)
 
-.onFinish((allTimetables) =>
-  TimetableRepository.writeTimetables(allTimetables)
-  .writeTeachersTimetable(allTimetables)
-  .writeRoomsTimetable(allTimetables)
-)
+  /* 
+  Callback invoked when all timetables are crawled
+*/
+  .onFinish((allTimetables) =>
+    TimetableRepository.writeTimetables(allTimetables)
+      .writeTeachersTimetable(allTimetables)
+      .writeRoomsTimetable(allTimetables)
+  )
 
-.start();
+  // start the worker 🔥
+  .start();

src/lib/TimetableRepository.ts

@@ -7,11 +7,22 @@ import {isJsonString} from "./util";
 import {WEEK_DAYS_NAME} from "../constants";
 import TimetableType from "../types/TimetableType";
 
+/**
+ * Repository for managing timetable data.
+ */
 class TimetableRepository {
+  /**
+   * Writes metadata to the database.
+   * @param metaData The metadata to write.
+   */
   public static writeMetaData(metaData: MetaDataType) {
     writeDB("meta_data", metaData, {hash: false});
   }
 
+  /**
+   * Writes timetables to the database.
+   * @param timetables The timetables to write.
+   */
   public static writeTimetables(timetables: TimetableDocType[]) {
     writeDB("all_timetables", timetables, {hash: false});
     writeDB("timetable_paths", timetables.map((t) => t.uid).map(hashStr), {hash: false});
@@ -31,6 +42,10 @@ class TimetableRepository {
     return this;
   }
 
+  /**
+   * Writes teachers' timetables to the database.
+   * @param timetables The timetables to write.
+   */
   public static writeTeachersTimetable(timetables: TimetableDocType[]) {
     const teachers = computeTeachers(timetables);
     writeDB("teachers", teachers, {hash: false});
@@ -51,6 +66,10 @@ class TimetableRepository {
     return this;
   }
 
+  /**
+   * Writes rooms' timetables to the database.
+   * @param timetables The timetables to write.
+   */
   public static writeRoomsTimetable(timetables: TimetableDocType[]) {
     const rooms = computeRooms(timetables);
     writeDB("rooms", rooms, {hash: false});
@@ -71,6 +90,12 @@ class TimetableRepository {
     return this;
   }
 
+  /**
+   * Compares the current timetable with the content stored in the database.
+   * @param curr The current timetable.
+   * @param dbContent The content stored in the database as a string.
+   * @returns true if the current timetable is different from the content in the database, false otherwise.
+   */
   private static compareDBTimetable(curr: TimetableDocType, dbContent: string) {
     if (!isJsonString(dbContent)) return false;
     const timetable = JSON.parse(dbContent) as any;

src/lib/computes.ts

@@ -1,6 +1,21 @@
+// **************************************************************************************************
+// BACKGROUND:
+// The Official Website maintains separate timetables for students, teachers, and rooms.
+// Retrieving each of these timetables individually can be a time-consuming process.
+// To tackle this issue, we have devised a strategy where we initially fetch students' timetables.
+// Subsequently, we utilize this data to derive timetables for teachers and rooms.
+// This approach significantly reduces the time required for data retrieval and minimizes the number
+// of requests made to the official website.
+// **************************************************************************************************
+
 import TimetableDocType from "../types/TimetableDocType";
 import {WEEK_DAYS_NAME} from "../constants";
 
+/**
+ * Computes the list of unique teachers from the provided timetables.
+ * @param timetables The list of timetables to process.
+ * @returns An array containing the unique teachers.
+ */
 export function computeTeachers(timetables: TimetableDocType[]) {
   const teachers = timetables
     .map(({timetable}) => {
@@ -13,17 +28,25 @@ export function computeTeachers(timetables: TimetableDocType[]) {
   return Array.from(new Set(teachers));
 }
 
+/**
+ * Computes the timetable for the given teacher from the provided timetables.
+ * @param teacher The teacher for whom to compute the timetable.
+ * @param timetables The list of timetables to process.
+ * @returns The timetable for the specified teacher.
+ */
 export function computeTeacherTimetable(teacher: string, timetables: TimetableDocType[]) {
   const teacherTimetable: TimetableDocType = {
     updatedAt: new Date(),
     uid: teacher,
     timetable: {},
   };
 
+  // Initialize timetable for each day of the week
   WEEK_DAYS_NAME.forEach((name) => {
     teacherTimetable.timetable[name] = [];
   });
 
+  // Populate teacher's timetable
   timetables.forEach(({uid, timetable}) => {
     Object.entries(timetable).forEach(([day, lectures]) => {
       lectures.forEach((lecture) => {
@@ -36,7 +59,7 @@ export function computeTeacherTimetable(teacher: string, timetables: TimetableDo
     });
   });
 
-  // sorts lectures
+  // Sort lectures by start time
   WEEK_DAYS_NAME.forEach((day) => {
     teacherTimetable.timetable[day] = teacherTimetable.timetable[day].sort(
       (lhs, rhs) =>
@@ -47,6 +70,11 @@ export function computeTeacherTimetable(teacher: string, timetables: TimetableDo
   return teacherTimetable;
 }
 
+/**
+ * Computes the list of unique rooms from the provided timetables.
+ * @param timetables The list of timetables to process.
+ * @returns An array containing the unique rooms.
+ */
 export function computeRooms(timetables: TimetableDocType[]) {
   const rooms = timetables
     .map(({timetable}) => {
@@ -59,17 +87,25 @@ export function computeRooms(timetables: TimetableDocType[]) {
   return Array.from(new Set(rooms));
 }
 
+/**
+ * Computes the timetable for the given room from the provided timetables.
+ * @param roomNo The room number for which to compute the timetable.
+ * @param timetables The list of timetables to process.
+ * @returns The timetable for the specified room.
+ */
 export function computeRoomTimetable(roomNo: string, timetables: TimetableDocType[]) {
   const roomTimetable: TimetableDocType = {
     updatedAt: new Date(),
     uid: roomNo,
     timetable: {},
   };
 
+  // Initialize timetable for each day of the week
   WEEK_DAYS_NAME.forEach((name) => {
     roomTimetable.timetable[name] = [];
   });
 
+  // Populate room's timetable
   timetables.forEach(({uid, timetable}) => {
     Object.entries(timetable).forEach(([day, lectures]) => {
       lectures.forEach((lecture) => {
@@ -82,7 +118,7 @@ export function computeRoomTimetable(roomNo: string, timetables: TimetableDocTyp
     });
   });
 
-  // sorts lectures
+  // Sort lectures by start time
   WEEK_DAYS_NAME.forEach((day) => {
     roomTimetable.timetable[day] = roomTimetable.timetable[day].sort(
       (lhs, rhs) =>

src/lib/logger.ts

@@ -1,6 +1,10 @@
 import TimetableDocType from "../types/TimetableDocType";
 import pc from "picocolors";
 
+/**
+ * Wrapper around console.log to print prettier output on console
+ * @param timetable
+ */
 export function logOnCrawlTimetable(timetable: TimetableDocType) {
   console.log(`🎯 ${pc.cyan("successfully crawled")} ${pc.magenta(timetable.uid)}`);
 }

src/lib/util.ts

@@ -1,7 +1,19 @@
+/**
+ * Returns a promise that resolves when all promises in the array have resolved.
+ * @param values An array of promises
+ * @returns A promise that resolves to an array of resolved values
+ */
 export async function promisify<T>(values: Promise<T>[]) {
   return await Promise.all(values);
 }
 
+/**
+ * Divides an array into smaller chunks based on the specified chunk size.
+ * @param arr The array to be divided into chunks
+ * @param chunks The number of chunks to create
+ * @returns An array of arrays, each containing a chunk of the original array
+ * @throws Error if number of chunks is less than or equal to 0
+ */
 export function chunkifyArray<T>(arr: T[], chunks: number) {
   if (chunks <= 0) throw new Error("Number of chunks should be greater than 0");
 
@@ -13,10 +25,22 @@ export function chunkifyArray<T>(arr: T[], chunks: number) {
   return res;
 }
 
+/**
+ * Ensures that a given number falls within a specified range.
+ * @param num The number to clamp
+ * @param min The minimum value of the range
+ * @param max The maximum value of the range
+ * @returns The clamped value within the specified range
+ */
 export function clamp(num: number, min: number, max: number) {
   return num <= min ? min : num >= max ? max : num;
 }
 
+/**
+ * Checks whether a given string is a valid JSON string.
+ * @param str The string to be checked
+ * @returns true if the string is a valid JSON string, false otherwise
+ */
 export function isJsonString(str: string) {
   try {
     JSON.parse(str);