Skip to content

Commit

Permalink
feat: prettier
Browse files Browse the repository at this point in the history
  • Loading branch information
Zain-ul-din committed Mar 19, 2024
1 parent ad16f8b commit 26cc890
Show file tree
Hide file tree
Showing 24 changed files with 203 additions and 149 deletions.
2 changes: 1 addition & 1 deletion scripts/env.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ const {writeFileSync, existsSync} = require("fs");
const ENV_PATH = process.cwd() + "./.env";
const DEFAULT_PLACEHOLDER = "YOUR_VALUE";

if (process.env !== 'production') process.exit(0);
if (process.env !== "production") process.exit(0);

writeFileSync(
`${process.cwd()}/.env`,
Expand Down
1 change: 0 additions & 1 deletion src/constants/env.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,3 @@ const ENV = {
};

export default ENV;

8 changes: 4 additions & 4 deletions src/constants/errors.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import ENV from "./env";
import ENV from "./env";

const ERRORS = {
INVALID_COOKIE: new Error(`Invalid Cookie (Php Session Id) '${ENV.PHPSESSID}'`)
}
INVALID_COOKIE: new Error(`Invalid Cookie (Php Session Id) '${ENV.PHPSESSID}'`),
};

export default ERRORS;
export default ERRORS;
6 changes: 1 addition & 5 deletions src/constants/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,4 @@ import WEEK_DAYS_NAME from "./week-days-name";
import ERRORS from "./errors";
import ENV from "./env";

export {
WEEK_DAYS_NAME, ERRORS, ENV
}


export {WEEK_DAYS_NAME, ERRORS, ENV};
10 changes: 1 addition & 9 deletions src/constants/week-days-name.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,3 @@
const WEEK_DAYS_NAME = [
'Monday',
'Tuesday',
'Wednesday',
'Thursday',
'Friday',
'Saturday',
'Sunday'
];
const WEEK_DAYS_NAME = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"];

export default WEEK_DAYS_NAME;
4 changes: 2 additions & 2 deletions src/crawlers/Crawler.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import {FetchOptions, ofetch} from "ofetch";
import request_headers from "../static/request_headers.json";
import {EventEmitter} from "stream";
import { ENV } from "../constants";
import {ENV} from "../constants";

type CrawlerEvents = "crawl";

Expand Down Expand Up @@ -45,7 +45,7 @@ abstract class Crawler<T = any> {
...request_headers,
Cookie: `PHPSESSID=${ENV.PHPSESSID}`,
};

/**
* Fetchs content by making http request
* @param url target URL
Expand Down
4 changes: 2 additions & 2 deletions src/crawlers/TimetableCrawler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ class TimetableCrawler extends Crawler<TimetableDocType> {

const timetable: TimetableDocType = {
updatedAt: new Date(),
payload: { semester, section, program },
payload: {semester, section, program},
uid: `${semester} ${program} ${section}`.replaceAll("/", ""),
timetable: this.timetableParser.parse(res),
};

this.event.emit(Crawler.ON_CRAWL, timetable);
return timetable;
}
Expand Down
26 changes: 13 additions & 13 deletions src/index.ts
Original file line number Diff line number Diff line change
@@ -1,21 +1,21 @@
import { ERRORS } from "./constants";
import {ERRORS} from "./constants";
import Worker from "./lib/worker";
import TimetableRepository from "./lib/TimetableRepository";
import { logOnCrawlTimetable } from "./lib/logger";
import {logOnCrawlTimetable} from "./lib/logger";

new Worker()

.onCrawlMetaData(({metaData, timeTableRequestPayloads}) => {
if(timeTableRequestPayloads.length == 0) throw ERRORS.INVALID_COOKIE;
TimetableRepository.writeMetaData(metaData)
})
.onCrawlMetaData(({metaData, timeTableRequestPayloads}) => {
if (timeTableRequestPayloads.length == 0) throw ERRORS.INVALID_COOKIE;
TimetableRepository.writeMetaData(metaData);
})

.onCrawlTimetable(logOnCrawlTimetable)
.onCrawlTimetable(logOnCrawlTimetable)

.onFinish((allTimetables) => TimetableRepository
.writeTimetables(allTimetables)
.writeTeachersTimetable(allTimetables)
.writeRoomsTimetable(allTimetables)
)
.onFinish((allTimetables) =>
TimetableRepository.writeTimetables(allTimetables)
.writeTeachersTimetable(allTimetables)
.writeRoomsTimetable(allTimetables)
)

.start();
.start();
114 changes: 78 additions & 36 deletions src/lib/TimetableRepository.ts
Original file line number Diff line number Diff line change
@@ -1,63 +1,105 @@
import { writeDB } from "../local-db";
import { hashStr } from "../local-db/cipher";
import {writeDB} from "../local-db";
import {hashStr} from "../local-db/cipher";
import MetaDataType from "../types/MetaDataType";
import TimetableDocType from "../types/TimetableDocType";
import { computeRoomTimetable, computeRooms, computeTeacherTimetable, computeTeachers } from "./computes";
import { isJsonString } from "./util";
import {computeRoomTimetable, computeRooms, computeTeacherTimetable, computeTeachers} from "./computes";
import {isJsonString} from "./util";
import {WEEK_DAYS_NAME} from "../constants";
import TimetableType from "../types/TimetableType";

class TimetableRepository {


public static writeMetaData(metaData: MetaDataType) {
writeDB("meta_data", metaData, { hash: false });
writeDB("meta_data", metaData, {hash: false});
}

public static writeTimetables(timetables: TimetableDocType[]) {
writeDB("all_timetables", timetables, { hash: false })
writeDB("timetable_paths", timetables.map(t=> t.uid).map(hashStr), { hash: false })
writeDB("all_timetables", timetables, {hash: false});
writeDB("timetable_paths", timetables.map((t) => t.uid).map(hashStr), {hash: false});

const timetableSnapShot = timetables.map((timetable)=> writeDB(timetable.uid, timetable,{
compare: TimetableRepository.compareDBTimetable
}))
const newTimetableChanges = timetableSnapShot.filter(({ similarity })=> similarity === 'different')
writeDB("timetables_new_changes", newTimetableChanges.map(({ content })=> hashStr(content.uid)), { hash: false });
const timetableSnapShot = timetables.map((timetable) =>
writeDB(timetable.uid, timetable, {
compare: TimetableRepository.compareDBTimetable,
})
);
const newTimetableChanges = timetableSnapShot.filter(({similarity}) => similarity === "different");
writeDB(
"timetables_new_changes",
newTimetableChanges.map(({content}) => hashStr(content.uid)),
{hash: false}
);

return this;
}

public static writeTeachersTimetable(timetables: TimetableDocType[]) {
const teachers = computeTeachers(timetables);
writeDB("teachers", teachers, { hash: false });
writeDB("teacher_paths", teachers.map(hashStr), { hash: false })
writeDB("teachers", teachers, {hash: false});
writeDB("teacher_paths", teachers.map(hashStr), {hash: false});

const teachersSnapShot = teachers.map((teacher) => writeDB(teacher, computeTeacherTimetable(teacher, timetables),{
compare: TimetableRepository.compareDBTimetable
}));
const teacherNewChanges = teachersSnapShot.filter(({ similarity }) => similarity === 'different')
writeDB("teachers_new_changes", teacherNewChanges.map(({ content}) => hashStr(content.uid)), { hash: false })
const teachersSnapShot = teachers.map((teacher) =>
writeDB(teacher, computeTeacherTimetable(teacher, timetables), {
compare: TimetableRepository.compareDBTimetable,
})
);
const teacherNewChanges = teachersSnapShot.filter(({similarity}) => similarity === "different");
writeDB(
"teachers_new_changes",
teacherNewChanges.map(({content}) => hashStr(content.uid)),
{hash: false}
);

return this;
}

public static writeRoomsTimetable(timetables: TimetableDocType[]) {
const rooms = computeRooms(timetables);
writeDB("rooms", rooms, { hash: false });
writeDB("rooms_paths", rooms.map(hashStr), { hash: false })
writeDB("rooms", rooms, {hash: false});
writeDB("rooms_paths", rooms.map(hashStr), {hash: false});

const roomsSnapShot = rooms.map((room) => writeDB(room, computeRoomTimetable(room, timetables), {
compare: TimetableRepository.compareDBTimetable
}));
const roomsNewChanges = roomsSnapShot.filter(({ similarity })=> similarity == 'different')
writeDB("rooms_new_changes", roomsNewChanges.map(({ content })=> hashStr(content.uid)), { hash: false })
const roomsSnapShot = rooms.map((room) =>
writeDB(room, computeRoomTimetable(room, timetables), {
compare: TimetableRepository.compareDBTimetable,
})
);
const roomsNewChanges = roomsSnapShot.filter(({similarity}) => similarity == "different");
writeDB(
"rooms_new_changes",
roomsNewChanges.map(({content}) => hashStr(content.uid)),
{hash: false}
);

return this;
}

public static compareDBTimetable(curr: TimetableDocType, dbContent: string) {
if(!isJsonString(dbContent)) return false;
const timetable = JSON.parse(dbContent) as any
if(!Object.hasOwn(timetable, 'timetable' as (keyof TimetableDocType))) return false;
return JSON.stringify(curr.timetable) === JSON.stringify((timetable as TimetableDocType).timetable);

private static compareDBTimetable(curr: TimetableDocType, dbContent: string) {
if (!isJsonString(dbContent)) return false;
const timetable = JSON.parse(dbContent) as any;
if (!Object.hasOwn(timetable, "timetable" as keyof TimetableDocType)) return false;
return (
JSON.stringify(TimetableRepository.removeSameTimingLectures(curr.timetable)) ===
JSON.stringify(TimetableRepository.removeSameTimingLectures((timetable as TimetableDocType).timetable))
);
}

/**
* Helper method to remove same timing lectures.
* On the Official Website we have scenario where one teacher have two lecture at a same time
* @param timetable
*/
private static removeSameTimingLectures(timetable: TimetableType) {
WEEK_DAYS_NAME.forEach((week) => {
timetable[week] = timetable[week].filter((lecture, idx, self) => {
return !self
.slice(idx + 1)
.some(
(other) =>
other.startTime.hours === lecture.startTime.hours &&
other.startTime.minutes === lecture.startTime.minutes &&
other.endTime.hours === lecture.endTime.hours &&
other.endTime.minutes === lecture.endTime.minutes
);
});
});
}
}

Expand Down
32 changes: 26 additions & 6 deletions src/lib/computes.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import TimetableDocType from "../types/TimetableDocType";
import { WEEK_DAYS_NAME } from "../constants";
import {WEEK_DAYS_NAME} from "../constants";

export function computeTeachers(timetables: TimetableDocType[]) {
const teachers = timetables
Expand All @@ -19,8 +19,10 @@ export function computeTeacherTimetable(teacher: string, timetables: TimetableDo
uid: teacher,
timetable: {},
};

WEEK_DAYS_NAME.forEach(name => { teacherTimetable.timetable[name]= [] })

WEEK_DAYS_NAME.forEach((name) => {
teacherTimetable.timetable[name] = [];
});

timetables.forEach(({uid, timetable}) => {
Object.entries(timetable).forEach(([day, lectures]) => {
Expand All @@ -33,7 +35,15 @@ export function computeTeacherTimetable(teacher: string, timetables: TimetableDo
});
});
});


// sorts lectures
WEEK_DAYS_NAME.forEach((day) => {
teacherTimetable.timetable[day] = teacherTimetable.timetable[day].sort(
(lhs, rhs) =>
lhs.startTime.hours * 60 + lhs.startTime.minutes - (rhs.startTime.hours * 60 + rhs.startTime.minutes)
);
});

return teacherTimetable;
}

Expand All @@ -56,7 +66,9 @@ export function computeRoomTimetable(roomNo: string, timetables: TimetableDocTyp
timetable: {},
};

WEEK_DAYS_NAME.forEach(name => { roomTimetable.timetable[name]= [] })
WEEK_DAYS_NAME.forEach((name) => {
roomTimetable.timetable[name] = [];
});

timetables.forEach(({uid, timetable}) => {
Object.entries(timetable).forEach(([day, lectures]) => {
Expand All @@ -69,6 +81,14 @@ export function computeRoomTimetable(roomNo: string, timetables: TimetableDocTyp
});
});
});


// sorts lectures
WEEK_DAYS_NAME.forEach((day) => {
roomTimetable.timetable[day] = roomTimetable.timetable[day].sort(
(lhs, rhs) =>
lhs.startTime.hours * 60 + lhs.startTime.minutes - (rhs.startTime.hours * 60 + rhs.startTime.minutes)
);
});

return roomTimetable;
}
4 changes: 2 additions & 2 deletions src/lib/logger.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import TimetableDocType from "../types/TimetableDocType";
import pc from "picocolors"
import pc from "picocolors";

export function logOnCrawlTimetable(timetable: TimetableDocType) {
console.log(`🎯 ${pc.cyan('successfully crawled')} ${pc.magenta(timetable.uid)}`)
console.log(`🎯 ${pc.cyan("successfully crawled")} ${pc.magenta(timetable.uid)}`);
}
6 changes: 5 additions & 1 deletion src/lib/util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ export function clamp(num: number, min: number, max: number) {
}

export function isJsonString(str: string) {
try { JSON.parse(str); } catch (e) { return false; }
try {
JSON.parse(str);
} catch (e) {
return false;
}
return true;
}
4 changes: 2 additions & 2 deletions src/lib/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ class Worker {
}

public constructor() {
this.availableCores = clamp(os.cpus().length, 2, 120);
this.availableCores = clamp(os.cpus().length, 2, 2);
this.onCrawlMetaDataEvent = new EventEmitter();
this.onCrawlTimetableEvent = new EventEmitter();
this.onCrawlFinishEvent = new EventEmitter();
Expand All @@ -50,7 +50,7 @@ class Worker {
this.registerWorkers();
return this;
}

private scrapMetaData() {
if (!cluster.isPrimary) return;

Expand Down
4 changes: 2 additions & 2 deletions src/local-db/cipher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {ENV} from "../constants";
import assert from "assert";

export const CIPHER_ALGO = "aes-256-cbc";
export const ENCRYPTED_DATA_ENCODING: Encoding = "hex"
export const ENCRYPTED_DATA_ENCODING: Encoding = "hex";

const getCredentials = () => ({
key: Buffer.from(ENV.OPEN_DB_KEY || "", "hex"),
Expand All @@ -15,7 +15,7 @@ export function encrypt(data: string) {
const cipher = createCipheriv(CIPHER_ALGO, key, iv);
var crypted = cipher.update(data, "utf8", ENCRYPTED_DATA_ENCODING);
crypted += cipher.final("hex");
assert(data === decrypt(crypted), '❌ decryption fail')
assert(data === decrypt(crypted), "❌ decryption fail");
return crypted;
}

Expand Down
9 changes: 3 additions & 6 deletions src/local-db/index.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
import initLocalDB from "./initDB";
import { writeDB } from "./writeDB";
import { readDB } from "./readDB";
import {writeDB} from "./writeDB";
import {readDB} from "./readDB";

initLocalDB();

export {
writeDB, readDB
}

export {writeDB, readDB};
Loading

0 comments on commit 26cc890

Please sign in to comment.