Skip to content

Commit

Permalink
✔ Coba Debug Rss Scheduler ~
Browse files Browse the repository at this point in the history
  • Loading branch information
@bifeldy
bifeldy committed Apr 2, 2024
1 parent 72b2713 commit 8a1dbfd
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 61 deletions.
2 changes: 1 addition & 1 deletion dist/main-site/server/main.js
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion projects/main-site/src/api/controllers/berkas.controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ export class BerkasController {
attachment.user_ = user;
berkas.attachment_ = await this.attachmentRepo.save(attachment);
} else {
throw 'Lampiran Tidak Ditemukan / Dalam Proses Penggabungan!';
throw new Error('Lampiran Tidak Ditemukan / Dalam Proses Penggabungan!');
}
}
} catch (error) {
Expand Down
132 changes: 73 additions & 59 deletions projects/main-site/src/api/scheduler/rss-feed-tasks.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,72 +57,86 @@ export class RssFeedTasksService {
try {
let halaman = 1;
let totalNew = 0;
let retry = 0;
while (true) {
const url = new URL(fs.rss_feed);
const params = {
'start-index': halaman.toString(),
paged: halaman.toString(),
page: halaman.toString(),
alt: 'rss',
'max-results': '10',
row: '10'
};
for (const [key, value] of Object.entries(params)) {
if (!url.searchParams.has(key)) {
url.searchParams.append(key, value);
let noNew = null;
try {
const url = new URL(fs.rss_feed);
const params = {
'start-index': halaman.toString(),
paged: halaman.toString(),
page: halaman.toString(),
alt: 'rss',
'max-results': '10',
row: '10'
};
for (const [key, value] of Object.entries(params)) {
if (!url.searchParams.has(key)) {
url.searchParams.append(key, value);
}
}
}
const ru = url.toString();
const uri = new URL(`${environment.baseUrl}/api/crawl`);
uri.searchParams.append('url', ru);
const feed = await parse(uri.toString(), null);
if (feed.items.length <= 0) {
break;
}
let inserted = 0;
for (let i = 0; i < feed.items.length; i++) {
try {
const f = this.rssFeedRepo.new();
f.fansub_ = fs;
f.title = feed.items[i].title;
f.created_at = new Date(feed.items[i].created || feed.items[i].published);
const link = feed.items[i].link;
let url: string = null;
if (typeof link === 'string') {
url = new URL(link).pathname;
} else {
let idx = link.findIndex(l => l.rel === 'alternate' && l.type === 'text/html');
if (idx < 0) {
continue;
const ru = url.toString();
const uri = new URL(`${environment.baseUrl}/api/crawl`);
uri.searchParams.append('url', ru);
const feed = await parse(uri.toString(), null);
if (feed.items.length <= 0) {
break;
}
let inserted = 0;
for (let i = 0; i < feed.items.length; i++) {
try {
const f = this.rssFeedRepo.new();
f.fansub_ = fs;
f.title = feed.items[i].title;
f.created_at = new Date(feed.items[i].created || feed.items[i].published);
const link = feed.items[i].link;
let url: string = null;
if (typeof link === 'string') {
url = new URL(link).pathname;
} else {
let idx = link.findIndex(l => l.rel === 'alternate' && l.type === 'text/html');
if (idx < 0) {
continue;
}
url = new URL(link[idx].href).pathname;
}
f.link = url;
await this.rssFeedRepo.insert(f);
inserted++;
} catch (err) {
if (inserted === 0 && i >= feed.items.length - 1) {
retry = 2;
// Bypass Max Retry
throw new Error(`${url.origin} :: Tidak Ada RSS Feed Terbaru Dari ${i + 1} Data!`);
} else {
this.gs.log(`[CRON_TASK_FANSUB_RSS_FEED-ERROR_INSERT] 🐾 ${url.origin} ::`, err, 'error');
}
url = new URL(link[idx].href).pathname;
}
f.link = url;
await this.rssFeedRepo.insert(f);
inserted++;
} catch (err) {
if (inserted === 0 && i >= feed.items.length - 1) {
throw new Error(`Tidak Ada RSS Feed Terbaru Dari ${i + 1} Data!`);
}
}
}
totalNew += inserted;
let noNew = null;
if (fs.rss_feed.includes('/feeds/posts/default')) {
halaman += feed.items.length;
// Default Blogger Feed Per Page => 25 Items, But Forced To 10 Items
// So The Total Of Items Will Be 30 After Page 3 Loaded
// Blogger Paging Is Not A Page But Skip Offset Of Items
if (totalNew === 0 && halaman > 30) {
noNew = `Tidak Ada RSS Feed Terbaru Setelah Mengecek ${halaman / 10} Halaman!`;
totalNew += inserted;
if (fs.rss_feed.includes('/feeds/posts/default')) {
halaman += feed.items.length;
// Default Blogger Feed Per Page => 25 Items, But Forced To 10 Items
// So The Total Of Items Will Be 30 After Page 3 Loaded
// Blogger Paging Is Not A Page But Skip Offset Of Items
if (totalNew === 0 && halaman > 30) {
noNew = `${url.origin} :: Tidak Ada RSS Feed Terbaru Setelah Mengecek ${halaman / 10} Halaman!`;
}
} else {
halaman += 1;
// Default Universal Feed (Wordpress, etc) Per Page => 10 Items
// So The Total Of Items Will Be 30 After Page 3 Loaded
if (totalNew === 0 && halaman > 3) {
noNew = `${url.origin} :: Tidak Ada RSS Feed Terbaru Setelah Mengecek ${halaman} Halaman!`;
}
}
} else {
halaman += 1;
// Default Universal Feed (Wordpress, etc) Per Page => 10 Items
// So The Total Of Items Will Be 30 After Page 3 Loaded
if (totalNew === 0 && halaman > 3) {
noNew = `Tidak Ada RSS Feed Terbaru Setelah Mengecek ${halaman} Halaman!`;
} catch (er) {
retry++;
// Max Retry 1
if (retry > 1) {
throw er;
}
continue;
}
if (noNew) {
throw new Error(noNew);
Expand Down

0 comments on commit 8a1dbfd

Please sign in to comment.