From 1df4d7af3dcca227f9b13333fa772b52df0ece4f Mon Sep 17 00:00:00 2001 From: dgudim Date: Sat, 2 Mar 2024 17:16:24 +0200 Subject: [PATCH] Update static, migrate to biome --- autogen/biome.json | 17 +++ autogen/index.ts | 137 +++++++++--------- autogen/package.json | 5 +- .../templates/parts/projects_content.html.j2 | 4 +- projects.html | 4 +- 5 files changed, 91 insertions(+), 76 deletions(-) create mode 100644 autogen/biome.json diff --git a/autogen/biome.json b/autogen/biome.json new file mode 100644 index 0000000..c695edf --- /dev/null +++ b/autogen/biome.json @@ -0,0 +1,17 @@ +{ + "organizeImports": { + "enabled": true + }, + "linter": { + "enabled": true, + "rules": { + "recommended": true + } + }, + "formatter": { + "enabled": true, + "indentStyle": "space", + "indentWidth": 4, + "lineWidth": 120 + } +} diff --git a/autogen/index.ts b/autogen/index.ts index 4342cd3..ea23b6d 100644 --- a/autogen/index.ts +++ b/autogen/index.ts @@ -1,41 +1,38 @@ -import puppeteer, { Browser, Page } from "puppeteer"; +import puppeteer from "puppeteer"; import fs from "fs"; import path from "path"; -import { fileURLToPath } from 'url'; +import { fileURLToPath } from "url"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -import util from 'util'; +import util from "util"; import { exec as ex } from "child_process"; const exec = util.promisify(ex); import js_beautify from "js-beautify"; -let browser: Browser; -let page: Page; -const github_user_url = "https://github.com/dgudim" -const github_user_repos_url = `${github_user_url}?tab=repositories` +const github_user_url = "https://github.com/dgudim"; +const github_user_repos_url = `${github_user_url}?tab=repositories`; console.log("Starting puppeteer"); -browser = await puppeteer.launch({ +const browser = await puppeteer.launch({ args: ["--no-sandbox", "--disable-setuid-sandbox"], }); -page = await browser.newPage(); +const page = await browser.newPage(); console.log("Puppeteer started"); await page.goto(github_user_repos_url); console.log(`Loaded ${github_user_repos_url}`); -function fetchRepos(selector: string) { - return page.evaluate((selector) => { - - function normalize(text: string | null | undefined): string { - return text?.replaceAll("\n", "").trim() ?? ""; - } +function fetchRepos(selector: string, title_selector = '[itemprop="name codeRepository"]') { + return page.evaluate( + (selector, title_selector) => { + function normalize(text: string | null | undefined): string { + return text?.replaceAll("\n", "").trim() ?? ""; + } - return Array.from(document.querySelectorAll(selector)) - .map(elem => { - const username_and_repo = elem.querySelector('[itemprop="name codeRepository"]')?.getAttribute("href"); + return Array.from(document.querySelectorAll(selector)).map((elem) => { + const username_and_repo = elem.querySelector(title_selector)?.getAttribute("href"); const name = username_and_repo?.split("/").pop(); return { url: `https://github.com${username_and_repo}`, @@ -43,14 +40,20 @@ function fetchRepos(selector: string) { name: name, description: normalize(elem.querySelector('[itemprop="description"]')?.textContent), lang: normalize(elem.querySelector('[itemprop="programmingLanguage"]')?.textContent), - lang_color: elem.querySelector(".repo-language-color")?.getAttribute("style")?.replace("background-color: ", ""), - stars: normalize(elem.querySelector(".octicon-star")?.parentElement?.textContent) || "0", - forks: normalize(elem.querySelector(".octicon-repo-forked")?.parentElement?.textContent) || "0", + lang_color: elem + .querySelector(".repo-language-color") + ?.getAttribute("style") + ?.replace("background-color: ", ""), + stars: normalize(elem.querySelector(`a[href="${username_and_repo}/stargazers"]`)?.textContent) || "0", + forks: normalize(elem.querySelector(`a[href="${username_and_repo}/forks"]`)?.textContent) || "0", org_name: "", - org_icon: "" - } + org_icon: "", + }; }); - }, selector); + }, + selector, + title_selector, + ); } let repos = await fetchRepos("#user-repositories-list li.col-12"); @@ -61,26 +64,24 @@ await page.goto(github_user_url); console.log(`Loaded ${github_user_url}`); const orgs = await page.evaluate(() => { - - return Array.from(document.querySelectorAll('[itemprop="follows"]')) - .map(elem => { - const orgName = elem.getAttribute("href"); - const icon_img = elem.querySelector("img"); - return { - repos_url: `https://github.com/orgs${orgName}/repositories`, - icon: icon_img?.src, - name: icon_img?.alt - } - }); + return Array.from(document.querySelectorAll('[itemprop="follows"]')).map((elem) => { + const orgName = elem.getAttribute("href"); + const icon_img = elem.querySelector("img"); + return { + repos_url: `https://github.com/orgs${orgName}/repositories`, + icon: icon_img?.src, + name: icon_img?.alt, + }; + }); }); console.log(`Fetched ${orgs.length} orgs: `); -for (let org of orgs) { +for (const org of orgs) { console.log(`Fetching ${org.name}`); await page.goto(org.repos_url); - const org_repos = await fetchRepos(".org-repos.repo-list .Box-row"); + const org_repos = await fetchRepos('li[id*="-listview-node-"]', 'a[data-testid="listitem-title-link"]'); console.log(`Fetched ${org_repos.length} repos from ${org.name}`); - for (let org_repo of org_repos) { + for (const org_repo of org_repos) { org_repo.org_name = org.name ?? ""; org_repo.org_icon = org.icon ?? ""; } @@ -88,23 +89,23 @@ for (let org of orgs) { } type RepoData = { - url: string, - username_and_repo: string, - description: string, - lang: string, - lang_color: string, - stars: string, - forks: string, - project_name: string, - html_id: string, - icon: string | undefined, - thumbnail: string, - org_name: string | undefined, - org_icon: string | undefined -} - -const repos_full = (await Promise.allSettled(repos.map(repo => { - return new Promise(async (resolve, reject) => { + url: string; + username_and_repo: string; + description: string; + lang: string; + lang_color: string; + stars: string; + forks: string; + project_name: string; + html_id: string; + icon: string | undefined; + thumbnail: string; + org_name: string | undefined; + org_icon: string | undefined; +}; + +const repo_tasks = repos.map((repo) => { + return (async () => { const page = await browser.newPage(); await page.goto(repo.url, { timeout: 0 }); console.log(`Loaded ${repo.url}`); @@ -122,20 +123,22 @@ const repos_full = (await Promise.allSettled(repos.map(repo => { html_id: project_name.toLowerCase().replaceAll(" ", "_"), icon: document.querySelector("#user-content-icon")?.getAttribute("src"), thumbnail: document.querySelector("#user-content-thumb")?.getAttribute("src"), - org_name: repo.org_name.length == 0 ? undefined : repo.org_name, - org_icon: repo.org_icon.length == 0 ? undefined : repo.org_icon + org_name: repo.org_name.length === 0 ? undefined : repo.org_name, + org_icon: repo.org_icon.length === 0 ? undefined : repo.org_icon, } as RepoData; }, repo); await page.close(); console.log(`Closed ${repo.url}, loaded additional data`); - if (repo_full.icon) { - resolve(repo_full); - } else { - reject("Icon is null"); + if (!repo_full.icon) { + throw new Error("Icon is null"); } - }); -}))).filter(repo => repo.status == "fulfilled").map(repo => (repo as PromiseFulfilledResult).value); + return repo_full; + })(); +}); +const repos_full = (await Promise.allSettled(repo_tasks)) + .filter((repo) => repo.status === "fulfilled") + .map((repo) => (repo as PromiseFulfilledResult).value); console.log(`Final repo list (${repos_full.length}), constructing page`); @@ -177,12 +180,12 @@ for (const repo_data of repos_full) {
${repo_data.forks} -
` + `; if (repo_data.org_icon) { htmlPage += `
org_icon ${repo_data.org_name} -
` + `; } htmlPage += ` @@ -219,7 +222,3 @@ await run(`python ${genFile}`); await run(`bash ${scssFile}`); process.exit(0); - - - - diff --git a/autogen/package.json b/autogen/package.json index 70f0a01..6c31bdd 100644 --- a/autogen/package.json +++ b/autogen/package.json @@ -5,7 +5,7 @@ "main": "index.ts", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", - "start": "ts-node index.ts" + "start": "bun index.ts" }, "repository": { "type": "git", @@ -23,7 +23,6 @@ }, "type": "module", "devDependencies": { - "@types/js-beautify": "^1.14.3", - "ts-node": "^10.9.2" + "@types/js-beautify": "^1.14.3" } } diff --git a/autogen/templates/parts/projects_content.html.j2 b/autogen/templates/parts/projects_content.html.j2 index 8a18e00..c356c15 100644 --- a/autogen/templates/parts/projects_content.html.j2 +++ b/autogen/templates/parts/projects_content.html.j2 @@ -22,11 +22,11 @@
- 0 + 1
- 0 + 1
diff --git a/projects.html b/projects.html index 4bece6f..7db2da0 100644 --- a/projects.html +++ b/projects.html @@ -119,11 +119,11 @@

Keyboard vis

- 0 + 1
- 0 + 1