From 6249e3f38b676ba8b49f0cae72818831ddcc9d9f Mon Sep 17 00:00:00 2001 From: activus-d Date: Mon, 16 Dec 2024 04:17:40 +0100 Subject: [PATCH 1/2] create a fetch data function in a seperate file to combine the fetch logic in the open-source page --- .../open-source/(components)/fetch-data.ts | 28 +++++++++++ .../open-source/(components)/get-commits.ts | 46 ++----------------- .../(components)/get-contributors.ts | 31 +++---------- .../open-source/(components)/get-forks.ts | 46 ++----------------- .../open-source/(components)/get-issues.ts | 30 ++---------- .../open-source/(components)/get-stars.ts | 45 ++---------------- 6 files changed, 52 insertions(+), 174 deletions(-) create mode 100644 website/src/app/[lang]/[region]/(website)/open-source/(components)/fetch-data.ts diff --git a/website/src/app/[lang]/[region]/(website)/open-source/(components)/fetch-data.ts b/website/src/app/[lang]/[region]/(website)/open-source/(components)/fetch-data.ts new file mode 100644 index 000000000..739ffda45 --- /dev/null +++ b/website/src/app/[lang]/[region]/(website)/open-source/(components)/fetch-data.ts @@ -0,0 +1,28 @@ +export async function fetchData(owner: string, repo: string, url: string) { + const headers: Record = { + Accept: 'application/vnd.github+json', + }; + // Conditionally add the Authorization header if GITHUB_PAT is available + if (process.env.GITHUB_PAT) { + headers['Authorization'] = `Bearer ${process.env.GITHUB_PAT}`; + } + const res = await fetch(url, { + headers, + }); + + if (!res.ok) { + const errorDetails = await res.text(); + const status = res.status; + if (status === 403) { + throw new Error( + 'GitHub API rate limit exceeded. Please try again later or increase rate limit by authenticating.', + ); + } else if (status === 404) { + throw new Error(`GitHub repository ${owner}/${repo} not found.`); + } else { + throw new Error(`Failed to fetch recent commits from GitHub: ${status} - ${errorDetails}`); + } + } + + return res; +} diff --git a/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-commits.ts b/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-commits.ts index c026f46a2..6bbed4d4e 100644 --- a/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-commits.ts +++ b/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-commits.ts @@ -1,3 +1,5 @@ +import { fetchData } from './fetch-data'; + const owner = 'socialincome-san'; const repo = 'public'; @@ -23,50 +25,12 @@ export async function getCommits() { // Fetch recent commits from the last 30 days const commitUrl = `https://api.github.com/repos/${owner}/${repo}/commits?since=${startDateISO}&until=${endDate}`; - const headers: Record = { - Accept: 'application/vnd.github+json', - }; - // Conditionally add the Authorization header if GITHUB_PAT is available - if (process.env.GITHUB_PAT) { - headers['Authorization'] = `Bearer ${process.env.GITHUB_PAT}`; - } - const res = await fetch(commitUrl, { - headers, - }); - - if (!res.ok) { - const errorDetails = await res.text(); - const status = res.status; - if (status === 403) { - throw new Error( - 'GitHub API rate limit exceeded. Please try again later or increase rate limit by authenticating.', - ); - } else if (status === 404) { - throw new Error(`GitHub repository ${owner}/${repo} not found.`); - } else { - throw new Error(`Failed to fetch recent commits from GitHub: ${status} - ${errorDetails}`); - } - } - - const recentCommits: GitHubCommit[] = await res.json(); + const recentCommitsRes = await fetchData(owner, repo, commitUrl); + const recentCommits: GitHubCommit[] = await recentCommitsRes.json(); // Fetch total commit count const totalCommitsUrl = `https://api.github.com/repos/${owner}/${repo}/commits?per_page=1`; - const totalCommitsRes = await fetch(totalCommitsUrl, { - headers, - }); - - if (!totalCommitsRes.ok) { - const errorDetails = await totalCommitsRes.text(); - const status = totalCommitsRes.status; - if (status === 403) { - throw new Error(`GitHub API rate limit exceeded: ${status} - ${errorDetails}.`); - } else if (status === 404) { - throw new Error(`GitHub repository ${owner}/${repo} not found while fetching total commits.`); - } else { - throw new Error(`Failed to fetch total commits from GitHub: ${status} - ${errorDetails}`); - } - } + const totalCommitsRes = await fetchData(owner, repo, totalCommitsUrl); // Extract the last page number from the Link header to get the total commit count const linkHeader = totalCommitsRes.headers.get('link'); diff --git a/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-contributors.ts b/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-contributors.ts index 6341993ec..a98f7e670 100644 --- a/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-contributors.ts +++ b/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-contributors.ts @@ -1,3 +1,5 @@ +import { fetchData } from './fetch-data'; + const owner = 'socialincome-san'; const repo = 'public'; @@ -17,31 +19,10 @@ interface GitHubContributor { total: number; } -export async function getContributors(): Promise { - const headers: Record = { - Accept: 'application/vnd.github+json', - }; - // Conditionally add the Authorization header if GITHUB_PAT is available - if (process.env.GITHUB_PAT) { - headers['Authorization'] = `Bearer ${process.env.GITHUB_PAT}`; - } - const res = await fetch(`https://api.github.com/repos/${owner}/${repo}/stats/contributors`, { - headers, - }); - - if (!res.ok) { - const errorDetails = await res.text(); - const status = res.status; - if (status === 403) { - throw new Error(`GitHub API rate limit exceeded: ${status} - ${errorDetails}.`); - } else if (status === 404) { - throw new Error(`GitHub repository ${owner}/${repo} not found.`); - } else { - throw new Error(`Failed to fetch contributors from GitHub: ${status} - ${errorDetails}`); - } - } - - const contributors = await res.json(); +export async function getContributors() { + const url = `https://api.github.com/repos/${owner}/${repo}/stats/contributors`; + const contributorsRes = await fetchData(owner, repo, url); + const contributors = await contributorsRes.json(); if (Object.keys(contributors).length === 0) { console.warn('No contributor data available. The API returned an empty object.'); diff --git a/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-forks.ts b/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-forks.ts index 34950a803..fbce1bf9c 100644 --- a/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-forks.ts +++ b/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-forks.ts @@ -1,3 +1,5 @@ +import { fetchData } from './fetch-data'; + const owner = 'socialincome-san'; const repo = 'public'; @@ -8,28 +10,8 @@ interface GitHubFork { export async function getForkCount(): Promise<{ totalForks: number; newForks: number }> { const repoUrl = `https://api.github.com/repos/${owner}/${repo}`; - const headers: Record = { - Accept: 'application/vnd.github+json', - }; - // Conditionally add the Authorization header if GITHUB_PAT is available - if (process.env.GITHUB_PAT) { - headers['Authorization'] = `Bearer ${process.env.GITHUB_PAT}`; - } - const repoRes = await fetch(repoUrl, { headers }); - - if (!repoRes.ok) { - const errorDetails = await repoRes.text(); - const status = repoRes.status; - if (status === 403) { - throw new Error(`GitHub API rate limit exceeded: ${status} - ${errorDetails}.`); - } else if (status === 404) { - throw new Error(`GitHub repository ${owner}/${repo} not found.`); - } else { - throw new Error(`Failed to fetch repository info from GitHub: ${status} - ${errorDetails}`); - } - } - - const repoData = await repoRes.json(); + const repoDataRes = await fetchData(owner, repo, repoUrl); + const repoData = await repoDataRes.json(); const totalForks = repoData.forks_count; // Calculate the date 30 days ago from today @@ -44,25 +26,7 @@ export async function getForkCount(): Promise<{ totalForks: number; newForks: nu let hasMore = true; while (hasMore) { - const pagedRes = await fetch(`${forksUrl}&page=${page}`, { - headers, - }); - - if (!pagedRes.ok) { - const errorDetails = await pagedRes.text(); - const status = pagedRes.status; - - if (status === 403 && errorDetails.includes('API rate limit exceeded')) { - throw new Error( - 'GitHub API rate limit exceeded during forks fetching. Please try again later or increase rate limit by authenticating.', - ); - } else if (status === 404) { - throw new Error(`GitHub repository ${owner}/${repo} forks not found.`); - } else { - throw new Error(`Failed to fetch forks from GitHub: ${status} - ${errorDetails}`); - } - } - + const pagedRes = await fetchData(owner, repo, `${forksUrl}&page=${page}`); const forks: GitHubFork[] = await pagedRes.json(); // Count new forks within the last 30 days diff --git a/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-issues.ts b/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-issues.ts index 7c44887e4..30ac74c5b 100644 --- a/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-issues.ts +++ b/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-issues.ts @@ -1,3 +1,5 @@ +import { fetchData } from './fetch-data'; + const owner = 'socialincome-san'; const repo = 'public'; @@ -14,38 +16,14 @@ interface IssuesResponse { } export async function getIssuesData(): Promise { - const headers: Record = { - Accept: 'application/vnd.github+json', - }; - if (process.env.GITHUB_PAT) { - headers['Authorization'] = `Bearer ${process.env.GITHUB_PAT}`; - } - const issues: Issue[] = []; const labels: string[] = []; let page = 1; let hasMore = true; while (hasMore) { - const res = await fetch( - `https://api.github.com/repos/${owner}/${repo}/issues?state=open&per_page=100&page=${page}`, - { headers }, - ); - - if (!res.ok) { - const errorDetails = await res.text(); - const status = res.status; - if (status === 403) { - throw new Error( - 'GitHub API rate limit exceeded. Please try again later or increase rate limit by authenticating.', - ); - } else if (status === 404) { - throw new Error(`GitHub repository ${owner}/${repo} not found.`); - } else { - throw new Error(`Failed to fetch issues from GitHub: ${status} - ${errorDetails}`); - } - } - + const url = `https://api.github.com/repos/${owner}/${repo}/issues?state=open&per_page=100&page=${page}`; + const res = await fetchData(owner, repo, url); const data = await res.json(); // Break if no more issues diff --git a/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-stars.ts b/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-stars.ts index 9f7cfa586..201db7006 100644 --- a/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-stars.ts +++ b/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-stars.ts @@ -1,3 +1,5 @@ +import { fetchData } from './fetch-data'; + const owner = 'socialincome-san'; const repo = 'public'; @@ -11,28 +13,7 @@ interface GitHubStar { export async function getStarCount(): Promise<{ totalStars: number; newStars: number }> { const repoUrl = `https://api.github.com/repos/${owner}/${repo}`; - const headers: Record = { - Accept: 'application/vnd.github.star+json', - }; - // Conditionally add the Authorization header if GITHUB_PAT is available - if (process.env.GITHUB_PAT) { - headers['Authorization'] = `Bearer ${process.env.GITHUB_PAT}`; - } - const repoRes = await fetch(repoUrl, { - headers, - }); - - if (!repoRes.ok) { - const errorDetails = await repoRes.text(); - const status = repoRes.status; - if (status === 403) { - throw new Error(`GitHub API rate limit exceeded: ${status} - ${errorDetails}.`); - } else if (status === 404) { - throw new Error(`GitHub repository ${owner}/${repo} not found.`); - } else { - throw new Error(`Failed to fetch repository info from GitHub: ${status} - ${errorDetails}`); - } - } + const repoRes = await fetchData(owner, repo, repoUrl); const repoData = await repoRes.json(); const totalStars = repoData.stargazers_count; @@ -49,25 +30,7 @@ export async function getStarCount(): Promise<{ totalStars: number; newStars: nu let hasMore = true; while (hasMore) { - const pagedRes = await fetch(`${starUrl}&page=${page}`, { - headers, - }); - - if (!pagedRes.ok) { - const errorDetails = await pagedRes.text(); - const status = pagedRes.status; - - if (status === 403 && errorDetails.includes('API rate limit exceeded')) { - throw new Error( - 'GitHub API rate limit exceeded during stargazers fetching. Please try again later or increase rate limit by authenticating.', - ); - } else if (status === 404) { - throw new Error(`GitHub repository ${owner}/${repo} stargazers not found.`); - } else { - throw new Error(`Failed to fetch stargazers from GitHub: ${status} - ${errorDetails}`); - } - } - + const pagedRes = await fetchData(owner, repo, `${starUrl}&page=${page}`); const stars: GitHubStar[] = await pagedRes.json(); // Count new stars within the last 30 days From c79c2a9c23a945fcdb66c133a86886d0692f44c6 Mon Sep 17 00:00:00 2001 From: activus-d Date: Mon, 16 Dec 2024 20:37:29 +0100 Subject: [PATCH 2/2] replace the '/stats/contributors' endpoint with ' /contributors' endpoint to avoid timing issue during the computation phase on github servers --- .../(components)/get-contributors.ts | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-contributors.ts b/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-contributors.ts index a98f7e670..061caac9a 100644 --- a/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-contributors.ts +++ b/website/src/app/[lang]/[region]/(website)/open-source/(components)/get-contributors.ts @@ -11,16 +11,14 @@ interface Contributor { } interface GitHubContributor { - author: { - id: number; - login: string; - avatar_url: string; - }; - total: number; + id: number; + login: string; + avatar_url: string; + contributions: number; } export async function getContributors() { - const url = `https://api.github.com/repos/${owner}/${repo}/stats/contributors`; + const url = `https://api.github.com/repos/${owner}/${repo}/contributors`; const contributorsRes = await fetchData(owner, repo, url); const contributors = await contributorsRes.json(); @@ -35,10 +33,10 @@ export async function getContributors() { return contributors .map((contributor: GitHubContributor) => ({ - id: contributor.author.id, - name: contributor.author.login, - commits: contributor.total, - avatarUrl: contributor.author.avatar_url, + id: contributor.id, + name: contributor.login, + commits: contributor.contributions, + avatarUrl: contributor.avatar_url, })) .sort((a: Contributor, b: Contributor) => b.commits - a.commits); }