Skip to content

Commit

Permalink
Merge pull request #104 from 0x4007/enhancement/cached-issues
Browse files Browse the repository at this point in the history 
Enhancement/cached issues
  • Loading branch information
@0x4007
0x4007 authored Oct 3, 2024
2 parents 831e182 + 2774938 commit db58076
Show file tree
Hide file tree
Showing 26 changed files with 206 additions and 456 deletions.
Binary file added bun.lockb
View file
Binary file not shown.
2 changes: 0 additions & 2 deletions src/home/authentication.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import { getGitHubUser } from "./getters/get-github-user";
import { GitHubUser } from "./github-types";
import { displayGitHubUserInformation } from "./rendering/display-github-user-information";
import { renderGitHubLoginButton } from "./rendering/render-github-login-button";
import { viewToggle } from "./fetch-github/fetch-and-display-previews";

export async function authentication() {
const accessToken = await getGitHubAccessToken();
Expand All @@ -16,6 +15,5 @@ export async function authentication() {
if (gitHubUser) {
trackDevRelReferral(gitHubUser.login + "|" + gitHubUser.id);
await displayGitHubUserInformation(gitHubUser);
viewToggle.disabled = false;
}
}
94 changes: 25 additions & 69 deletions src/home/fetch-github/fetch-and-display-previews.ts
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,19 @@
import { getGitHubAccessToken } from "../getters/get-github-access-token";
import { getImageFromCache } from "../getters/get-indexed-db";
import { getLocalStore } from "../getters/get-local-store";
import { GITHUB_TASKS_STORAGE_KEY, TaskStorageItems } from "../github-types";
import { GITHUB_TASKS_STORAGE_KEY, GitHubIssue, TaskStorageItems } from "../github-types";
import { taskManager } from "../home";
import { applyAvatarsToIssues, renderGitHubIssues } from "../rendering/render-github-issues";
import { Sorting } from "../sorting/generate-sorting-buttons";
import { sortIssuesController } from "../sorting/sort-issues-controller";
import { fetchAvatar } from "./fetch-avatar";
import { organizationImageCache } from "./fetch-issues-full";
import { fetchIssuePreviews } from "./fetch-issues-preview";
import { TaskMaybeFull, TaskNoFull, TaskWithFull } from "./preview-to-full-mapping";

export type Options = {
ordering: "normal" | "reverse";
};

let isProposalOnlyViewer = false; // or proposal viewer
let isProposalOnlyViewer = false;

export const viewToggle = document.getElementById("view-toggle") as HTMLInputElement;
if (!viewToggle) {
Expand All @@ -31,19 +29,16 @@ export async function fetchAndDisplayPreviewsFromCache(sorting?: Sorting, option
let _cachedTasks = getLocalStore(GITHUB_TASKS_STORAGE_KEY) as TaskStorageItems;
const _accessToken = await getGitHubAccessToken();

// Refresh the storage if there is no logged-in object in cachedTasks but there is one now.
if (_cachedTasks && !_cachedTasks.loggedIn && _accessToken) {
localStorage.removeItem(GITHUB_TASKS_STORAGE_KEY);
return fetchAndDisplayPreviewsFromNetwork(sorting, options);
return fetchAndDisplayIssuesFromNetwork(sorting, options);
}

// If previously logged in but not anymore, clear cache and fetch from network.
if (_cachedTasks && _cachedTasks.loggedIn && !_accessToken) {
localStorage.removeItem(GITHUB_TASKS_STORAGE_KEY);
return fetchAndDisplayPreviewsFromNetwork(sorting, options);
return fetchAndDisplayIssuesFromNetwork(sorting, options);
}

// makes sure tasks have a timestamp to know how old the cache is, or refresh if older than 15 minutes
if (!_cachedTasks || !_cachedTasks.timestamp || _cachedTasks.timestamp + 60 * 1000 * 15 <= Date.now()) {
_cachedTasks = {
timestamp: Date.now(),
Expand All @@ -52,34 +47,27 @@ export async function fetchAndDisplayPreviewsFromCache(sorting?: Sorting, option
};
}

const cachedTasks = _cachedTasks.tasks as TaskMaybeFull[];
const cachedTasks = _cachedTasks.tasks;
taskManager.syncTasks(cachedTasks);

if (!cachedTasks.length) {
// load from network if there are no cached issues
return fetchAndDisplayPreviewsFromNetwork(sorting, options);
return fetchAndDisplayIssuesFromNetwork(sorting, options);
} else {
displayGitHubIssues(sorting, options);
return fetchAvatars();
}
}

export async function fetchAndDisplayPreviewsFromNetwork(sorting?: Sorting, options = { ordering: "normal" }) {
const fetchedPreviews = await fetchIssuePreviews();
const cachedTasks = taskManager.getTasks();
const updatedCachedIssues = verifyGitHubIssueState(cachedTasks, fetchedPreviews);
taskManager.syncTasks(updatedCachedIssues);
export async function fetchAndDisplayIssuesFromNetwork(sorting?: Sorting, options = { ordering: "normal" }) {
displayGitHubIssues(sorting, options);
return fetchAvatars();
}

export async function fetchAvatars() {
const cachedTasks = taskManager.getTasks();
const urlPattern = /https:\/\/github\.com\/(?<org>[^/]+)\/(?<repo>[^/]+)\/issues\/(?<issue_number>\d+)/;

const avatarPromises = cachedTasks.map(async (task) => {
const match = task.preview.body?.match(urlPattern);
const orgName = match?.groups?.org;
const avatarPromises = cachedTasks.map(async (task: GitHubIssue) => {
const [orgName] = task.repository_url.split("/").slice(-2);
if (orgName) {
return fetchAvatar(orgName);
}
Expand All @@ -88,68 +76,36 @@ export async function fetchAvatars() {

await Promise.allSettled(avatarPromises);
applyAvatarsToIssues();
return cachedTasks;
}

export function taskWithFullTest(task: TaskNoFull | TaskWithFull): task is TaskWithFull {
return (task as TaskWithFull).full !== null && (task as TaskWithFull).full !== undefined;
}

export function verifyGitHubIssueState(cachedTasks: TaskMaybeFull[], fetchedPreviews: TaskNoFull[]): (TaskNoFull | TaskWithFull)[] {
return fetchedPreviews.map((fetched) => {
const cachedTask = cachedTasks.find((c) => c.full?.id === fetched.preview.id);
if (cachedTask) {
if (taskWithFullTest(cachedTask)) {
const cachedFullIssue = cachedTask.full;
const task = { ...fetched, full: cachedFullIssue };
return task;
} else {
// no full issue in task
}
} else {
// no cached task
}
return {
preview: fetched.preview,
} as TaskNoFull;
});
}

export function displayGitHubIssues(sorting?: Sorting, options = { ordering: "normal" }) {
// Load avatars from cache
const urlPattern = /https:\/\/github\.com\/(?<org>[^/]+)\/(?<repo>[^/]+)\/issues\/(?<issue_number>\d+)/;
const cachedTasks = taskManager.getTasks();
cachedTasks.forEach(async ({ preview }) => {
if (!preview.body) {
throw new Error(`Preview body is undefined for task with id: ${preview.id}`);
}
const match = preview.body.match(urlPattern);
const orgName = match?.groups?.org;
if (orgName) {
const avatarUrl = await getImageFromCache({ dbName: "GitHubAvatars", storeName: "ImageStore", orgName: `avatarUrl-${orgName}` });
if (avatarUrl) {
organizationImageCache.set(orgName, avatarUrl);
}
}
const cached = taskManager.getTasks();
cached.forEach(async (gitHubIssue) => {
const [orgName] = gitHubIssue.repository_url.split("/").slice(-2);

getImageFromCache({
dbName: "GitHubAvatars",
storeName: "ImageStore",
orgName: `avatarUrl-${orgName}`,
})
.then((avatarUrl) => organizationImageCache.set(orgName, avatarUrl))
.catch(console.error);
});

// Render issues
const sortedIssues = sortIssuesController(cachedTasks, sorting, options);
const sortedIssues = sortIssuesController(cached, sorting, options);
const sortedAndFiltered = sortedIssues.filter(getProposalsOnlyFilter(isProposalOnlyViewer));
renderGitHubIssues(sortedAndFiltered);
}

function getProposalsOnlyFilter(getProposals: boolean) {
return (task: TaskMaybeFull) => {
if (!task.full?.labels) return false;
return (issue: GitHubIssue) => {
if (!issue?.labels) return false;

const hasPriceLabel = task.full.labels.some((label) => {
const hasPriceLabel = issue.labels.some((label) => {
if (typeof label === "string") return false;
return label.name?.startsWith("Price: ") || label.name?.startsWith("Pricing: ");
return label.name?.startsWith("Price: ") || label.name?.startsWith("Price: ");
});

// If getProposals is true, we want tasks WITHOUT price labels
// If getProposals is false, we want tasks WITH price labels
return getProposals ? !hasPriceLabel : hasPriceLabel;
};
}
49 changes: 4 additions & 45 deletions src/home/fetch-github/fetch-issues-full.ts
View file
Original file line number Diff line number Diff line change
@@ -1,49 +1,8 @@
import { Octokit } from "@octokit/rest";
import { getGitHubAccessToken } from "../getters/get-github-access-token";
import { GitHubIssue } from "../github-types";
import { taskWithFullTest } from "./fetch-and-display-previews";
import { fetchAvatar } from "./fetch-avatar";
import { TaskMaybeFull, TaskWithFull } from "./preview-to-full-mapping";

export const organizationImageCache = new Map<string, Blob | null>();

export async function fetchIssuesFull(taskPreviews: TaskMaybeFull[]): Promise<TaskWithFull[]> {
const octokit = new Octokit({ auth: await getGitHubAccessToken() });
const urlPattern = /https:\/\/github\.com\/(?<org>[^/]+)\/(?<repo>[^/]+)\/issues\/(?<issue_number>\d+)/;

const fullTaskPromises = taskPreviews.map(async (task) => {
const match = task.preview.body.match(urlPattern);

if (!match || !match.groups) {
console.error("Invalid issue body URL format");
return Promise.resolve(null);
}

const { org, repo, issue_number } = match.groups;

const { data: response } = await octokit.request("GET /repos/{org}/{repo}/issues/{issue_number}", { issue_number, repo, org });

task.full = response as GitHubIssue;

const urlMatch = task.full.html_url.match(urlPattern);
const orgName = urlMatch?.groups?.org;
if (orgName) {
await fetchAvatar(orgName);
}
const isTaskWithFull = taskWithFullTest(task);

if (isTaskWithFull) {
return task;
} else {
throw new Error("Task is not a TaskWithFull");
}
});

const settled = await Promise.allSettled(fullTaskPromises);
const fullTasks = settled
.filter((result): result is PromiseFulfilledResult<TaskWithFull> => result.status === "fulfilled")
.map((result) => result.value)
.filter((issue): issue is TaskWithFull => issue !== null);

return fullTasks;
export async function fetchIssuesFull(): Promise<GitHubIssue[]> {
const response = await fetch("https://raw.githubusercontent.com/ubiquity/devpool-directory/refs/heads/development/devpool-issues.json");
const jsonData = await response.json();
return jsonData;
}
100 changes: 0 additions & 100 deletions src/home/fetch-github/fetch-issues-preview.ts
View file
Original file line number Diff line number Diff line change
@@ -1,104 +1,4 @@
import { RequestError } from "@octokit/request-error";
import { Octokit } from "@octokit/rest";
import { getGitHubAccessToken, getGitHubUserName } from "../getters/get-github-access-token";
import { GitHubIssue } from "../github-types";
import { displayPopupMessage } from "../rendering/display-popup-modal";
import { handleRateLimit } from "./handle-rate-limit";
import { TaskNoFull } from "./preview-to-full-mapping";

async function checkPrivateRepoAccess(): Promise<boolean> {
const octokit = new Octokit({ auth: await getGitHubAccessToken() });
const username = getGitHubUserName();

if (username) {
try {
const response = await octokit.repos.checkCollaborator({
owner: "ubiquity",
repo: "devpool-directory-private",
username,
});

if (response.status === 204) {
// If the response is successful, it means the user has access to the private repository
return true;
}
return false;
} catch (error) {
if (!!error && typeof error === "object" && "status" in error && (error.status === 404 || error.status === 401)) {
// If the status is 404, it means the user is not a collaborator, hence no access
return false;
} else {
// Handle other errors if needed
console.error("Error checking repository access:", error);
throw error;
}
}
}

return false;
}

export async function fetchIssuePreviews(): Promise<TaskNoFull[]> {
const octokit = new Octokit({ auth: await getGitHubAccessToken() });
let freshIssues: GitHubIssue[] = [];
let hasPrivateRepoAccess = false; // Flag to track access to the private repository

try {
// Check if the user has access to the private repository
hasPrivateRepoAccess = await checkPrivateRepoAccess();

// Fetch issues from public repository
const publicResponse = await octokit.paginate(octokit.issues.listForRepo, {
owner: "ubiquity",
repo: "devpool-directory",
state: "open",
});

const publicIssues = publicResponse.filter((issue: GitHubIssue) => !issue.pull_request);

// Fetch issues from the private repository only if the user has access
if (hasPrivateRepoAccess) {
await fetchPrivateIssues(publicIssues);
} else {
// If user doesn't have access, only load issues from the public repository
freshIssues = publicIssues;
}
} catch (error) {
if (!!error && typeof error === "object" && "status" in error && error.status === 403) {
await handleRateLimit(octokit, error as RequestError);
} else {
throw error;
}
}

const tasks = freshIssues.map((preview: GitHubIssue) => ({
preview: preview,
full: null,
isNew: true,
isModified: true,
})) as TaskNoFull[];

return tasks;

async function fetchPrivateIssues(publicIssues: GitHubIssue[]) {
const privateResponse = await octokit.paginate(octokit.issues.listForRepo, {
owner: "ubiquity",
repo: "devpool-directory-private",
state: "open",
});
const privateIssues = privateResponse.filter((issue: GitHubIssue) => !issue.pull_request);

// Mark private issues
// TODO: indicate private issues in the UI

// const privateIssuesWithFlag = privateIssues.map((issue) => {
// return issue;
// });

// Combine public and private issues
freshIssues = [...privateIssues, ...publicIssues];
}
}

export function rateLimitModal(message: string) {
displayPopupMessage({ modalHeader: `GitHub API rate limit exceeded.`, modalBody: message, isError: false });
Expand Down
8 changes: 4 additions & 4 deletions src/home/fetch-github/handle-rate-limit.ts
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import { RequestError } from "@octokit/request-error";
import { Octokit } from "@octokit/rest";
import { getGitHubUser } from "../getters/get-github-user";
import { toolbar } from "../ready-toolbar";
import { renderErrorInModal } from "../rendering/display-popup-modal";
import { rateLimitModal } from "./fetch-issues-preview";
import { gitHubLoginButton } from "../rendering/render-github-login-button";
import { preview } from "../rendering/render-preview-modal";
import { toolbar } from "../ready-toolbar";
import { modal } from "../rendering/render-preview-modal";
import { rateLimitModal } from "./fetch-issues-preview";

type RateLimit = {
reset: number | null;
Expand All @@ -18,7 +18,7 @@ export async function handleRateLimit(octokit?: Octokit, error?: RequestError) {
user: false,
};

preview.classList.add("active");
modal.classList.add("active");
document.body.classList.add("preview-active");

if (toolbar) {
Expand Down
21 changes: 0 additions & 21 deletions src/home/fetch-github/preview-to-full-mapping.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,24 +4,3 @@ export type TaskNoState = {
preview: GitHubIssue;
full: null | GitHubIssue;
};

export type TaskNoFull = {
preview: GitHubIssue;
full: null;
isNew: boolean;
isModified: boolean;
};

export type TaskMaybeFull = {
preview: GitHubIssue;
full: null | GitHubIssue;
isNew: boolean;
isModified: boolean;
};

export type TaskWithFull = {
preview: GitHubIssue;
full: GitHubIssue;
isNew: boolean;
isModified: boolean;
};
Loading

0 comments on commit db58076

Please sign in to comment.