From ed3ad2e9c09eaf1a920b8f48bc4570aaa63efd19 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Sat, 4 Jan 2025 01:37:48 -0500 Subject: [PATCH] fix: removing org subs check temp --- .github/sync-env-vars.sh | 2 +- functions/issue-scraper.ts | 481 +++++++++++++++++----------------- functions/referral-manager.ts | 6 +- functions/types.ts | 4 +- functions/validators.ts | 2 +- wrangler.toml | 7 +- 6 files changed, 253 insertions(+), 249 deletions(-) diff --git a/.github/sync-env-vars.sh b/.github/sync-env-vars.sh index 58fe2374..db504a60 100644 --- a/.github/sync-env-vars.sh +++ b/.github/sync-env-vars.sh @@ -24,7 +24,7 @@ curl -X PATCH \ "deployment_configs": { "production": { "env_vars": { - "VOYAGE_API_KEY": { + "VOYAGEAI_API_KEY": { "value": "'"${VOYAGEAI_API_KEY}"'", "type": "secret_text" }, diff --git a/functions/issue-scraper.ts b/functions/issue-scraper.ts index b558f90b..335d37ab 100644 --- a/functions/issue-scraper.ts +++ b/functions/issue-scraper.ts @@ -7,118 +7,118 @@ import plainTextPlugin from "markdown-it-plain-text"; import { validatePOST } from "./validators"; interface MarkdownItWithPlainText extends markdownit { - plainText: string; + plainText: string; } interface IssueMetadata { - nodeId: string; - number: number; - title: string; - body: string; - state: string; - repositoryName: string; - repositoryId: number; - assignees: string[]; - authorId: number; - createdAt: string; - closedAt: string | null; - stateReason: string | null; - updatedAt: string; + nodeId: string; + number: number; + title: string; + body: string; + state: string; + repositoryName: string; + repositoryId: number; + assignees: string[]; + authorId: number; + createdAt: string; + closedAt: string | null; + stateReason: string | null; + updatedAt: string; } interface IssueNode { - id: string; - number: number; - title: string; - body: string; - state: string; - stateReason: string | null; - createdAt: string; - updatedAt: string; - closedAt: string | null; - author: { - login: string; - } | null; - assignees: { - nodes: Array<{ - login: string; - }>; - }; - repository: { id: string; - name: string; - owner: { - login: string; + number: number; + title: string; + body: string; + state: string; + stateReason: string | null; + createdAt: string; + updatedAt: string; + closedAt: string | null; + author: { + login: string; + } | null; + assignees: { + nodes: Array<{ + login: string; + }>; + }; + repository: { + id: string; + name: string; + owner: { + login: string; + }; }; - }; } interface GraphQlSearchResponse { - search: { - pageInfo: { - hasNextPage: boolean; - endCursor: string | null; + search: { + pageInfo: { + hasNextPage: boolean; + endCursor: string | null; + }; + nodes: Array; }; - nodes: Array; - }; } export const corsHeaders = { - "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Methods": "GET", - "Access-Control-Allow-Headers": "Content-Type", + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET", + "Access-Control-Allow-Headers": "Content-Type", }; export async function onRequest(ctx: Context): Promise { - const { request, env } = ctx; + const { request, env } = ctx; + try { + switch (request.method) { + case "POST": { + const result = await validatePOST(request); + if (!result.isValid || !result.gitHubUser) { + return new Response("Unauthorized", { + headers: corsHeaders, + status: 400, + }); + } + const githubUserName = result.gitHubUser.login; + try { + const supabase = new SupabaseClient(env.SUPABASE_URL, env.SUPABASE_KEY); + const response = await issueScraper(githubUserName, supabase, env.VOYAGEAI_API_KEY, result.authToken); + return new Response(response, { + headers: corsHeaders, + status: 200, + }); + } catch (error) { + console.error("Error processing request:", error); + return new Response("Internal Server Error", { + headers: corsHeaders, + status: 500, + }); + } + } - try { - switch (request.method) { - case "POST": { - const result = await validatePOST(request); - if (!result.isValid || !result.gitHubUserId) { - return new Response("Unauthorized", { - headers: corsHeaders, - status: 400, - }); + default: + return new Response("Method Not Allowed", { + headers: corsHeaders, + status: 405, + }); } - try { - const supabase = new SupabaseClient(env.SUPABASE_URL, env.SUPABASE_ANON_KEY); - const response = await issueScraper(result.gitHubUserId, supabase, env.VOYAGEAI_API_KEY, result.authToken); - return new Response(response, { - headers: corsHeaders, - status: 200, - }); - } catch (error) { - console.error("Error processing request:", error); - return new Response("Internal Server Error", { + } catch (error) { + console.error("Error processing request:", error); + return new Response("Internal Server Error", { headers: corsHeaders, status: 500, - }); - } - } - - default: - return new Response("Method Not Allowed", { - headers: corsHeaders, - status: 405, }); } - } catch (error) { - console.error("Error processing request:", error); - return new Response("Internal Server Error", { - headers: corsHeaders, - status: 500, - }); - } } function markdownToPlainText(markdown: string | null): string | null { - if (!markdown) return markdown; - const md = markdownit() as MarkdownItWithPlainText; - md.use(plainTextPlugin); - md.render(markdown); - return md.plainText; + if (!markdown) return markdown; + const md = markdownit() as MarkdownItWithPlainText; + md.use(plainTextPlugin); + md.render(markdown); + return md.plainText; } const SEARCH_ISSUES_QUERY = ` @@ -166,181 +166,180 @@ const SEARCH_ISSUES_QUERY = ` `; async function fetchAuthorId(octokit: InstanceType, login: string): Promise { - try { - const response = await octokit.rest.users.getByUsername({ username: login }); - return response.data.id; - } catch (error) { - console.error(`Error fetching author ID for ${login}:`, error); - return -1; - } + try { + const response = await octokit.rest.users.getByUsername({ username: login }); + return response.data.id; + } catch (error) { + console.error(`Error fetching author ID for ${login}:`, error); + return -1; + } } async function fetchUserIssues(octokit: InstanceType, username: string): Promise { - const allIssues: IssueNode[] = []; - let hasNextPage = true; - let cursor: string | null = null; + const allIssues: IssueNode[] = []; + let hasNextPage = true; + let cursor: string | null = null; - const searchText = `assignee:${username} is:issue is:closed`; + const searchText = `assignee:${username} is:issue is:closed`; - while (hasNextPage) { - const variables: { searchText: string; after?: string } = { - searchText, - }; - if (cursor) { - variables.after = cursor; - } + while (hasNextPage) { + const variables: { searchText: string; after?: string } = { + searchText, + }; + if (cursor) { + variables.after = cursor; + } - const response: GraphQlSearchResponse = await octokit.graphql(SEARCH_ISSUES_QUERY, variables); + const response: GraphQlSearchResponse = await octokit.graphql(SEARCH_ISSUES_QUERY, variables); - const completedIssues = response.search.nodes.filter((issue) => issue.stateReason === "COMPLETED"); - allIssues.push(...completedIssues); + const completedIssues = response.search.nodes.filter((issue) => issue.stateReason === "COMPLETED"); + allIssues.push(...completedIssues); - hasNextPage = response.search.pageInfo.hasNextPage; - cursor = response.search.pageInfo.endCursor; + hasNextPage = response.search.pageInfo.hasNextPage; + cursor = response.search.pageInfo.endCursor; - if (!cursor) break; - } + if (!cursor) break; + } - return allIssues; + return allIssues; } // Pulls issues from GitHub and stores them in Supabase async function issueScraper(username: string, supabase: SupabaseClient, voyageApiKey: string, token?: string): Promise { - try { - if (!username) { - throw new Error("Username is required"); - } + try { + if (!username) { + throw new Error("Username is required"); + } - const context = { - adapters: {}, - logger: { - info: (message: string, data: Record) => console.log("INFO:", message + ":", data), - error: (message: string, data: Record) => console.error("ERROR:", message + ":", data), - }, - octokit: new Octokit(token ? { auth: token } : {}), - }; + const context = { + adapters: {}, + logger: { + info: (message: string, data: Record) => console.log("INFO:", message + ":", data), + error: (message: string, data: Record) => console.error("ERROR:", message + ":", data), + }, + octokit: new Octokit(token ? { auth: token } : {}), + }; - const voyageClient = new VoyageAIClient({ apiKey: voyageApiKey }); - const issues = await fetchUserIssues(context.octokit, username); - const processedIssues: Array<{ issue: IssueMetadata; error?: string }> = []; + const voyageClient = new VoyageAIClient({ apiKey: voyageApiKey }); + const issues = await fetchUserIssues(context.octokit, username); + const processedIssues: Array<{ issue: IssueMetadata; error?: string }> = []; + for (const issue of issues) { + try { + const authorId = issue.author?.login ? await fetchAuthorId(context.octokit, issue.author.login) : -1; + const repoOwner = issue.repository.owner.login; - for (const issue of issues) { - try { - const authorId = issue.author?.login ? await fetchAuthorId(context.octokit, issue.author.login) : -1; - const repoOwner = issue.repository.owner.login; + const metadata: IssueMetadata = { + nodeId: issue.id, + number: issue.number, + title: issue.title || "", + body: issue.body || "", + state: issue.state, + stateReason: issue.stateReason, + repositoryName: issue.repository.name, + repositoryId: parseInt(issue.repository.id), + assignees: (issue.assignees?.nodes || []).map((assignee) => assignee.login), + authorId, + createdAt: issue.createdAt, + closedAt: issue.closedAt, + updatedAt: issue.updatedAt, + }; + const markdown = metadata.body + " " + metadata.title; + const plaintext = markdownToPlainText(markdown); + if (!plaintext || plaintext === null) { + throw new Error("Error converting markdown to plaintext"); + } + const embeddingObject = await voyageClient.embed({ + input: markdown, + model: "voyage-large-2-instruct", + inputType: "document", + }); + const embedding = (embeddingObject.data && embeddingObject.data[0]?.embedding) || {}; + const payload = { + issue: metadata, + action: "created", + sender: { + login: username, + }, + repository: { + id: parseInt(issue.repository.id), + node_id: issue.repository.id, + name: issue.repository.name, + full_name: `${repoOwner}/${issue.repository.name}`, + owner: { + login: repoOwner, + id: authorId, + type: "User", + site_admin: false, + }, + }, + }; + //Check if the user is authenticated + if (!supabase.auth.getUser()) { + throw new Error("User is not authenticated"); + } - const metadata: IssueMetadata = { - nodeId: issue.id, - number: issue.number, - title: issue.title || "", - body: issue.body || "", - state: issue.state, - stateReason: issue.stateReason, - repositoryName: issue.repository.name, - repositoryId: parseInt(issue.repository.id), - assignees: (issue.assignees?.nodes || []).map((assignee) => assignee.login), - authorId, - createdAt: issue.createdAt, - closedAt: issue.closedAt, - updatedAt: issue.updatedAt, - }; - const markdown = metadata.body + " " + metadata.title; - const plaintext = markdownToPlainText(markdown); - if (!plaintext || plaintext === null) { - throw new Error("Error converting markdown to plaintext"); - } - const embeddingObject = await voyageClient.embed({ - input: markdown, - model: "voyage-large-2-instruct", - inputType: "document", - }); - const embedding = (embeddingObject.data && embeddingObject.data[0]?.embedding) || {}; - const payload = { - issue: metadata, - action: "created", - sender: { - login: username, - }, - repository: { - id: parseInt(issue.repository.id), - node_id: issue.repository.id, - name: issue.repository.name, - full_name: `${repoOwner}/${issue.repository.name}`, - owner: { - login: repoOwner, - id: authorId, - type: "User", - site_admin: false, - }, - }, - }; - //Check if the user is authenticated - if (!supabase.auth.getUser()) { - throw new Error("User is not authenticated"); - } + const { error } = await supabase.from("issues").upsert({ + id: metadata.nodeId, + markdown, + plaintext, + embedding: JSON.stringify(embedding), + author_id: metadata.authorId, + modified_at: metadata.updatedAt, + payload: payload, + }); - const { error } = await supabase.from("issues").upsert({ - id: metadata.nodeId, - markdown, - plaintext, - embedding: JSON.stringify(embedding), - author_id: metadata.authorId, - modified_at: metadata.updatedAt, - payload: payload, - }); + processedIssues.push({ + issue: metadata, + error: error ? `Error storing issue: ${error.message}` : undefined, + }); + } catch (error) { + processedIssues.push({ + issue: { + nodeId: issue.id, + number: issue.number, + title: issue.title || "", + body: issue.body || "", + state: issue.state, + stateReason: issue.stateReason, + repositoryName: issue.repository.name, + repositoryId: parseInt(issue.repository.id), + assignees: [], + authorId: -1, + createdAt: issue.createdAt, + closedAt: issue.closedAt, + updatedAt: issue.updatedAt, + }, + error: `Error processing issue: ${error instanceof Error ? error.message : "Unknown error"}`, + }); + } + } - processedIssues.push({ - issue: metadata, - error: error ? `Error storing issue: ${error.message}` : undefined, - }); - } catch (error) { - processedIssues.push({ - issue: { - nodeId: issue.id, - number: issue.number, - title: issue.title || "", - body: issue.body || "", - state: issue.state, - stateReason: issue.stateReason, - repositoryName: issue.repository.name, - repositoryId: parseInt(issue.repository.id), - assignees: [], - authorId: -1, - createdAt: issue.createdAt, - closedAt: issue.closedAt, - updatedAt: issue.updatedAt, - }, - error: `Error processing issue: ${error instanceof Error ? error.message : "Unknown error"}`, - }); - } + return JSON.stringify( + { + success: true, + stats: { + storageSuccessful: processedIssues.filter((p) => !p.error).length, + storageFailed: processedIssues.filter((p) => p.error).length, + }, + errors: processedIssues + .filter((p) => p.error) + .map((p) => ({ + type: "storage", + name: `${p.issue.repositoryName}#${p.issue.number}`, + error: p.error, + })), + issues: processedIssues.map((p) => ({ + number: p.issue.number, + title: p.issue.title, + repo: p.issue.repositoryName, + error: p.error, + })), + }, + null, + 2 + ); + } catch (error) { + console.error("Error in issueScraper:", error); + throw error; } - - return JSON.stringify( - { - success: true, - stats: { - storageSuccessful: processedIssues.filter((p) => !p.error).length, - storageFailed: processedIssues.filter((p) => p.error).length, - }, - errors: processedIssues - .filter((p) => p.error) - .map((p) => ({ - type: "storage", - name: `${p.issue.repositoryName}#${p.issue.number}`, - error: p.error, - })), - issues: processedIssues.map((p) => ({ - number: p.issue.number, - title: p.issue.title, - repo: p.issue.repositoryName, - error: p.error, - })), - }, - null, - 2 - ); - } catch (error) { - console.error("Error in issueScraper:", error); - throw error; - } } diff --git a/functions/referral-manager.ts b/functions/referral-manager.ts index f83ddd4b..0996173b 100644 --- a/functions/referral-manager.ts +++ b/functions/referral-manager.ts @@ -50,14 +50,16 @@ export async function onRequest(ctx: Context): Promise { async function handleSet(env: Env, request: Request): Promise { const result = await validatePOST(request); - if (!result.isValid || !result.gitHubUserId || !result.referralCode) { + if (!result.isValid || !result.gitHubUser || !result.referralCode) { return new Response("Unauthorized", { headers: corsHeaders, status: 400, }); } - const { gitHubUserId, referralCode } = result; + const { gitHubUser, referralCode } = result; + + const gitHubUserId = gitHubUser.id.toString(); const oldRefCode = await env.KVNamespace.get(gitHubUserId); diff --git a/functions/types.ts b/functions/types.ts index 5b06f3cf..b9e062e8 100644 --- a/functions/types.ts +++ b/functions/types.ts @@ -1,9 +1,11 @@ import { EventContext, KVNamespace } from "@cloudflare/workers-types"; +import { GitHubUser } from "../src/home/github-types"; export interface Env { KVNamespace: KVNamespace; SUPABASE_URL: string; SUPABASE_ANON_KEY: string; + SUPABASE_KEY: string; VOYAGEAI_API_KEY: string; } @@ -14,7 +16,7 @@ export interface POSTRequestBody { export interface ValidationResult { isValid: boolean; - gitHubUserId?: string; + gitHubUser?: GitHubUser; referralCode?: string; authToken?: string; } diff --git a/functions/validators.ts b/functions/validators.ts index 51335482..f608b407 100644 --- a/functions/validators.ts +++ b/functions/validators.ts @@ -15,7 +15,7 @@ export async function validatePOST(request: Request): Promise const gitHubUser = response.data; - return { isValid: true, gitHubUserId: gitHubUser.id.toString(), referralCode: referralCode }; + return { isValid: true, gitHubUser: gitHubUser, referralCode: referralCode, authToken: authToken }; } catch (error) { console.error("User is not logged in"); return { isValid: false }; diff --git a/wrangler.toml b/wrangler.toml index de7a09a0..771f2580 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -10,7 +10,8 @@ id = "0a6aaf0a6edb428189606b116da58ef7" [vars] YARN_VERSION = "1.22.22" -VOYAGE_API_KEY = "" -SUPABASE_URL = "" -SUPABASE_ANON_KEY = "" +SUPABASE_URL="" +SUPABASE_ANON_KEY="" +VOYAGEAI_API_KEY="" +SUPABASE_KEY=""