From 258427f900237bf3893b73ce0ba41f097a55f95f Mon Sep 17 00:00:00 2001 From: aavetis Date: Sat, 31 Jan 2026 17:17:02 -0500 Subject: [PATCH 1/4] Add update command to refresh sources --- README.md | 11 ++++ src/commands/update.test.ts | 94 +++++++++++++++++++++++++++ src/commands/update.ts | 123 ++++++++++++++++++++++++++++++++++++ src/index.ts | 54 ++++++++++++++-- 4 files changed, 277 insertions(+), 5 deletions(-) create mode 100644 src/commands/update.test.ts create mode 100644 src/commands/update.ts diff --git a/README.md b/README.md index b0faec8..54de775 100644 --- a/README.md +++ b/README.md @@ -67,11 +67,22 @@ GitHub repos are stored as `opensrc/owner--repo/`. # List fetched sources opensrc list +# Update all fetched sources +opensrc update + +# Update only repos or a specific registry +opensrc update --repos +opensrc update --pypi + # Remove a source (package or repo) opensrc remove zod opensrc remove owner--repo ``` +`opensrc update` follows the same behavior as re-running `opensrc ` or +`opensrc /`: packages resolve to installed/latest versions, and +repos reuse the stored ref. + ### File Modifications On first run, opensrc will ask for permission to modify these files: diff --git a/src/commands/update.test.ts b/src/commands/update.test.ts new file mode 100644 index 0000000..845c481 --- /dev/null +++ b/src/commands/update.test.ts @@ -0,0 +1,94 @@ +import { describe, it, expect } from "vitest"; +import { buildUpdateSpecs } from "./update.js"; +import type { PackageEntry, RepoEntry } from "../lib/agents.js"; + +const sources = { + packages: [ + { + name: "zod", + version: "3.22.0", + registry: "npm", + path: "zod", + fetchedAt: "2024-01-01T00:00:00.000Z", + }, + { + name: "requests", + version: "2.31.0", + registry: "pypi", + path: "requests", + fetchedAt: "2024-01-01T00:00:00.000Z", + }, + { + name: "serde", + version: "1.0.0", + registry: "crates", + path: "serde", + fetchedAt: "2024-01-01T00:00:00.000Z", + }, + ], + repos: [ + { + name: "github.com/vercel/ai", + version: "main", + path: "repos/github.com/vercel/ai", + fetchedAt: "2024-01-01T00:00:00.000Z", + }, + { + name: "example.com/foo/bar", + version: "HEAD", + path: "repos/example.com/foo/bar", + fetchedAt: "2024-01-01T00:00:00.000Z", + }, + ], +} satisfies { packages: PackageEntry[]; repos: RepoEntry[] }; + +describe("buildUpdateSpecs", () => { + it("builds specs for all sources by default", () => { + const { specs, packageCount, repoCount } = buildUpdateSpecs(sources); + + expect(packageCount).toBe(3); + expect(repoCount).toBe(2); + expect(specs).toEqual([ + "npm:zod", + "pypi:requests", + "crates:serde", + "https://github.com/vercel/ai#main", + "https://example.com/foo/bar", + ]); + }); + + it("filters by registry", () => { + const { specs, packageCount, repoCount } = buildUpdateSpecs(sources, { + registry: "pypi", + }); + + expect(packageCount).toBe(1); + expect(repoCount).toBe(0); + expect(specs).toEqual(["pypi:requests"]); + }); + + it("updates only packages when requested", () => { + const { specs, packageCount, repoCount } = buildUpdateSpecs(sources, { + packages: true, + repos: false, + }); + + expect(packageCount).toBe(3); + expect(repoCount).toBe(0); + expect(specs).toEqual(["npm:zod", "pypi:requests", "crates:serde"]); + }); + + it("updates only repos when requested", () => { + const { specs, packageCount, repoCount } = buildUpdateSpecs(sources, { + packages: false, + repos: true, + }); + + expect(packageCount).toBe(0); + expect(repoCount).toBe(2); + expect(specs).toEqual([ + "https://github.com/vercel/ai#main", + "https://example.com/foo/bar", + ]); + }); +}); diff --git a/src/commands/update.ts b/src/commands/update.ts new file mode 100644 index 0000000..ab12d37 --- /dev/null +++ b/src/commands/update.ts @@ -0,0 +1,123 @@ +import { listSources } from "../lib/git.js"; +import type { Registry } from "../types.js"; +import { fetchCommand } from "./fetch.js"; + +export interface UpdateOptions { + cwd?: string; + /** Only update packages (all registries) */ + packages?: boolean; + /** Only update repos */ + repos?: boolean; + /** Only update specific registry */ + registry?: Registry; + /** Override file modification permission */ + allowModifications?: boolean; +} + +type Sources = Awaited>; + +function shouldUpdatePackages(options: UpdateOptions): boolean { + return options.packages || (!options.packages && !options.repos); +} + +function shouldUpdateRepos(options: UpdateOptions): boolean { + return ( + options.repos || (!options.packages && !options.repos && !options.registry) + ); +} + +function buildRepoSpec(name: string, version: string): string { + const base = `https://${name}`; + if (!version || version === "HEAD") { + return base; + } + return `${base}#${version}`; +} + +export function buildUpdateSpecs( + sources: Sources, + options: UpdateOptions = {}, +): { specs: string[]; packageCount: number; repoCount: number } { + const updatePackages = shouldUpdatePackages(options); + const updateRepos = shouldUpdateRepos(options); + + let packages = updatePackages ? sources.packages : []; + if (options.registry) { + packages = packages.filter((p) => p.registry === options.registry); + } + + const repos = updateRepos ? sources.repos : []; + + const specs: string[] = []; + const seen = new Set(); + + for (const pkg of packages) { + const spec = `${pkg.registry}:${pkg.name}`; + if (!seen.has(spec)) { + specs.push(spec); + seen.add(spec); + } + } + + for (const repo of repos) { + const spec = buildRepoSpec(repo.name, repo.version); + if (!seen.has(spec)) { + specs.push(spec); + seen.add(spec); + } + } + + return { specs, packageCount: packages.length, repoCount: repos.length }; +} + +/** + * Update all fetched packages and/or repositories + */ +export async function updateCommand( + options: UpdateOptions = {}, +): Promise { + const cwd = options.cwd || process.cwd(); + const sources = await listSources(cwd); + const totalCount = sources.packages.length + sources.repos.length; + + if (totalCount === 0) { + console.log("No sources fetched yet."); + console.log( + "\nUse `opensrc ` to fetch source code for a package.", + ); + console.log("Use `opensrc /` to fetch a GitHub repository."); + console.log("\nSupported registries:"); + console.log(" • npm: opensrc zod, opensrc npm:react"); + console.log(" • PyPI: opensrc pypi:requests"); + console.log(" • crates: opensrc crates:serde"); + return; + } + + const { specs, packageCount, repoCount } = buildUpdateSpecs(sources, options); + + if (specs.length === 0) { + const updatePackages = shouldUpdatePackages(options); + const updateRepos = shouldUpdateRepos(options); + + if (options.registry) { + console.log(`No ${options.registry} packages to update.`); + } else { + if (updatePackages && packageCount === 0) { + console.log("No packages to update"); + } + if (updateRepos && repoCount === 0) { + console.log("No repos to update"); + } + } + + console.log("\nNothing to update."); + return; + } + + console.log(`Updating ${specs.length} source(s)...`); + + await fetchCommand(specs, { + cwd, + allowModifications: options.allowModifications, + }); +} diff --git a/src/index.ts b/src/index.ts index 2562651..53a22b5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,12 +3,19 @@ import { Command } from "commander"; import { fetchCommand } from "./commands/fetch.js"; import { listCommand } from "./commands/list.js"; +import { updateCommand } from "./commands/update.js"; import { removeCommand } from "./commands/remove.js"; import { cleanCommand } from "./commands/clean.js"; import type { Registry } from "./types.js"; const program = new Command(); +function parseModifyOption(val?: string): boolean { + if (val === undefined || val === "" || val === "true") return true; + if (val === "false") return false; + return true; +} + program .name("opensrc") .description( @@ -26,11 +33,7 @@ program .option( "--modify [value]", "allow/deny modifying .gitignore, tsconfig.json, AGENTS.md", - (val) => { - if (val === undefined || val === "" || val === "true") return true; - if (val === "false") return false; - return true; - }, + parseModifyOption, ) .action( async (packages: string[], options: { cwd?: string; modify?: boolean }) => { @@ -59,6 +62,47 @@ program }); }); +// Update command +program + .command("update") + .description("Update all fetched package sources") + .option("--packages", "only update packages (all registries)") + .option("--repos", "only update repos") + .option("--npm", "only update npm packages") + .option("--pypi", "only update PyPI packages") + .option("--crates", "only update crates.io packages") + .option("--cwd ", "working directory (default: current directory)") + .option( + "--modify [value]", + "allow/deny modifying .gitignore, tsconfig.json, AGENTS.md", + parseModifyOption, + ) + .action( + async (options: { + packages?: boolean; + repos?: boolean; + npm?: boolean; + pypi?: boolean; + crates?: boolean; + cwd?: string; + modify?: boolean; + }) => { + // Determine registry from flags + let registry: Registry | undefined; + if (options.npm) registry = "npm"; + else if (options.pypi) registry = "pypi"; + else if (options.crates) registry = "crates"; + + await updateCommand({ + packages: options.packages || !!registry, + repos: options.repos, + registry, + cwd: options.cwd, + allowModifications: options.modify, + }); + }, + ); + // Remove command program .command("remove ") From 53613ba9a5f2d24d7a50e226a4e51a36105cfed5 Mon Sep 17 00:00:00 2001 From: aavetis Date: Sat, 31 Jan 2026 17:23:17 -0500 Subject: [PATCH 2/4] Parse URL hash refs in repo specs --- src/lib/repo.test.ts | 20 ++++++++++++++++++++ src/lib/repo.ts | 8 ++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/lib/repo.test.ts b/src/lib/repo.test.ts index 41582f2..29a08d0 100644 --- a/src/lib/repo.test.ts +++ b/src/lib/repo.test.ts @@ -84,6 +84,26 @@ describe("parseRepoSpec", () => { }); }); + it("parses https://github.com/owner/repo#ref", () => { + const result = parseRepoSpec("https://github.com/vercel/ai#canary"); + expect(result).toEqual({ + host: "github.com", + owner: "vercel", + repo: "ai", + ref: "canary", + }); + }); + + it("parses https://github.com/owner/repo#ref/with/slash", () => { + const result = parseRepoSpec("https://github.com/vercel/ai#feature/foo"); + expect(result).toEqual({ + host: "github.com", + owner: "vercel", + repo: "ai", + ref: "feature/foo", + }); + }); + it("parses https://github.com/owner/repo.git", () => { const result = parseRepoSpec("https://github.com/vercel/ai.git"); expect(result).toEqual({ diff --git a/src/lib/repo.ts b/src/lib/repo.ts index b4e6f81..1a57edf 100644 --- a/src/lib/repo.ts +++ b/src/lib/repo.ts @@ -14,6 +14,7 @@ const DEFAULT_HOST = "github.com"; * - owner/repo@ref * - owner/repo#ref * - https://github.com/owner/repo + * - https://github.com/owner/repo#ref * - https://gitlab.com/owner/repo * - https://github.com/owner/repo/tree/branch * - github.com/owner/repo @@ -53,11 +54,14 @@ export function parseRepoSpec(spec: string): RepoSpec | null { repo = repo.slice(0, -4); } - // Handle /tree/branch or /blob/branch URLs - if ( + const hashRef = url.hash ? decodeURIComponent(url.hash.slice(1)) : ""; + if (hashRef) { + ref = hashRef; + } else if ( pathParts.length >= 4 && (pathParts[2] === "tree" || pathParts[2] === "blob") ) { + // Handle /tree/branch or /blob/branch URLs ref = pathParts[3]; } From cf891f9beef599b122bebc322fed754a8839cb96 Mon Sep 17 00:00:00 2001 From: aavetis Date: Sat, 31 Jan 2026 20:24:30 -0500 Subject: [PATCH 3/4] Recognize repo URLs on any host --- src/lib/repo.test.ts | 4 ++++ src/lib/repo.ts | 14 +++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/lib/repo.test.ts b/src/lib/repo.test.ts index 29a08d0..33cb9f5 100644 --- a/src/lib/repo.test.ts +++ b/src/lib/repo.test.ts @@ -276,6 +276,10 @@ describe("isRepoSpec", () => { expect(isRepoSpec("https://bitbucket.org/owner/repo")).toBe(true); }); + it("non-standard host URLs", () => { + expect(isRepoSpec("https://example.com/owner/repo")).toBe(true); + }); + it("host/owner/repo format", () => { expect(isRepoSpec("github.com/vercel/ai")).toBe(true); }); diff --git a/src/lib/repo.ts b/src/lib/repo.ts index 1a57edf..d55756b 100644 --- a/src/lib/repo.ts +++ b/src/lib/repo.ts @@ -127,9 +127,17 @@ export function isRepoSpec(spec: string): boolean { return true; } - // Git host URL - if (trimmed.match(/^https?:\/\/(github\.com|gitlab\.com|bitbucket\.org)\//)) { - return true; + // Git host URL (any host) + if (trimmed.match(/^https?:\/\//)) { + try { + const url = new URL(trimmed); + const parts = url.pathname.split("/").filter(Boolean); + if (parts.length >= 2) { + return true; + } + } catch { + // Fall through + } } // host/owner/repo format From d8bd6cf98b9106d1ac4479fe3811d35bd3188a9b Mon Sep 17 00:00:00 2001 From: aavetis Date: Mon, 2 Feb 2026 16:04:51 -0500 Subject: [PATCH 4/4] Tighten repo URL detection heuristics --- src/lib/repo.test.ts | 10 ++++++++-- src/lib/repo.ts | 23 ++++++++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/lib/repo.test.ts b/src/lib/repo.test.ts index 33cb9f5..fc0885c 100644 --- a/src/lib/repo.test.ts +++ b/src/lib/repo.test.ts @@ -276,8 +276,10 @@ describe("isRepoSpec", () => { expect(isRepoSpec("https://bitbucket.org/owner/repo")).toBe(true); }); - it("non-standard host URLs", () => { - expect(isRepoSpec("https://example.com/owner/repo")).toBe(true); + it("non-standard host URLs with git signals", () => { + expect(isRepoSpec("https://git.example.com/owner/repo")).toBe(true); + expect(isRepoSpec("https://example.com/owner/repo.git")).toBe(true); + expect(isRepoSpec("https://example.com/owner/repo/tree/main")).toBe(true); }); it("host/owner/repo format", () => { @@ -312,6 +314,10 @@ describe("isRepoSpec", () => { expect(isRepoSpec("lodash@4.17.0")).toBe(false); expect(isRepoSpec("react@18.2.0")).toBe(false); }); + + it("non-standard host URLs without git signals", () => { + expect(isRepoSpec("https://example.com/owner/repo")).toBe(false); + }); }); }); diff --git a/src/lib/repo.ts b/src/lib/repo.ts index d55756b..8e570bd 100644 --- a/src/lib/repo.ts +++ b/src/lib/repo.ts @@ -127,14 +127,31 @@ export function isRepoSpec(spec: string): boolean { return true; } - // Git host URL (any host) + // Git host URL (known hosts or repo-looking URLs) if (trimmed.match(/^https?:\/\//)) { try { const url = new URL(trimmed); - const parts = url.pathname.split("/").filter(Boolean); - if (parts.length >= 2) { + const host = url.hostname.toLowerCase(); + const path = url.pathname; + const parts = path.split("/").filter(Boolean); + if (parts.length < 2) { + return false; + } + + if (SUPPORTED_HOSTS.includes(host)) { return true; } + + const lowerPath = path.toLowerCase(); + const hasGitSuffix = lowerPath.endsWith(".git"); + const hasTreeOrBlob = + lowerPath.includes("/tree/") || lowerPath.includes("/blob/"); + const hostSignals = ["gitlab", "gitea", "gogs", "github", "bitbucket"]; + const hasGitHostSignal = + host.startsWith("git.") || + hostSignals.some((signal) => host.includes(signal)); + + return hasGitSuffix || hasTreeOrBlob || hasGitHostSignal; } catch { // Fall through }