From 2bf6147a8d6bbe209170e77b062a2e27cf19747c Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 3 Dec 2024 11:25:10 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A5=20Remove=20lots=20of=20stuff?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/blob/src/consts.ts | 1 - packages/blob/src/error.ts | 46 -- packages/blob/src/index.ts | 26 +- .../blob/src/lib/cache-management.spec.ts | 137 ---- packages/blob/src/lib/cache-management.ts | 265 ------- .../blob/src/lib/check-repo-access.spec.ts | 34 - packages/blob/src/lib/check-repo-access.ts | 32 - packages/blob/src/lib/commit.spec.ts | 275 ------- packages/blob/src/lib/commit.ts | 597 --------------- packages/blob/src/lib/count-commits.spec.ts | 16 - packages/blob/src/lib/count-commits.ts | 35 - packages/blob/src/lib/create-repo.spec.ts | 103 --- packages/blob/src/lib/create-repo.ts | 75 -- packages/blob/src/lib/dataset-info.spec.ts | 56 -- packages/blob/src/lib/dataset-info.ts | 59 -- packages/blob/src/lib/delete-file.spec.ts | 64 -- packages/blob/src/lib/delete-file.ts | 35 - packages/blob/src/lib/delete-files.spec.ts | 81 --- packages/blob/src/lib/delete-files.ts | 33 - packages/blob/src/lib/delete-repo.ts | 37 - .../lib/download-file-to-cache-dir.spec.ts | 234 ------ .../src/lib/download-file-to-cache-dir.ts | 129 ---- packages/blob/src/lib/download-file.spec.ts | 65 -- packages/blob/src/lib/download-file.ts | 65 -- .../blob/src/lib/file-download-info.spec.ts | 49 -- packages/blob/src/lib/file-download-info.ts | 92 --- packages/blob/src/lib/file-exists.spec.ts | 30 - packages/blob/src/lib/file-exists.ts | 41 -- packages/blob/src/lib/index.ts | 29 - packages/blob/src/lib/list-commits.spec.ts | 117 --- packages/blob/src/lib/list-commits.ts | 70 -- packages/blob/src/lib/list-datasets.spec.ts | 47 -- packages/blob/src/lib/list-datasets.ts | 121 ---- packages/blob/src/lib/list-files.spec.ts | 169 ----- packages/blob/src/lib/list-files.ts | 90 --- packages/blob/src/lib/list-models.spec.ts | 83 --- packages/blob/src/lib/list-models.ts | 131 ---- packages/blob/src/lib/list-spaces.spec.ts | 40 - packages/blob/src/lib/list-spaces.ts | 111 --- packages/blob/src/lib/model-info.spec.ts | 59 -- packages/blob/src/lib/model-info.ts | 60 -- .../blob/src/lib/oauth-handle-redirect.ts | 233 ------ packages/blob/src/lib/oauth-login-url.ts | 132 ---- .../lib/parse-safetensors-metadata.spec.ts | 122 ---- .../src/lib/parse-safetensors-metadata.ts | 285 -------- packages/blob/src/lib/paths-info.spec.ts | 75 -- packages/blob/src/lib/paths-info.ts | 120 --- .../blob/src/lib/snapshot-download.spec.ts | 275 ------- packages/blob/src/lib/snapshot-download.ts | 124 ---- packages/blob/src/lib/space-info.spec.ts | 53 -- packages/blob/src/lib/space-info.ts | 59 -- packages/blob/src/lib/upload-file.spec.ts | 98 --- packages/blob/src/lib/upload-file.ts | 47 -- .../lib/upload-files-with-progress.spec.ts | 168 ----- .../src/lib/upload-files-with-progress.ts | 153 ---- packages/blob/src/lib/upload-files.spec.ts | 95 --- packages/blob/src/lib/upload-files.ts | 38 - packages/blob/src/lib/who-am-i.spec.ts | 35 - packages/blob/src/lib/who-am-i.ts | 91 --- packages/blob/src/test/consts.ts | 3 - packages/blob/src/types/api/api-commit.ts | 191 ----- .../blob/src/types/api/api-create-repo.ts | 25 - packages/blob/src/types/api/api-dataset.ts | 89 --- packages/blob/src/types/api/api-index-tree.ts | 46 -- packages/blob/src/types/api/api-model.ts | 270 ------- packages/blob/src/types/api/api-space.ts | 93 --- packages/blob/src/types/api/api-who-am-i.ts | 51 -- packages/blob/src/types/public.ts | 181 ----- packages/blob/src/utils/checkCredentials.ts | 18 - packages/blob/src/utils/omit.ts | 14 - packages/blob/src/utils/parseLinkHeader.ts | 8 - packages/blob/src/utils/pick.ts | 13 - packages/blob/src/utils/sha256-node.ts | 26 - packages/blob/src/utils/sha256.spec.ts | 50 -- packages/blob/src/utils/sha256.ts | 166 ----- packages/blob/src/utils/sum.ts | 6 - packages/blob/src/utils/toRepoId.ts | 54 -- packages/blob/src/utils/typedEntries.ts | 5 - packages/blob/src/utils/typedInclude.ts | 3 - packages/blob/src/vendor/hash-wasm/build.sh | 33 - .../src/vendor/hash-wasm/sha256-wrapper.ts | 62 -- packages/blob/src/vendor/hash-wasm/sha256.c | 432 ----------- .../blob/src/vendor/hash-wasm/sha256.d.ts | 8 - packages/blob/src/vendor/hash-wasm/sha256.js | 685 ------------------ packages/blob/src/vendor/type-fest/basic.ts | 31 - packages/blob/src/vendor/type-fest/entries.ts | 65 -- packages/blob/src/vendor/type-fest/entry.ts | 68 -- packages/blob/src/vendor/type-fest/except.ts | 71 -- .../blob/src/vendor/type-fest/is-equal.ts | 27 - .../blob/src/vendor/type-fest/set-required.ts | 34 - .../blob/src/vendor/type-fest/simplify.ts | 59 -- 91 files changed, 2 insertions(+), 8927 deletions(-) delete mode 100644 packages/blob/src/consts.ts delete mode 100644 packages/blob/src/error.ts delete mode 100644 packages/blob/src/lib/cache-management.spec.ts delete mode 100644 packages/blob/src/lib/cache-management.ts delete mode 100644 packages/blob/src/lib/check-repo-access.spec.ts delete mode 100644 packages/blob/src/lib/check-repo-access.ts delete mode 100644 packages/blob/src/lib/commit.spec.ts delete mode 100644 packages/blob/src/lib/commit.ts delete mode 100644 packages/blob/src/lib/count-commits.spec.ts delete mode 100644 packages/blob/src/lib/count-commits.ts delete mode 100644 packages/blob/src/lib/create-repo.spec.ts delete mode 100644 packages/blob/src/lib/create-repo.ts delete mode 100644 packages/blob/src/lib/dataset-info.spec.ts delete mode 100644 packages/blob/src/lib/dataset-info.ts delete mode 100644 packages/blob/src/lib/delete-file.spec.ts delete mode 100644 packages/blob/src/lib/delete-file.ts delete mode 100644 packages/blob/src/lib/delete-files.spec.ts delete mode 100644 packages/blob/src/lib/delete-files.ts delete mode 100644 packages/blob/src/lib/delete-repo.ts delete mode 100644 packages/blob/src/lib/download-file-to-cache-dir.spec.ts delete mode 100644 packages/blob/src/lib/download-file-to-cache-dir.ts delete mode 100644 packages/blob/src/lib/download-file.spec.ts delete mode 100644 packages/blob/src/lib/download-file.ts delete mode 100644 packages/blob/src/lib/file-download-info.spec.ts delete mode 100644 packages/blob/src/lib/file-download-info.ts delete mode 100644 packages/blob/src/lib/file-exists.spec.ts delete mode 100644 packages/blob/src/lib/file-exists.ts delete mode 100644 packages/blob/src/lib/index.ts delete mode 100644 packages/blob/src/lib/list-commits.spec.ts delete mode 100644 packages/blob/src/lib/list-commits.ts delete mode 100644 packages/blob/src/lib/list-datasets.spec.ts delete mode 100644 packages/blob/src/lib/list-datasets.ts delete mode 100644 packages/blob/src/lib/list-files.spec.ts delete mode 100644 packages/blob/src/lib/list-files.ts delete mode 100644 packages/blob/src/lib/list-models.spec.ts delete mode 100644 packages/blob/src/lib/list-models.ts delete mode 100644 packages/blob/src/lib/list-spaces.spec.ts delete mode 100644 packages/blob/src/lib/list-spaces.ts delete mode 100644 packages/blob/src/lib/model-info.spec.ts delete mode 100644 packages/blob/src/lib/model-info.ts delete mode 100644 packages/blob/src/lib/oauth-handle-redirect.ts delete mode 100644 packages/blob/src/lib/oauth-login-url.ts delete mode 100644 packages/blob/src/lib/parse-safetensors-metadata.spec.ts delete mode 100644 packages/blob/src/lib/parse-safetensors-metadata.ts delete mode 100644 packages/blob/src/lib/paths-info.spec.ts delete mode 100644 packages/blob/src/lib/paths-info.ts delete mode 100644 packages/blob/src/lib/snapshot-download.spec.ts delete mode 100644 packages/blob/src/lib/snapshot-download.ts delete mode 100644 packages/blob/src/lib/space-info.spec.ts delete mode 100644 packages/blob/src/lib/space-info.ts delete mode 100644 packages/blob/src/lib/upload-file.spec.ts delete mode 100644 packages/blob/src/lib/upload-file.ts delete mode 100644 packages/blob/src/lib/upload-files-with-progress.spec.ts delete mode 100644 packages/blob/src/lib/upload-files-with-progress.ts delete mode 100644 packages/blob/src/lib/upload-files.spec.ts delete mode 100644 packages/blob/src/lib/upload-files.ts delete mode 100644 packages/blob/src/lib/who-am-i.spec.ts delete mode 100644 packages/blob/src/lib/who-am-i.ts delete mode 100644 packages/blob/src/test/consts.ts delete mode 100644 packages/blob/src/types/api/api-commit.ts delete mode 100644 packages/blob/src/types/api/api-create-repo.ts delete mode 100644 packages/blob/src/types/api/api-dataset.ts delete mode 100644 packages/blob/src/types/api/api-index-tree.ts delete mode 100644 packages/blob/src/types/api/api-model.ts delete mode 100644 packages/blob/src/types/api/api-space.ts delete mode 100644 packages/blob/src/types/api/api-who-am-i.ts delete mode 100644 packages/blob/src/types/public.ts delete mode 100644 packages/blob/src/utils/checkCredentials.ts delete mode 100644 packages/blob/src/utils/omit.ts delete mode 100644 packages/blob/src/utils/parseLinkHeader.ts delete mode 100644 packages/blob/src/utils/pick.ts delete mode 100644 packages/blob/src/utils/sha256-node.ts delete mode 100644 packages/blob/src/utils/sha256.spec.ts delete mode 100644 packages/blob/src/utils/sha256.ts delete mode 100644 packages/blob/src/utils/sum.ts delete mode 100644 packages/blob/src/utils/toRepoId.ts delete mode 100644 packages/blob/src/utils/typedEntries.ts delete mode 100644 packages/blob/src/utils/typedInclude.ts delete mode 100755 packages/blob/src/vendor/hash-wasm/build.sh delete mode 100644 packages/blob/src/vendor/hash-wasm/sha256-wrapper.ts delete mode 100644 packages/blob/src/vendor/hash-wasm/sha256.c delete mode 100644 packages/blob/src/vendor/hash-wasm/sha256.d.ts delete mode 100644 packages/blob/src/vendor/hash-wasm/sha256.js delete mode 100644 packages/blob/src/vendor/type-fest/basic.ts delete mode 100644 packages/blob/src/vendor/type-fest/entries.ts delete mode 100644 packages/blob/src/vendor/type-fest/entry.ts delete mode 100644 packages/blob/src/vendor/type-fest/except.ts delete mode 100644 packages/blob/src/vendor/type-fest/is-equal.ts delete mode 100644 packages/blob/src/vendor/type-fest/set-required.ts delete mode 100644 packages/blob/src/vendor/type-fest/simplify.ts diff --git a/packages/blob/src/consts.ts b/packages/blob/src/consts.ts deleted file mode 100644 index 5d34e9cae..000000000 --- a/packages/blob/src/consts.ts +++ /dev/null @@ -1 +0,0 @@ -export const HUB_URL = "https://huggingface.co"; diff --git a/packages/blob/src/error.ts b/packages/blob/src/error.ts deleted file mode 100644 index f0a3e33c4..000000000 --- a/packages/blob/src/error.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { JsonObject } from "./vendor/type-fest/basic"; - -export async function createApiError( - response: Response, - opts?: { requestId?: string; message?: string } -): Promise { - const error = new HubApiError(response.url, response.status, response.headers.get("X-Request-Id") ?? opts?.requestId); - - error.message = `Api error with status ${error.statusCode}${opts?.message ? `. ${opts.message}` : ""}`; - - const trailer = [`URL: ${error.url}`, error.requestId ? `Request ID: ${error.requestId}` : undefined] - .filter(Boolean) - .join(". "); - - if (response.headers.get("Content-Type")?.startsWith("application/json")) { - const json = await response.json(); - error.message = json.error || json.message || error.message; - error.data = json; - } else { - error.data = { message: await response.text() }; - } - - error.message += `. ${trailer}`; - - throw error; -} - -/** - * Error thrown when an API call to the Hugging Face Hub fails. - */ -export class HubApiError extends Error { - statusCode: number; - url: string; - requestId?: string; - data?: JsonObject; - - constructor(url: string, statusCode: number, requestId?: string, message?: string) { - super(message); - - this.statusCode = statusCode; - this.requestId = requestId; - this.url = url; - } -} - -export class InvalidApiResponseFormatError extends Error {} diff --git a/packages/blob/src/index.ts b/packages/blob/src/index.ts index 98a88e185..983542526 100644 --- a/packages/blob/src/index.ts +++ b/packages/blob/src/index.ts @@ -1,24 +1,2 @@ -export * from "./lib"; -// Typescript 5 will add 'export type *' -export type { - AccessToken, - AccessTokenRole, - AuthType, - Credentials, - PipelineType, - RepoDesignation, - RepoFullName, - RepoId, - RepoType, - SpaceHardwareFlavor, - SpaceResourceConfig, - SpaceResourceRequirement, - SpaceRuntime, - SpaceSdk, - SpaceStage, -} from "./types/public"; -export { HubApiError, InvalidApiResponseFormatError } from "./error"; -/** - * Only exported for E2Es convenience - */ -export { sha256 as __internal_sha256 } from "./utils/sha256"; +export * from "./utils/createBlob"; +export * from "./utils/WebBlob"; diff --git a/packages/blob/src/lib/cache-management.spec.ts b/packages/blob/src/lib/cache-management.spec.ts deleted file mode 100644 index 3bfed63d9..000000000 --- a/packages/blob/src/lib/cache-management.spec.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { describe, test, expect, vi, beforeEach } from "vitest"; -import { - scanCacheDir, - scanCachedRepo, - scanSnapshotDir, - parseRepoType, - getBlobStat, - type CachedFileInfo, -} from "./cache-management"; -import { stat, readdir, realpath, lstat } from "node:fs/promises"; -import type { Dirent, Stats } from "node:fs"; -import { join } from "node:path"; - -// Mocks -vi.mock("node:fs/promises"); - -beforeEach(() => { - vi.resetAllMocks(); - vi.restoreAllMocks(); -}); - -describe("scanCacheDir", () => { - test("should throw an error if cacheDir is not a directory", async () => { - vi.mocked(stat).mockResolvedValueOnce({ - isDirectory: () => false, - } as Stats); - - await expect(scanCacheDir("/fake/dir")).rejects.toThrow("Scan cache expects a directory"); - }); - - test("empty directory should return an empty set of repository and no warnings", async () => { - vi.mocked(stat).mockResolvedValueOnce({ - isDirectory: () => true, - } as Stats); - - // mock empty cache folder - vi.mocked(readdir).mockResolvedValue([]); - - const result = await scanCacheDir("/fake/dir"); - - // cacheDir must have been read - expect(readdir).toHaveBeenCalledWith("/fake/dir"); - - expect(result.warnings.length).toBe(0); - expect(result.repos).toHaveLength(0); - expect(result.size).toBe(0); - }); -}); - -describe("scanCachedRepo", () => { - test("should throw an error for invalid repo path", async () => { - await expect(() => { - return scanCachedRepo("/fake/repo_path"); - }).rejects.toThrow("Repo path is not a valid HuggingFace cache directory"); - }); - - test("should throw an error if the snapshot folder does not exist", async () => { - vi.mocked(readdir).mockResolvedValue([]); - vi.mocked(stat).mockResolvedValue({ - isDirectory: () => false, - } as Stats); - - await expect(() => { - return scanCachedRepo("/fake/cacheDir/models--hello-world--name"); - }).rejects.toThrow("Snapshots dir doesn't exist in cached repo"); - }); - - test("should properly parse the repository name", async () => { - const repoPath = "/fake/cacheDir/models--hello-world--name"; - vi.mocked(readdir).mockResolvedValue([]); - vi.mocked(stat).mockResolvedValue({ - isDirectory: () => true, - } as Stats); - - const result = await scanCachedRepo(repoPath); - expect(readdir).toHaveBeenCalledWith(join(repoPath, "refs"), { - withFileTypes: true, - }); - - expect(result.id.name).toBe("hello-world/name"); - expect(result.id.type).toBe("model"); - }); -}); - -describe("scanSnapshotDir", () => { - test("should scan a valid snapshot directory", async () => { - const cachedFiles: CachedFileInfo[] = []; - const blobStats = new Map(); - vi.mocked(readdir).mockResolvedValueOnce([{ name: "file1", isDirectory: () => false } as Dirent]); - - vi.mocked(realpath).mockResolvedValueOnce("/fake/realpath"); - vi.mocked(lstat).mockResolvedValueOnce({ size: 1024, atimeMs: Date.now(), mtimeMs: Date.now() } as Stats); - - await scanSnapshotDir("/fake/revision", cachedFiles, blobStats); - - expect(cachedFiles).toHaveLength(1); - expect(blobStats.size).toBe(1); - }); -}); - -describe("getBlobStat", () => { - test("should retrieve blob stat if already cached", async () => { - const blobStats = new Map([["/fake/blob", { size: 1024 } as Stats]]); - const result = await getBlobStat("/fake/blob", blobStats); - - expect(lstat).not.toHaveBeenCalled(); - expect(result.size).toBe(1024); - }); - - test("should fetch and cache blob stat if not cached", async () => { - const blobStats = new Map(); - vi.mocked(lstat).mockResolvedValueOnce({ size: 2048 } as Stats); - - const result = await getBlobStat("/fake/blob", blobStats); - - expect(result.size).toBe(2048); - expect(blobStats.size).toBe(1); - }); -}); - -describe("parseRepoType", () => { - test("should parse models repo type", () => { - expect(parseRepoType("models")).toBe("model"); - }); - - test("should parse dataset repo type", () => { - expect(parseRepoType("datasets")).toBe("dataset"); - }); - - test("should parse space repo type", () => { - expect(parseRepoType("spaces")).toBe("space"); - }); - - test("should throw an error for invalid repo type", () => { - expect(() => parseRepoType("invalid")).toThrowError("Invalid repo type: invalid"); - }); -}); diff --git a/packages/blob/src/lib/cache-management.ts b/packages/blob/src/lib/cache-management.ts deleted file mode 100644 index 98c3be5b4..000000000 --- a/packages/blob/src/lib/cache-management.ts +++ /dev/null @@ -1,265 +0,0 @@ -import { homedir } from "node:os"; -import { join, basename } from "node:path"; -import { stat, readdir, readFile, realpath, lstat } from "node:fs/promises"; -import type { Stats } from "node:fs"; -import type { RepoType, RepoId } from "../types/public"; - -function getDefaultHome(): string { - return join(homedir(), ".cache"); -} - -function getDefaultCachePath(): string { - return join(process.env["HF_HOME"] ?? join(process.env["XDG_CACHE_HOME"] ?? getDefaultHome(), "huggingface"), "hub"); -} - -function getHuggingFaceHubCache(): string { - return process.env["HUGGINGFACE_HUB_CACHE"] ?? getDefaultCachePath(); -} - -export function getHFHubCachePath(): string { - return process.env["HF_HUB_CACHE"] ?? getHuggingFaceHubCache(); -} - -const FILES_TO_IGNORE: string[] = [".DS_Store"]; - -export const REPO_ID_SEPARATOR: string = "--"; - -export function getRepoFolderName({ name, type }: RepoId): string { - const parts = [`${type}s`, ...name.split("/")] - return parts.join(REPO_ID_SEPARATOR); -} - -export interface CachedFileInfo { - path: string; - /** - * Underlying file - which `path` is symlinked to - */ - blob: { - size: number; - path: string; - lastModifiedAt: Date; - lastAccessedAt: Date; - }; -} - -export interface CachedRevisionInfo { - commitOid: string; - path: string; - size: number; - files: CachedFileInfo[]; - refs: string[]; - - lastModifiedAt: Date; -} - -export interface CachedRepoInfo { - id: RepoId; - path: string; - size: number; - filesCount: number; - revisions: CachedRevisionInfo[]; - - lastAccessedAt: Date; - lastModifiedAt: Date; -} - -export interface HFCacheInfo { - size: number; - repos: CachedRepoInfo[]; - warnings: Error[]; -} - -export async function scanCacheDir(cacheDir: string | undefined = undefined): Promise { - if (!cacheDir) cacheDir = getHFHubCachePath(); - - const s = await stat(cacheDir); - if (!s.isDirectory()) { - throw new Error( - `Scan cache expects a directory but found a file: ${cacheDir}. Please use \`cacheDir\` argument or set \`HF_HUB_CACHE\` environment variable.` - ); - } - - const repos: CachedRepoInfo[] = []; - const warnings: Error[] = []; - - const directories = await readdir(cacheDir); - for (const repo of directories) { - // skip .locks folder - if (repo === ".locks") continue; - - // get the absolute path of the repo - const absolute = join(cacheDir, repo); - - // ignore non-directory element - const s = await stat(absolute); - if (!s.isDirectory()) { - continue; - } - - try { - const cached = await scanCachedRepo(absolute); - repos.push(cached); - } catch (err: unknown) { - warnings.push(err as Error); - } - } - - return { - repos: repos, - size: [...repos.values()].reduce((sum, repo) => sum + repo.size, 0), - warnings: warnings, - }; -} - -export async function scanCachedRepo(repoPath: string): Promise { - // get the directory name - const name = basename(repoPath); - if (!name.includes(REPO_ID_SEPARATOR)) { - throw new Error(`Repo path is not a valid HuggingFace cache directory: ${name}`); - } - - // parse the repoId from directory name - const [type, ...remaining] = name.split(REPO_ID_SEPARATOR); - const repoType = parseRepoType(type); - const repoId = remaining.join("/"); - - const snapshotsPath = join(repoPath, "snapshots"); - const refsPath = join(repoPath, "refs"); - - const snapshotStat = await stat(snapshotsPath); - if (!snapshotStat.isDirectory()) { - throw new Error(`Snapshots dir doesn't exist in cached repo ${snapshotsPath}`); - } - - // Check if the refs directory exists and scan it - const refsByHash: Map = new Map(); - const refsStat = await stat(refsPath); - if (refsStat.isDirectory()) { - await scanRefsDir(refsPath, refsByHash); - } - - // Scan snapshots directory and collect cached revision information - const cachedRevisions: CachedRevisionInfo[] = []; - const blobStats: Map = new Map(); // Store blob stats - - const snapshotDirs = await readdir(snapshotsPath); - for (const dir of snapshotDirs) { - if (FILES_TO_IGNORE.includes(dir)) continue; // Ignore unwanted files - - const revisionPath = join(snapshotsPath, dir); - const revisionStat = await stat(revisionPath); - if (!revisionStat.isDirectory()) { - throw new Error(`Snapshots folder corrupted. Found a file: ${revisionPath}`); - } - - const cachedFiles: CachedFileInfo[] = []; - await scanSnapshotDir(revisionPath, cachedFiles, blobStats); - - const revisionLastModified = - cachedFiles.length > 0 - ? Math.max(...[...cachedFiles].map((file) => file.blob.lastModifiedAt.getTime())) - : revisionStat.mtimeMs; - - cachedRevisions.push({ - commitOid: dir, - files: cachedFiles, - refs: refsByHash.get(dir) || [], - size: [...cachedFiles].reduce((sum, file) => sum + file.blob.size, 0), - path: revisionPath, - lastModifiedAt: new Date(revisionLastModified), - }); - - refsByHash.delete(dir); - } - - // Verify that all refs refer to a valid revision - if (refsByHash.size > 0) { - throw new Error( - `Reference(s) refer to missing commit hashes: ${JSON.stringify(Object.fromEntries(refsByHash))} (${repoPath})` - ); - } - - const repoStats = await stat(repoPath); - const repoLastAccessed = - blobStats.size > 0 ? Math.max(...[...blobStats.values()].map((stat) => stat.atimeMs)) : repoStats.atimeMs; - - const repoLastModified = - blobStats.size > 0 ? Math.max(...[...blobStats.values()].map((stat) => stat.mtimeMs)) : repoStats.mtimeMs; - - // Return the constructed CachedRepoInfo object - return { - id: { - name: repoId, - type: repoType, - }, - path: repoPath, - filesCount: blobStats.size, - revisions: cachedRevisions, - size: [...blobStats.values()].reduce((sum, stat) => sum + stat.size, 0), - lastAccessedAt: new Date(repoLastAccessed), - lastModifiedAt: new Date(repoLastModified), - }; -} - -export async function scanRefsDir(refsPath: string, refsByHash: Map): Promise { - const refFiles = await readdir(refsPath, { withFileTypes: true }); - for (const refFile of refFiles) { - const refFilePath = join(refsPath, refFile.name); - if (refFile.isDirectory()) continue; // Skip directories - - const commitHash = await readFile(refFilePath, "utf-8"); - const refName = refFile.name; - if (!refsByHash.has(commitHash)) { - refsByHash.set(commitHash, []); - } - refsByHash.get(commitHash)?.push(refName); - } -} - -export async function scanSnapshotDir( - revisionPath: string, - cachedFiles: CachedFileInfo[], - blobStats: Map -): Promise { - const files = await readdir(revisionPath, { withFileTypes: true }); - for (const file of files) { - if (file.isDirectory()) continue; // Skip directories - - const filePath = join(revisionPath, file.name); - const blobPath = await realpath(filePath); - const blobStat = await getBlobStat(blobPath, blobStats); - - cachedFiles.push({ - path: filePath, - blob: { - path: blobPath, - size: blobStat.size, - lastAccessedAt: new Date(blobStat.atimeMs), - lastModifiedAt: new Date(blobStat.mtimeMs), - }, - }); - } -} - -export async function getBlobStat(blobPath: string, blobStats: Map): Promise { - const blob = blobStats.get(blobPath); - if (!blob) { - const statResult = await lstat(blobPath); - blobStats.set(blobPath, statResult); - return statResult; - } - return blob; -} - -export function parseRepoType(type: string): RepoType { - switch (type) { - case "models": - return "model"; - case "datasets": - return "dataset"; - case "spaces": - return "space"; - default: - throw new TypeError(`Invalid repo type: ${type}`); - } -} diff --git a/packages/blob/src/lib/check-repo-access.spec.ts b/packages/blob/src/lib/check-repo-access.spec.ts deleted file mode 100644 index 12ad5cd92..000000000 --- a/packages/blob/src/lib/check-repo-access.spec.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { assert, describe, expect, it } from "vitest"; -import { checkRepoAccess } from "./check-repo-access"; -import { HubApiError } from "../error"; -import { TEST_ACCESS_TOKEN, TEST_HUB_URL } from "../test/consts"; - -describe("checkRepoAccess", () => { - it("should throw 401 when accessing unexisting repo unauthenticated", async () => { - try { - await checkRepoAccess({ repo: { name: "i--d/dont", type: "model" } }); - assert(false, "should have thrown"); - } catch (err) { - expect(err).toBeInstanceOf(HubApiError); - expect((err as HubApiError).statusCode).toBe(401); - } - }); - - it("should throw 404 when accessing unexisting repo authenticated", async () => { - try { - await checkRepoAccess({ - repo: { name: "i--d/dont", type: "model" }, - hubUrl: TEST_HUB_URL, - accessToken: TEST_ACCESS_TOKEN, - }); - assert(false, "should have thrown"); - } catch (err) { - expect(err).toBeInstanceOf(HubApiError); - expect((err as HubApiError).statusCode).toBe(404); - } - }); - - it("should not throw when accessing public repo", async () => { - await checkRepoAccess({ repo: { name: "openai-community/gpt2", type: "model" } }); - }); -}); diff --git a/packages/blob/src/lib/check-repo-access.ts b/packages/blob/src/lib/check-repo-access.ts deleted file mode 100644 index 3107c9bd7..000000000 --- a/packages/blob/src/lib/check-repo-access.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { HUB_URL } from "../consts"; -// eslint-disable-next-line @typescript-eslint/no-unused-vars -import { createApiError, type HubApiError } from "../error"; -import type { CredentialsParams, RepoDesignation } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; -import { toRepoId } from "../utils/toRepoId"; - -/** - * Check if we have read access to a repository. - * - * Throw a {@link HubApiError} error if we don't have access. HubApiError.statusCode will be 401, 403 or 404. - */ -export async function checkRepoAccess( - params: { - repo: RepoDesignation; - hubUrl?: string; - fetch?: typeof fetch; - } & Partial -): Promise { - const accessToken = params && checkCredentials(params); - const repoId = toRepoId(params.repo); - - const response = await (params.fetch || fetch)(`${params?.hubUrl || HUB_URL}/api/${repoId.type}s/${repoId.name}`, { - headers: { - ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), - }, - }); - - if (!response.ok) { - throw await createApiError(response); - } -} diff --git a/packages/blob/src/lib/commit.spec.ts b/packages/blob/src/lib/commit.spec.ts deleted file mode 100644 index 617be8ee8..000000000 --- a/packages/blob/src/lib/commit.spec.ts +++ /dev/null @@ -1,275 +0,0 @@ -import { assert, it, describe } from "vitest"; - -import { TEST_HUB_URL, TEST_ACCESS_TOKEN, TEST_USER } from "../test/consts"; -import type { RepoId } from "../types/public"; -import type { CommitFile } from "./commit"; -import { commit } from "./commit"; -import { createRepo } from "./create-repo"; -import { deleteRepo } from "./delete-repo"; -import { downloadFile } from "./download-file"; -import { fileDownloadInfo } from "./file-download-info"; -import { insecureRandomString } from "../utils/insecureRandomString"; -import { isFrontend } from "../utils/isFrontend"; - -const lfsContent = "O123456789".repeat(100_000); - -describe("commit", () => { - it("should commit to a repo with blobs", async function () { - const tokenizerJsonUrl = new URL( - "https://huggingface.co/spaces/aschen/push-model-from-web/raw/main/mobilenet/model.json" - ); - const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; - const repo: RepoId = { - name: repoName, - type: "model", - }; - - await createRepo({ - accessToken: TEST_ACCESS_TOKEN, - hubUrl: TEST_HUB_URL, - repo, - license: "mit", - }); - - try { - const readme1 = await downloadFile({ repo, path: "README.md", hubUrl: TEST_HUB_URL }); - assert.strictEqual(readme1?.status, 200); - - const nodeOperation: CommitFile[] = isFrontend - ? [] - : [ - { - operation: "addOrUpdate", - path: "tsconfig.json", - content: (await import("node:url")).pathToFileURL("./tsconfig.json") as URL, - }, - ]; - - await commit({ - repo, - title: "Some commit", - accessToken: TEST_ACCESS_TOKEN, - hubUrl: TEST_HUB_URL, - operations: [ - { - operation: "addOrUpdate", - content: new Blob(["This is me"]), - path: "test.txt", - }, - { - operation: "addOrUpdate", - content: new Blob([lfsContent]), - path: "test.lfs.txt", - }, - ...nodeOperation, - { - operation: "addOrUpdate", - content: tokenizerJsonUrl, - path: "lamaral.json", - }, - { - operation: "delete", - path: "README.md", - }, - ], - // To test web workers in the front-end - useWebWorkers: { minSize: 5_000 }, - }); - - const fileContent = await downloadFile({ repo, path: "test.txt", hubUrl: TEST_HUB_URL }); - assert.strictEqual(fileContent?.status, 200); - assert.strictEqual(await fileContent?.text(), "This is me"); - - const lfsFileContent = await downloadFile({ repo, path: "test.lfs.txt", hubUrl: TEST_HUB_URL }); - assert.strictEqual(lfsFileContent?.status, 200); - assert.strictEqual(await lfsFileContent?.text(), lfsContent); - - const lfsFileUrl = `${TEST_HUB_URL}/${repoName}/raw/main/test.lfs.txt`; - const lfsFilePointer = await fetch(lfsFileUrl); - assert.strictEqual(lfsFilePointer.status, 200); - assert.strictEqual( - (await lfsFilePointer.text()).trim(), - ` -version https://git-lfs.github.com/spec/v1 -oid sha256:a3bbce7ee1df7233d85b5f4d60faa3755f93f537804f8b540c72b0739239ddf8 -size ${lfsContent.length} - `.trim() - ); - - if (!isFrontend) { - const fileUrlContent = await downloadFile({ repo, path: "tsconfig.json", hubUrl: TEST_HUB_URL }); - assert.strictEqual(fileUrlContent?.status, 200); - assert.strictEqual( - await fileUrlContent?.text(), - (await import("node:fs")).readFileSync("./tsconfig.json", "utf-8") - ); - } - - const webResourceContent = await downloadFile({ repo, path: "lamaral.json", hubUrl: TEST_HUB_URL }); - assert.strictEqual(webResourceContent?.status, 200); - assert.strictEqual(await webResourceContent?.text(), await (await fetch(tokenizerJsonUrl)).text()); - - const readme2 = await downloadFile({ repo, path: "README.md", hubUrl: TEST_HUB_URL }); - assert.strictEqual(readme2, null); - } finally { - await deleteRepo({ - repo: { - name: repoName, - type: "model", - }, - hubUrl: TEST_HUB_URL, - credentials: { accessToken: TEST_ACCESS_TOKEN }, - }); - } - }, 60_000); - - it("should commit a full repo from HF with web urls", async function () { - const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; - const repo: RepoId = { - name: repoName, - type: "model", - }; - - await createRepo({ - accessToken: TEST_ACCESS_TOKEN, - repo, - hubUrl: TEST_HUB_URL, - }); - - try { - const FILES_TO_UPLOAD = [ - `https://huggingface.co/spaces/huggingfacejs/push-model-from-web/resolve/main/mobilenet/model.json`, - `https://huggingface.co/spaces/huggingfacejs/push-model-from-web/resolve/main/mobilenet/group1-shard1of2`, - `https://huggingface.co/spaces/huggingfacejs/push-model-from-web/resolve/main/mobilenet/group1-shard2of2`, - `https://huggingface.co/spaces/huggingfacejs/push-model-from-web/resolve/main/mobilenet/coffee.jpg`, - `https://huggingface.co/spaces/huggingfacejs/push-model-from-web/resolve/main/mobilenet/README.md`, - ]; - - const operations: CommitFile[] = await Promise.all( - FILES_TO_UPLOAD.map(async (file) => { - return { - operation: "addOrUpdate", - path: file.slice(file.indexOf("main/") + "main/".length), - // upload remote file - content: new URL(file), - }; - }) - ); - await commit({ - repo, - accessToken: TEST_ACCESS_TOKEN, - hubUrl: TEST_HUB_URL, - title: "upload model", - operations, - }); - - const LFSSize = (await fileDownloadInfo({ repo, path: "mobilenet/group1-shard1of2", hubUrl: TEST_HUB_URL })) - ?.size; - - assert.strictEqual(LFSSize, 4_194_304); - - const pointerFile = await downloadFile({ - repo, - path: "mobilenet/group1-shard1of2", - raw: true, - hubUrl: TEST_HUB_URL, - }); - - // Make sure SHA is computed properly as well - assert.strictEqual( - (await pointerFile?.text())?.trim(), - ` -version https://git-lfs.github.com/spec/v1 -oid sha256:3fb621eb9b37478239504ee083042d5b18699e8b8618e569478b03b119a85a69 -size 4194304 - `.trim() - ); - } finally { - await deleteRepo({ - repo: { - name: repoName, - type: "model", - }, - hubUrl: TEST_HUB_URL, - credentials: { accessToken: TEST_ACCESS_TOKEN }, - }); - } - // https://huggingfacejs-push-model-from-web.hf.space/ - }, 60_000); - - it("should be able to create a PR and then commit to it", async function () { - const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; - const repo: RepoId = { - name: repoName, - type: "model", - }; - - await createRepo({ - credentials: { - accessToken: TEST_ACCESS_TOKEN, - }, - repo, - hubUrl: TEST_HUB_URL, - }); - - try { - const pr = await commit({ - repo, - credentials: { - accessToken: TEST_ACCESS_TOKEN, - }, - hubUrl: TEST_HUB_URL, - title: "Create PR", - isPullRequest: true, - operations: [ - { - operation: "addOrUpdate", - content: new Blob(["This is me"]), - path: "test.txt", - }, - ], - }); - - if (!pr) { - throw new Error("PR creation failed"); - } - - if (!pr.pullRequestUrl) { - throw new Error("No pull request url"); - } - - const prNumber = pr.pullRequestUrl.split("/").pop(); - const prRef = `refs/pr/${prNumber}`; - - await commit({ - repo, - credentials: { - accessToken: TEST_ACCESS_TOKEN, - }, - hubUrl: TEST_HUB_URL, - branch: prRef, - title: "Some commit", - operations: [ - { - operation: "addOrUpdate", - content: new URL( - `https://huggingface.co/spaces/huggingfacejs/push-model-from-web/resolve/main/mobilenet/group1-shard1of2` - ), - path: "mobilenet/group1-shard1of2", - }, - ], - }); - - assert(commit, "PR commit failed"); - } finally { - await deleteRepo({ - repo: { - name: repoName, - type: "model", - }, - hubUrl: TEST_HUB_URL, - credentials: { accessToken: TEST_ACCESS_TOKEN }, - }); - } - }, 60_000); -}); diff --git a/packages/blob/src/lib/commit.ts b/packages/blob/src/lib/commit.ts deleted file mode 100644 index bd623a96c..000000000 --- a/packages/blob/src/lib/commit.ts +++ /dev/null @@ -1,597 +0,0 @@ -import { HUB_URL } from "../consts"; -import { HubApiError, createApiError, InvalidApiResponseFormatError } from "../error"; -import type { - ApiCommitHeader, - ApiCommitLfsFile, - ApiCommitOperation, - ApiLfsBatchRequest, - ApiLfsBatchResponse, - ApiLfsCompleteMultipartRequest, - ApiPreuploadRequest, - ApiPreuploadResponse, -} from "../types/api/api-commit"; -import type { CredentialsParams, RepoDesignation } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; -import { chunk } from "../utils/chunk"; -import { promisesQueue } from "../utils/promisesQueue"; -import { promisesQueueStreaming } from "../utils/promisesQueueStreaming"; -import { sha256 } from "../utils/sha256"; -import { toRepoId } from "../utils/toRepoId"; -import { WebBlob } from "../utils/WebBlob"; -import { createBlob } from "../utils/createBlob"; -import { eventToGenerator } from "../utils/eventToGenerator"; -import { base64FromBytes } from "../utils/base64FromBytes"; -import { isFrontend } from "../utils/isFrontend"; - -const CONCURRENT_SHAS = 5; -const CONCURRENT_LFS_UPLOADS = 5; -const MULTIPART_PARALLEL_UPLOAD = 5; - -export interface CommitDeletedEntry { - operation: "delete"; - path: string; -} - -export type ContentSource = Blob | URL; - -export interface CommitFile { - operation: "addOrUpdate"; - path: string; - content: ContentSource; - // forceLfs?: boolean -} - -type CommitBlob = Omit & { content: Blob }; - -// TODO: find a nice way to handle LFS & non-LFS files in an uniform manner, see https://github.com/huggingface/moon-landing/issues/4370 -// export type CommitRenameFile = { -// operation: "rename"; -// path: string; -// oldPath: string; -// content?: ContentSource; -// }; - -export type CommitOperation = CommitDeletedEntry | CommitFile /* | CommitRenameFile */; -type CommitBlobOperation = Exclude | CommitBlob; - -export type CommitParams = { - title: string; - description?: string; - repo: RepoDesignation; - operations: CommitOperation[]; - /** @default "main" */ - branch?: string; - /** - * Parent commit. Optional - * - * - When opening a PR: will use parentCommit as the parent commit - * - When committing on a branch: Will make sure that there were no intermediate commits - */ - parentCommit?: string; - isPullRequest?: boolean; - hubUrl?: string; - /** - * Whether to use web workers to compute SHA256 hashes. - * - * We load hash-wasm from a CDN inside the web worker. Not sure how to do otherwise and still have a "clean" bundle. - */ - useWebWorkers?: boolean | { minSize?: number; poolSize?: number }; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - abortSignal?: AbortSignal; - // Credentials are optional due to custom fetch functions or cookie auth -} & Partial; - -export interface CommitOutput { - pullRequestUrl?: string; - commit: { - oid: string; - url: string; - }; - hookOutput: string; -} - -function isFileOperation(op: CommitOperation): op is CommitBlob { - const ret = op.operation === "addOrUpdate"; - - if (ret && !(op.content instanceof Blob)) { - throw new TypeError("Precondition failed: op.content should be a Blob"); - } - - return ret; -} - -export type CommitProgressEvent = - | { - event: "phase"; - phase: "preuploading" | "uploadingLargeFiles" | "committing"; - } - | { - event: "fileProgress"; - path: string; - progress: number; - state: "hashing" | "uploading"; - }; - -/** - * Internal function for now, used by commit. - * - * Can be exposed later to offer fine-tuned progress info - */ -export async function* commitIter(params: CommitParams): AsyncGenerator { - const accessToken = checkCredentials(params); - const repoId = toRepoId(params.repo); - yield { event: "phase", phase: "preuploading" }; - - const lfsShas = new Map(); - - const abortController = new AbortController(); - const abortSignal = abortController.signal; - - // Polyfill see https://discuss.huggingface.co/t/why-cant-i-upload-a-parquet-file-to-my-dataset-error-o-throwifaborted-is-not-a-function/62245 - if (!abortSignal.throwIfAborted) { - abortSignal.throwIfAborted = () => { - if (abortSignal.aborted) { - throw new DOMException("Aborted", "AbortError"); - } - }; - } - - if (params.abortSignal) { - params.abortSignal.addEventListener("abort", () => abortController.abort()); - } - - try { - const allOperations = await Promise.all( - params.operations.map(async (operation) => { - if (operation.operation !== "addOrUpdate") { - return operation; - } - - if (!(operation.content instanceof URL)) { - /** TS trick to enforce `content` to be a `Blob` */ - return { ...operation, content: operation.content }; - } - - const lazyBlob = await createBlob(operation.content, { fetch: params.fetch }); - - abortSignal?.throwIfAborted(); - - return { - ...operation, - content: lazyBlob, - }; - }) - ); - - const gitAttributes = allOperations.filter(isFileOperation).find((op) => op.path === ".gitattributes")?.content; - - for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { - const payload: ApiPreuploadRequest = { - gitAttributes: gitAttributes && (await gitAttributes.text()), - files: await Promise.all( - operations.map(async (operation) => ({ - path: operation.path, - size: operation.content.size, - sample: base64FromBytes(new Uint8Array(await operation.content.slice(0, 512).arrayBuffer())), - })) - ), - }; - - abortSignal?.throwIfAborted(); - - const res = await (params.fetch ?? fetch)( - `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/preupload/${encodeURIComponent( - params.branch ?? "main" - )}` + (params.isPullRequest ? "?create_pr=1" : ""), - { - method: "POST", - headers: { - ...(accessToken && { Authorization: `Bearer ${accessToken}` }), - "Content-Type": "application/json", - }, - body: JSON.stringify(payload), - signal: abortSignal, - } - ); - - if (!res.ok) { - throw await createApiError(res); - } - - const json: ApiPreuploadResponse = await res.json(); - - for (const file of json.files) { - if (file.uploadMode === "lfs") { - lfsShas.set(file.path, null); - } - } - } - - yield { event: "phase", phase: "uploadingLargeFiles" }; - - for (const operations of chunk( - allOperations.filter(isFileOperation).filter((op) => lfsShas.has(op.path)), - 100 - )) { - const shas = yield* eventToGenerator< - { event: "fileProgress"; state: "hashing"; path: string; progress: number }, - string[] - >((yieldCallback, returnCallback, rejectCallack) => { - return promisesQueue( - operations.map((op) => async () => { - const iterator = sha256(op.content, { useWebWorker: params.useWebWorkers, abortSignal: abortSignal }); - let res: IteratorResult; - do { - res = await iterator.next(); - if (!res.done) { - yieldCallback({ event: "fileProgress", path: op.path, progress: res.value, state: "hashing" }); - } - } while (!res.done); - const sha = res.value; - lfsShas.set(op.path, res.value); - return sha; - }), - CONCURRENT_SHAS - ).then(returnCallback, rejectCallack); - }); - - abortSignal?.throwIfAborted(); - - const payload: ApiLfsBatchRequest = { - operation: "upload", - // multipart is a custom protocol for HF - transfers: ["basic", "multipart"], - hash_algo: "sha_256", - ...(!params.isPullRequest && { - ref: { - name: params.branch ?? "main", - }, - }), - objects: operations.map((op, i) => ({ - oid: shas[i], - size: op.content.size, - })), - }; - - const res = await (params.fetch ?? fetch)( - `${params.hubUrl ?? HUB_URL}/${repoId.type === "model" ? "" : repoId.type + "s/"}${ - repoId.name - }.git/info/lfs/objects/batch`, - { - method: "POST", - headers: { - ...(accessToken && { Authorization: `Bearer ${accessToken}` }), - Accept: "application/vnd.git-lfs+json", - "Content-Type": "application/vnd.git-lfs+json", - }, - body: JSON.stringify(payload), - signal: abortSignal, - } - ); - - if (!res.ok) { - throw await createApiError(res); - } - - const json: ApiLfsBatchResponse = await res.json(); - const batchRequestId = res.headers.get("X-Request-Id") || undefined; - - const shaToOperation = new Map(operations.map((op, i) => [shas[i], op])); - - yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { - return promisesQueueStreaming( - json.objects.map((obj) => async () => { - const op = shaToOperation.get(obj.oid); - - if (!op) { - throw new InvalidApiResponseFormatError("Unrequested object ID in response"); - } - - abortSignal?.throwIfAborted(); - - if (obj.error) { - const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${ - obj.error.message - }${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; - throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); - } - if (!obj.actions?.upload) { - // Already uploaded - yieldCallback({ - event: "fileProgress", - path: op.path, - progress: 1, - state: "uploading", - }); - return; - } - yieldCallback({ - event: "fileProgress", - path: op.path, - progress: 0, - state: "uploading", - }); - const content = op.content; - const header = obj.actions.upload.header; - if (header?.chunk_size) { - const chunkSize = parseInt(header.chunk_size); - - // multipart upload - // parts are in upload.header['00001'] to upload.header['99999'] - - const completionUrl = obj.actions.upload.href; - const parts = Object.keys(header).filter((key) => /^[0-9]+$/.test(key)); - - if (parts.length !== Math.ceil(content.size / chunkSize)) { - throw new Error("Invalid server response to upload large LFS file, wrong number of parts"); - } - - const completeReq: ApiLfsCompleteMultipartRequest = { - oid: obj.oid, - parts: parts.map((part) => ({ - partNumber: +part, - etag: "", - })), - }; - - // Defined here so that it's not redefined at each iteration (and the caller can tell it's for the same file) - const progressCallback = (progress: number) => - yieldCallback({ event: "fileProgress", path: op.path, progress, state: "uploading" }); - - await promisesQueueStreaming( - parts.map((part) => async () => { - abortSignal?.throwIfAborted(); - - const index = parseInt(part) - 1; - const slice = content.slice(index * chunkSize, (index + 1) * chunkSize); - - const res = await (params.fetch ?? fetch)(header[part], { - method: "PUT", - /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ - body: slice instanceof WebBlob && isFrontend ? await slice.arrayBuffer() : slice, - signal: abortSignal, - ...({ - progressHint: { - path: op.path, - part: index, - numParts: parts.length, - progressCallback, - }, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any), - }); - - if (!res.ok) { - throw await createApiError(res, { - requestId: batchRequestId, - message: `Error while uploading part ${part} of ${ - operations[shas.indexOf(obj.oid)].path - } to LFS storage`, - }); - } - - const eTag = res.headers.get("ETag"); - - if (!eTag) { - throw new Error("Cannot get ETag of part during multipart upload"); - } - - completeReq.parts[Number(part) - 1].etag = eTag; - }), - MULTIPART_PARALLEL_UPLOAD - ); - - abortSignal?.throwIfAborted(); - - const res = await (params.fetch ?? fetch)(completionUrl, { - method: "POST", - body: JSON.stringify(completeReq), - headers: { - Accept: "application/vnd.git-lfs+json", - "Content-Type": "application/vnd.git-lfs+json", - }, - signal: abortSignal, - }); - - if (!res.ok) { - throw await createApiError(res, { - requestId: batchRequestId, - message: `Error completing multipart upload of ${ - operations[shas.indexOf(obj.oid)].path - } to LFS storage`, - }); - } - - yieldCallback({ - event: "fileProgress", - path: op.path, - progress: 1, - state: "uploading", - }); - } else { - const res = await (params.fetch ?? fetch)(obj.actions.upload.href, { - method: "PUT", - headers: { - ...(batchRequestId ? { "X-Request-Id": batchRequestId } : undefined), - }, - /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ - body: content instanceof WebBlob && isFrontend ? await content.arrayBuffer() : content, - signal: abortSignal, - ...({ - progressHint: { - path: op.path, - progressCallback: (progress: number) => - yieldCallback({ - event: "fileProgress", - path: op.path, - progress, - state: "uploading", - }), - }, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any), - }); - - if (!res.ok) { - throw await createApiError(res, { - requestId: batchRequestId, - message: `Error while uploading ${operations[shas.indexOf(obj.oid)].path} to LFS storage`, - }); - } - - yieldCallback({ - event: "fileProgress", - path: op.path, - progress: 1, - state: "uploading", - }); - } - }), - CONCURRENT_LFS_UPLOADS - ).then(returnCallback, rejectCallback); - }); - } - - abortSignal?.throwIfAborted(); - - yield { event: "phase", phase: "committing" }; - - return yield* eventToGenerator( - async (yieldCallback, returnCallback, rejectCallback) => - (params.fetch ?? fetch)( - `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commit/${encodeURIComponent( - params.branch ?? "main" - )}` + (params.isPullRequest ? "?create_pr=1" : ""), - { - method: "POST", - headers: { - ...(accessToken && { Authorization: `Bearer ${accessToken}` }), - "Content-Type": "application/x-ndjson", - }, - body: [ - { - key: "header", - value: { - summary: params.title, - description: params.description, - parentCommit: params.parentCommit, - } satisfies ApiCommitHeader, - }, - ...((await Promise.all( - allOperations.map((operation) => { - if (isFileOperation(operation)) { - const sha = lfsShas.get(operation.path); - if (sha) { - return { - key: "lfsFile", - value: { - path: operation.path, - algo: "sha256", - size: operation.content.size, - oid: sha, - } satisfies ApiCommitLfsFile, - }; - } - } - - return convertOperationToNdJson(operation); - }) - )) satisfies ApiCommitOperation[]), - ] - .map((x) => JSON.stringify(x)) - .join("\n"), - signal: abortSignal, - ...({ - progressHint: { - progressCallback: (progress: number) => { - // For now, we display equal progress for all files - // We could compute the progress based on the size of `convertOperationToNdJson` for each of the files instead - for (const op of allOperations) { - if (isFileOperation(op) && !lfsShas.has(op.path)) { - yieldCallback({ - event: "fileProgress", - path: op.path, - progress, - state: "uploading", - }); - } - } - }, - }, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any), - } - ) - .then(async (res) => { - if (!res.ok) { - throw await createApiError(res); - } - - const json = await res.json(); - - returnCallback({ - pullRequestUrl: json.pullRequestUrl, - commit: { - oid: json.commitOid, - url: json.commitUrl, - }, - hookOutput: json.hookOutput, - }); - }) - .catch(rejectCallback) - ); - } catch (err) { - // For parallel requests, cancel them all if one fails - abortController.abort(); - throw err; - } -} - -export async function commit(params: CommitParams): Promise { - const iterator = commitIter(params); - let res = await iterator.next(); - while (!res.done) { - res = await iterator.next(); - } - return res.value; -} - -async function convertOperationToNdJson(operation: CommitBlobOperation): Promise { - switch (operation.operation) { - case "addOrUpdate": { - // todo: handle LFS - return { - key: "file", - value: { - content: base64FromBytes(new Uint8Array(await operation.content.arrayBuffer())), - path: operation.path, - encoding: "base64", - }, - }; - } - // case "rename": { - // // todo: detect when remote file is already LFS, and in that case rename as LFS - // return { - // key: "file", - // value: { - // content: operation.content, - // path: operation.path, - // oldPath: operation.oldPath - // } - // }; - // } - case "delete": { - return { - key: "deletedFile", - value: { - path: operation.path, - }, - }; - } - default: - throw new TypeError("Unknown operation: " + (operation as { operation: string }).operation); - } -} diff --git a/packages/blob/src/lib/count-commits.spec.ts b/packages/blob/src/lib/count-commits.spec.ts deleted file mode 100644 index f60754789..000000000 --- a/packages/blob/src/lib/count-commits.spec.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { assert, it, describe } from "vitest"; -import { countCommits } from "./count-commits"; - -describe("countCommits", () => { - it("should fetch paginated commits from the repo", async () => { - const count = await countCommits({ - repo: { - name: "openai-community/gpt2", - type: "model", - }, - revision: "607a30d783dfa663caf39e06633721c8d4cfcd7e", - }); - - assert.equal(count, 26); - }); -}); diff --git a/packages/blob/src/lib/count-commits.ts b/packages/blob/src/lib/count-commits.ts deleted file mode 100644 index 0e1332532..000000000 --- a/packages/blob/src/lib/count-commits.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; -import type { CredentialsParams, RepoDesignation } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; -import { toRepoId } from "../utils/toRepoId"; - -export async function countCommits( - params: { - repo: RepoDesignation; - /** - * Revision to list commits from. Defaults to the default branch. - */ - revision?: string; - hubUrl?: string; - fetch?: typeof fetch; - } & Partial -): Promise { - const accessToken = checkCredentials(params); - const repoId = toRepoId(params.repo); - - // Could upgrade to 1000 commits per page - const url: string | undefined = `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commits/${ - params.revision ?? "main" - }?limit=1`; - - const res: Response = await (params.fetch ?? fetch)(url, { - headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {}, - }); - - if (!res.ok) { - throw await createApiError(res); - } - - return parseInt(res.headers.get("x-total-count") ?? "0", 10); -} diff --git a/packages/blob/src/lib/create-repo.spec.ts b/packages/blob/src/lib/create-repo.spec.ts deleted file mode 100644 index c1a39b9f8..000000000 --- a/packages/blob/src/lib/create-repo.spec.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { assert, it, describe, expect } from "vitest"; - -import { TEST_HUB_URL, TEST_ACCESS_TOKEN, TEST_USER } from "../test/consts"; -import { insecureRandomString } from "../utils/insecureRandomString"; -import { createRepo } from "./create-repo"; -import { deleteRepo } from "./delete-repo"; -import { downloadFile } from "./download-file"; - -describe("createRepo", () => { - it("should create a repo", async () => { - const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; - - const result = await createRepo({ - accessToken: TEST_ACCESS_TOKEN, - repo: { - name: repoName, - type: "model", - }, - hubUrl: TEST_HUB_URL, - files: [{ path: ".gitattributes", content: new Blob(["*.html filter=lfs diff=lfs merge=lfs -text"]) }], - }); - - assert.deepStrictEqual(result, { - repoUrl: `${TEST_HUB_URL}/${repoName}`, - }); - - const content = await downloadFile({ - repo: { - name: repoName, - type: "model", - }, - path: ".gitattributes", - hubUrl: TEST_HUB_URL, - }); - - assert(content); - assert.strictEqual(await content.text(), "*.html filter=lfs diff=lfs merge=lfs -text"); - - await deleteRepo({ - repo: { - name: repoName, - type: "model", - }, - credentials: { accessToken: TEST_ACCESS_TOKEN }, - hubUrl: TEST_HUB_URL, - }); - }); - - it("should throw a client error when trying to create a repo without a fully-qualified name", async () => { - const tryCreate = createRepo({ - repo: { name: "canonical", type: "model" }, - credentials: { accessToken: TEST_ACCESS_TOKEN }, - hubUrl: TEST_HUB_URL, - }); - - await expect(tryCreate).rejects.toBeInstanceOf(TypeError); - }); - - it("should create a model with a string as name", async () => { - const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; - - const result = await createRepo({ - accessToken: TEST_ACCESS_TOKEN, - hubUrl: TEST_HUB_URL, - repo: repoName, - files: [{ path: ".gitattributes", content: new Blob(["*.html filter=lfs diff=lfs merge=lfs -text"]) }], - }); - - assert.deepStrictEqual(result, { - repoUrl: `${TEST_HUB_URL}/${repoName}`, - }); - - await deleteRepo({ - repo: { - name: repoName, - type: "model", - }, - hubUrl: TEST_HUB_URL, - credentials: { accessToken: TEST_ACCESS_TOKEN }, - }); - }); - - it("should create a dataset with a string as name", async () => { - const repoName = `datasets/${TEST_USER}/TEST-${insecureRandomString()}`; - - const result = await createRepo({ - accessToken: TEST_ACCESS_TOKEN, - hubUrl: TEST_HUB_URL, - repo: repoName, - files: [{ path: ".gitattributes", content: new Blob(["*.html filter=lfs diff=lfs merge=lfs -text"]) }], - }); - - assert.deepStrictEqual(result, { - repoUrl: `${TEST_HUB_URL}/${repoName}`, - }); - - await deleteRepo({ - repo: repoName, - hubUrl: TEST_HUB_URL, - credentials: { accessToken: TEST_ACCESS_TOKEN }, - }); - }); -}, 10_000); diff --git a/packages/blob/src/lib/create-repo.ts b/packages/blob/src/lib/create-repo.ts deleted file mode 100644 index 4005844d6..000000000 --- a/packages/blob/src/lib/create-repo.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; -import type { ApiCreateRepoPayload } from "../types/api/api-create-repo"; -import type { CredentialsParams, RepoDesignation, SpaceSdk } from "../types/public"; -import { base64FromBytes } from "../utils/base64FromBytes"; -import { checkCredentials } from "../utils/checkCredentials"; -import { toRepoId } from "../utils/toRepoId"; - -export async function createRepo( - params: { - repo: RepoDesignation; - private?: boolean; - license?: string; - /** - * Only a few lightweight files are supported at repo creation - */ - files?: Array<{ content: ArrayBuffer | Blob; path: string }>; - /** @required for when {@link repo.type} === "space" */ - sdk?: SpaceSdk; - hubUrl?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & CredentialsParams -): Promise<{ repoUrl: string }> { - const accessToken = checkCredentials(params); - const repoId = toRepoId(params.repo); - const [namespace, repoName] = repoId.name.split("/"); - - if (!namespace || !repoName) { - throw new TypeError( - `"${repoId.name}" is not a fully qualified repo name. It should be of the form "{namespace}/{repoName}".` - ); - } - - const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/repos/create`, { - method: "POST", - body: JSON.stringify({ - name: repoName, - private: params.private, - organization: namespace, - license: params.license, - ...(repoId.type === "space" - ? { - type: "space", - sdk: "static", - } - : { - type: repoId.type, - }), - files: params.files - ? await Promise.all( - params.files.map(async (file) => ({ - encoding: "base64", - path: file.path, - content: base64FromBytes( - new Uint8Array(file.content instanceof Blob ? await file.content.arrayBuffer() : file.content) - ), - })) - ) - : undefined, - } satisfies ApiCreateRepoPayload), - headers: { - Authorization: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - - if (!res.ok) { - throw await createApiError(res); - } - const output = await res.json(); - return { repoUrl: output.url }; -} diff --git a/packages/blob/src/lib/dataset-info.spec.ts b/packages/blob/src/lib/dataset-info.spec.ts deleted file mode 100644 index 982bdcf40..000000000 --- a/packages/blob/src/lib/dataset-info.spec.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { datasetInfo } from "./dataset-info"; -import type { DatasetEntry } from "./list-datasets"; -import type { ApiDatasetInfo } from "../types/api/api-dataset"; - -describe("datasetInfo", () => { - it("should return the dataset info", async () => { - const info = await datasetInfo({ - name: "nyu-mll/glue", - }); - expect(info).toEqual({ - id: "621ffdd236468d709f181e3f", - downloads: expect.any(Number), - gated: false, - name: "nyu-mll/glue", - updatedAt: expect.any(Date), - likes: expect.any(Number), - private: false, - }); - }); - - it("should return the dataset info with author", async () => { - const info: DatasetEntry & Pick = await datasetInfo({ - name: "nyu-mll/glue", - additionalFields: ['author'], - }); - expect(info).toEqual({ - id: "621ffdd236468d709f181e3f", - downloads: expect.any(Number), - gated: false, - name: "nyu-mll/glue", - updatedAt: expect.any(Date), - likes: expect.any(Number), - private: false, - author: 'nyu-mll' - }); - }); - - it("should return the dataset info for a specific revision", async () => { - const info: DatasetEntry & Pick = await datasetInfo({ - name: "nyu-mll/glue", - revision: "cb2099c76426ff97da7aa591cbd317d91fb5fcb7", - additionalFields: ["sha"], - }); - expect(info).toEqual({ - id: "621ffdd236468d709f181e3f", - downloads: expect.any(Number), - gated: false, - name: "nyu-mll/glue", - updatedAt: expect.any(Date), - likes: expect.any(Number), - private: false, - sha: 'cb2099c76426ff97da7aa591cbd317d91fb5fcb7' - }); - }); -}); diff --git a/packages/blob/src/lib/dataset-info.ts b/packages/blob/src/lib/dataset-info.ts deleted file mode 100644 index bb62df7f8..000000000 --- a/packages/blob/src/lib/dataset-info.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; -import type { ApiDatasetInfo } from "../types/api/api-dataset"; -import type { CredentialsParams } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; -import { pick } from "../utils/pick"; -import { type DATASET_EXPANDABLE_KEYS, DATASET_EXPAND_KEYS, type DatasetEntry } from "./list-datasets"; - -export async function datasetInfo< - const T extends Exclude<(typeof DATASET_EXPANDABLE_KEYS)[number], (typeof DATASET_EXPAND_KEYS)[number]> = never, ->( - params: { - name: string; - hubUrl?: string; - additionalFields?: T[]; - /** - * An optional Git revision id which can be a branch name, a tag, or a commit hash. - */ - revision?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): Promise> { - const accessToken = params && checkCredentials(params); - - const search = new URLSearchParams([ - ...DATASET_EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), - ...(params?.additionalFields?.map((val) => ["expand", val] satisfies [string, string]) ?? []), - ]).toString(); - - const response = await (params.fetch || fetch)( - `${params?.hubUrl || HUB_URL}/api/datasets/${params.name}/revision/${encodeURIComponent(params.revision ?? "HEAD")}?${search.toString()}`, - { - headers: { - ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), - Accepts: "application/json", - }, - } - ); - - if (!response.ok) { - throw await createApiError(response); - } - - const data = await response.json(); - - return { - ...(params?.additionalFields && pick(data, params.additionalFields)), - id: data._id, - name: data.id, - private: data.private, - downloads: data.downloads, - likes: data.likes, - gated: data.gated, - updatedAt: new Date(data.lastModified), - } as DatasetEntry & Pick; -} diff --git a/packages/blob/src/lib/delete-file.spec.ts b/packages/blob/src/lib/delete-file.spec.ts deleted file mode 100644 index ec38a916d..000000000 --- a/packages/blob/src/lib/delete-file.spec.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { assert, it, describe } from "vitest"; - -import { TEST_ACCESS_TOKEN, TEST_HUB_URL, TEST_USER } from "../test/consts"; -import type { RepoId } from "../types/public"; -import { insecureRandomString } from "../utils/insecureRandomString"; -import { createRepo } from "./create-repo"; -import { deleteRepo } from "./delete-repo"; -import { deleteFile } from "./delete-file"; -import { downloadFile } from "./download-file"; - -describe("deleteFile", () => { - it("should delete a file", async () => { - const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; - const repo = { type: "model", name: repoName } satisfies RepoId; - - try { - const result = await createRepo({ - accessToken: TEST_ACCESS_TOKEN, - hubUrl: TEST_HUB_URL, - repo, - files: [ - { path: "file1", content: new Blob(["file1"]) }, - { path: "file2", content: new Blob(["file2"]) }, - ], - }); - - assert.deepStrictEqual(result, { - repoUrl: `${TEST_HUB_URL}/${repoName}`, - }); - - let content = await downloadFile({ - hubUrl: TEST_HUB_URL, - repo, - path: "file1", - }); - - assert.strictEqual(await content?.text(), "file1"); - - await deleteFile({ path: "file1", repo, accessToken: TEST_ACCESS_TOKEN, hubUrl: TEST_HUB_URL }); - - content = await downloadFile({ - repo, - path: "file1", - hubUrl: TEST_HUB_URL, - }); - - assert.strictEqual(content, null); - - content = await downloadFile({ - repo, - path: "file2", - hubUrl: TEST_HUB_URL, - }); - - assert.strictEqual(await content?.text(), "file2"); - } finally { - await deleteRepo({ - repo, - accessToken: TEST_ACCESS_TOKEN, - hubUrl: TEST_HUB_URL, - }); - } - }); -}); diff --git a/packages/blob/src/lib/delete-file.ts b/packages/blob/src/lib/delete-file.ts deleted file mode 100644 index 58b19f198..000000000 --- a/packages/blob/src/lib/delete-file.ts +++ /dev/null @@ -1,35 +0,0 @@ -import type { CredentialsParams } from "../types/public"; -import type { CommitOutput, CommitParams } from "./commit"; -import { commit } from "./commit"; - -export function deleteFile( - params: { - repo: CommitParams["repo"]; - path: string; - commitTitle?: CommitParams["title"]; - commitDescription?: CommitParams["description"]; - hubUrl?: CommitParams["hubUrl"]; - fetch?: CommitParams["fetch"]; - branch?: CommitParams["branch"]; - isPullRequest?: CommitParams["isPullRequest"]; - parentCommit?: CommitParams["parentCommit"]; - } & CredentialsParams -): Promise { - return commit({ - ...(params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }), - repo: params.repo, - operations: [ - { - operation: "delete", - path: params.path, - }, - ], - title: params.commitTitle ?? `Delete ${params.path}`, - description: params.commitDescription, - hubUrl: params.hubUrl, - branch: params.branch, - isPullRequest: params.isPullRequest, - parentCommit: params.parentCommit, - fetch: params.fetch, - }); -} diff --git a/packages/blob/src/lib/delete-files.spec.ts b/packages/blob/src/lib/delete-files.spec.ts deleted file mode 100644 index 558da6a6b..000000000 --- a/packages/blob/src/lib/delete-files.spec.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { assert, it, describe } from "vitest"; - -import { TEST_HUB_URL, TEST_ACCESS_TOKEN, TEST_USER } from "../test/consts"; -import type { RepoId } from "../types/public"; -import { insecureRandomString } from "../utils/insecureRandomString"; -import { createRepo } from "./create-repo"; -import { deleteRepo } from "./delete-repo"; -import { deleteFiles } from "./delete-files"; -import { downloadFile } from "./download-file"; - -describe("deleteFiles", () => { - it("should delete multiple files", async () => { - const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; - const repo = { type: "model", name: repoName } satisfies RepoId; - - try { - const result = await createRepo({ - accessToken: TEST_ACCESS_TOKEN, - repo, - files: [ - { path: "file1", content: new Blob(["file1"]) }, - { path: "file2", content: new Blob(["file2"]) }, - { path: "file3", content: new Blob(["file3"]) }, - ], - hubUrl: TEST_HUB_URL, - }); - - assert.deepStrictEqual(result, { - repoUrl: `${TEST_HUB_URL}/${repoName}`, - }); - - let content = await downloadFile({ - repo, - path: "file1", - hubUrl: TEST_HUB_URL, - }); - - assert.strictEqual(await content?.text(), "file1"); - - content = await downloadFile({ - repo, - path: "file2", - hubUrl: TEST_HUB_URL, - }); - - assert.strictEqual(await content?.text(), "file2"); - - await deleteFiles({ paths: ["file1", "file2"], repo, accessToken: TEST_ACCESS_TOKEN, hubUrl: TEST_HUB_URL }); - - content = await downloadFile({ - repo, - path: "file1", - hubUrl: TEST_HUB_URL, - }); - - assert.strictEqual(content, null); - - content = await downloadFile({ - repo, - path: "file2", - hubUrl: TEST_HUB_URL, - }); - - assert.strictEqual(content, null); - - content = await downloadFile({ - repo, - path: "file3", - hubUrl: TEST_HUB_URL, - }); - - assert.strictEqual(await content?.text(), "file3"); - } finally { - await deleteRepo({ - repo, - accessToken: TEST_ACCESS_TOKEN, - hubUrl: TEST_HUB_URL, - }); - } - }); -}, 10_000); diff --git a/packages/blob/src/lib/delete-files.ts b/packages/blob/src/lib/delete-files.ts deleted file mode 100644 index 956bd473e..000000000 --- a/packages/blob/src/lib/delete-files.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { CredentialsParams } from "../types/public"; -import type { CommitOutput, CommitParams } from "./commit"; -import { commit } from "./commit"; - -export function deleteFiles( - params: { - repo: CommitParams["repo"]; - paths: string[]; - commitTitle?: CommitParams["title"]; - commitDescription?: CommitParams["description"]; - hubUrl?: CommitParams["hubUrl"]; - branch?: CommitParams["branch"]; - isPullRequest?: CommitParams["isPullRequest"]; - parentCommit?: CommitParams["parentCommit"]; - fetch?: CommitParams["fetch"]; - } & CredentialsParams -): Promise { - return commit({ - ...(params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }), - repo: params.repo, - operations: params.paths.map((path) => ({ - operation: "delete", - path, - })), - title: params.commitTitle ?? `Deletes ${params.paths.length} files`, - description: params.commitDescription, - hubUrl: params.hubUrl, - branch: params.branch, - isPullRequest: params.isPullRequest, - parentCommit: params.parentCommit, - fetch: params.fetch, - }); -} diff --git a/packages/blob/src/lib/delete-repo.ts b/packages/blob/src/lib/delete-repo.ts deleted file mode 100644 index 7b34d1b8e..000000000 --- a/packages/blob/src/lib/delete-repo.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; -import type { CredentialsParams, RepoDesignation } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; -import { toRepoId } from "../utils/toRepoId"; - -export async function deleteRepo( - params: { - repo: RepoDesignation; - hubUrl?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & CredentialsParams -): Promise { - const accessToken = checkCredentials(params); - const repoId = toRepoId(params.repo); - const [namespace, repoName] = repoId.name.split("/"); - - const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/repos/delete`, { - method: "DELETE", - body: JSON.stringify({ - name: repoName, - organization: namespace, - type: repoId.type, - }), - headers: { - Authorization: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - - if (!res.ok) { - throw await createApiError(res); - } -} diff --git a/packages/blob/src/lib/download-file-to-cache-dir.spec.ts b/packages/blob/src/lib/download-file-to-cache-dir.spec.ts deleted file mode 100644 index 05e2e6de9..000000000 --- a/packages/blob/src/lib/download-file-to-cache-dir.spec.ts +++ /dev/null @@ -1,234 +0,0 @@ -import { expect, test, describe, vi, beforeEach } from "vitest"; -import type { RepoDesignation, RepoId } from "../types/public"; -import { dirname, join } from "node:path"; -import { lstat, mkdir, stat, symlink, writeFile, rename } from "node:fs/promises"; -import { pathsInfo } from "./paths-info"; -import type { Stats } from "node:fs"; -import { getHFHubCachePath, getRepoFolderName } from "./cache-management"; -import { toRepoId } from "../utils/toRepoId"; -import { downloadFileToCacheDir } from "./download-file-to-cache-dir"; - -vi.mock('node:fs/promises', () => ({ - writeFile: vi.fn(), - rename: vi.fn(), - symlink: vi.fn(), - lstat: vi.fn(), - mkdir: vi.fn(), - stat: vi.fn() -})); - -vi.mock('./paths-info', () => ({ - pathsInfo: vi.fn(), -})); - -const DUMMY_REPO: RepoId = { - name: 'hello-world', - type: 'model', -}; - -const DUMMY_ETAG = "dummy-etag"; - -// utility test method to get blob file path -function _getBlobFile(params: { - repo: RepoDesignation; - etag: string; - cacheDir?: string, // default to {@link getHFHubCache} -}) { - return join(params.cacheDir ?? getHFHubCachePath(), getRepoFolderName(toRepoId(params.repo)), "blobs", params.etag); -} - -// utility test method to get snapshot file path -function _getSnapshotFile(params: { - repo: RepoDesignation; - path: string; - revision : string; - cacheDir?: string, // default to {@link getHFHubCache} -}) { - return join(params.cacheDir ?? getHFHubCachePath(), getRepoFolderName(toRepoId(params.repo)), "snapshots", params.revision, params.path); -} - -describe('downloadFileToCacheDir', () => { - const fetchMock: typeof fetch = vi.fn(); - beforeEach(() => { - vi.resetAllMocks(); - // mock 200 request - vi.mocked(fetchMock).mockResolvedValue({ - status: 200, - ok: true, - body: 'dummy-body' - } as unknown as Response); - - // prevent to use caching - vi.mocked(stat).mockRejectedValue(new Error('Do not exists')); - vi.mocked(lstat).mockRejectedValue(new Error('Do not exists')); - }); - - test('should throw an error if fileDownloadInfo return nothing', async () => { - await expect(async () => { - await downloadFileToCacheDir({ - repo: DUMMY_REPO, - path: '/README.md', - fetch: fetchMock, - }); - }).rejects.toThrowError('cannot get path info for /README.md'); - - expect(pathsInfo).toHaveBeenCalledWith(expect.objectContaining({ - repo: DUMMY_REPO, - paths: ['/README.md'], - fetch: fetchMock, - })); - }); - - test('existing symlinked and blob should not re-download it', async () => { - // ///snapshots/README.md - const expectPointer = _getSnapshotFile({ - repo: DUMMY_REPO, - path: '/README.md', - revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", - }); - // stat ensure a symlink and the pointed file exists - vi.mocked(stat).mockResolvedValue({} as Stats) // prevent default mocked reject - - const output = await downloadFileToCacheDir({ - repo: DUMMY_REPO, - path: '/README.md', - fetch: fetchMock, - revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", - }); - - expect(stat).toHaveBeenCalledOnce(); - // Get call argument for stat - const starArg = vi.mocked(stat).mock.calls[0][0]; - - expect(starArg).toBe(expectPointer) - expect(fetchMock).not.toHaveBeenCalledWith(); - - expect(output).toBe(expectPointer); - }); - - test('existing blob should only create the symlink', async () => { - // ///snapshots/README.md - const expectPointer = _getSnapshotFile({ - repo: DUMMY_REPO, - path: '/README.md', - revision: "dummy-commit-hash", - }); - // //blobs/ - const expectedBlob = _getBlobFile({ - repo: DUMMY_REPO, - etag: DUMMY_ETAG, - }); - - // mock existing blob only no symlink - vi.mocked(lstat).mockResolvedValue({} as Stats); - // mock pathsInfo resolve content - vi.mocked(pathsInfo).mockResolvedValue([{ - oid: DUMMY_ETAG, - size: 55, - path: 'README.md', - type: 'file', - lastCommit: { - date: new Date(), - id: 'dummy-commit-hash', - title: 'Commit msg', - }, - }]); - - const output = await downloadFileToCacheDir({ - repo: DUMMY_REPO, - path: '/README.md', - fetch: fetchMock, - }); - - expect(stat).not.toHaveBeenCalled(); - // should have check for the blob - expect(lstat).toHaveBeenCalled(); - expect(vi.mocked(lstat).mock.calls[0][0]).toBe(expectedBlob); - - // symlink should have been created - expect(symlink).toHaveBeenCalledOnce(); - // no download done - expect(fetchMock).not.toHaveBeenCalled(); - - expect(output).toBe(expectPointer); - }); - - test('expect resolve value to be the pointer path of downloaded file', async () => { - // ///snapshots/README.md - const expectPointer = _getSnapshotFile({ - repo: DUMMY_REPO, - path: '/README.md', - revision: "dummy-commit-hash", - }); - // //blobs/ - const expectedBlob = _getBlobFile({ - repo: DUMMY_REPO, - etag: DUMMY_ETAG, - }); - - vi.mocked(pathsInfo).mockResolvedValue([{ - oid: DUMMY_ETAG, - size: 55, - path: 'README.md', - type: 'file', - lastCommit: { - date: new Date(), - id: 'dummy-commit-hash', - title: 'Commit msg', - }, - }]); - - const output = await downloadFileToCacheDir({ - repo: DUMMY_REPO, - path: '/README.md', - fetch: fetchMock, - }); - - // expect blobs and snapshots folder to have been mkdir - expect(vi.mocked(mkdir).mock.calls[0][0]).toBe(dirname(expectedBlob)); - expect(vi.mocked(mkdir).mock.calls[1][0]).toBe(dirname(expectPointer)); - - expect(output).toBe(expectPointer); - }); - - test('should write fetch response to blob', async () => { - // ///snapshots/README.md - const expectPointer = _getSnapshotFile({ - repo: DUMMY_REPO, - path: '/README.md', - revision: "dummy-commit-hash", - }); - // //blobs/ - const expectedBlob = _getBlobFile({ - repo: DUMMY_REPO, - etag: DUMMY_ETAG, - }); - - // mock pathsInfo resolve content - vi.mocked(pathsInfo).mockResolvedValue([{ - oid: DUMMY_ETAG, - size: 55, - path: 'README.md', - type: 'file', - lastCommit: { - date: new Date(), - id: 'dummy-commit-hash', - title: 'Commit msg', - }, - }]); - - await downloadFileToCacheDir({ - repo: DUMMY_REPO, - path: '/README.md', - fetch: fetchMock, - }); - - const incomplete = `${expectedBlob}.incomplete`; - // 1. should write fetch#response#body to incomplete file - expect(writeFile).toHaveBeenCalledWith(incomplete, 'dummy-body'); - // 2. should rename the incomplete to the blob expected name - expect(rename).toHaveBeenCalledWith(incomplete, expectedBlob); - // 3. should create symlink pointing to blob - expect(symlink).toHaveBeenCalledWith(expectedBlob, expectPointer); - }); -}); \ No newline at end of file diff --git a/packages/blob/src/lib/download-file-to-cache-dir.ts b/packages/blob/src/lib/download-file-to-cache-dir.ts deleted file mode 100644 index 72869f307..000000000 --- a/packages/blob/src/lib/download-file-to-cache-dir.ts +++ /dev/null @@ -1,129 +0,0 @@ -import { getHFHubCachePath, getRepoFolderName } from "./cache-management"; -import { dirname, join } from "node:path"; -import { writeFile, rename, symlink, lstat, mkdir, stat } from "node:fs/promises"; -import type { CommitInfo, PathInfo } from "./paths-info"; -import { pathsInfo } from "./paths-info"; -import type { CredentialsParams, RepoDesignation } from "../types/public"; -import { toRepoId } from "../utils/toRepoId"; -import { downloadFile } from "./download-file"; - -export const REGEX_COMMIT_HASH: RegExp = new RegExp("^[0-9a-f]{40}$"); - -function getFilePointer(storageFolder: string, revision: string, relativeFilename: string): string { - const snapshotPath = join(storageFolder, "snapshots"); - return join(snapshotPath, revision, relativeFilename); -} - -/** - * handy method to check if a file exists, or the pointer of a symlinks exists - * @param path - * @param followSymlinks - */ -async function exists(path: string, followSymlinks?: boolean): Promise { - try { - if(followSymlinks) { - await stat(path); - } else { - await lstat(path); - } - return true; - } catch (err: unknown) { - return false; - } -} - -/** - * Download a given file if it's not already present in the local cache. - * @param params - * @return the symlink to the blob object - */ -export async function downloadFileToCacheDir( - params: { - repo: RepoDesignation; - path: string; - /** - * If true, will download the raw git file. - * - * For example, when calling on a file stored with Git LFS, the pointer file will be downloaded instead. - */ - raw?: boolean; - /** - * An optional Git revision id which can be a branch name, a tag, or a commit hash. - * - * @default "main" - */ - revision?: string; - hubUrl?: string; - cacheDir?: string, - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): Promise { - // get revision provided or default to main - const revision = params.revision ?? "main"; - const cacheDir = params.cacheDir ?? getHFHubCachePath(); - // get repo id - const repoId = toRepoId(params.repo); - // get storage folder - const storageFolder = join(cacheDir, getRepoFolderName(repoId)); - - let commitHash: string | undefined; - - // if user provides a commitHash as revision, and they already have the file on disk, shortcut everything. - if (REGEX_COMMIT_HASH.test(revision)) { - commitHash = revision; - const pointerPath = getFilePointer(storageFolder, revision, params.path); - if (await exists(pointerPath, true)) return pointerPath; - } - - const pathsInformation: (PathInfo & { lastCommit: CommitInfo })[] = await pathsInfo({ - ...params, - paths: [params.path], - revision: revision, - expand: true, - }); - if (!pathsInformation || pathsInformation.length !== 1) throw new Error(`cannot get path info for ${params.path}`); - - let etag: string; - if (pathsInformation[0].lfs) { - etag = pathsInformation[0].lfs.oid; // get the LFS pointed file oid - } else { - etag = pathsInformation[0].oid; // get the repo file if not a LFS pointer - } - - const pointerPath = getFilePointer(storageFolder, commitHash ?? pathsInformation[0].lastCommit.id, params.path); - const blobPath = join(storageFolder, "blobs", etag); - - // mkdir blob and pointer path parent directory - await mkdir(dirname(blobPath), { recursive: true }); - await mkdir(dirname(pointerPath), { recursive: true }); - - // We might already have the blob but not the pointer - // shortcut the download if needed - if (await exists(blobPath)) { - // create symlinks in snapshot folder to blob object - await symlink(blobPath, pointerPath); - return pointerPath; - } - - const incomplete = `${blobPath}.incomplete`; - console.debug(`Downloading ${params.path} to ${incomplete}`); - - const response: Response | null = await downloadFile({ - ...params, - revision: commitHash, - }); - - if (!response || !response.ok || !response.body) throw new Error(`invalid response for file ${params.path}`); - - // @ts-expect-error resp.body is a Stream, but Stream in internal to node - await writeFile(incomplete, response.body); - - // rename .incomplete file to expect blob - await rename(incomplete, blobPath); - // create symlinks in snapshot folder to blob object - await symlink(blobPath, pointerPath); - return pointerPath; -} diff --git a/packages/blob/src/lib/download-file.spec.ts b/packages/blob/src/lib/download-file.spec.ts deleted file mode 100644 index f442f152a..000000000 --- a/packages/blob/src/lib/download-file.spec.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { expect, test, describe, vi } from "vitest"; -import { downloadFile } from "./download-file"; -import type { RepoId } from "../types/public"; - -const DUMMY_REPO: RepoId = { - name: 'hello-world', - type: 'model', -}; - -describe("downloadFile", () => { - test("hubUrl params should overwrite HUB_URL", async () => { - const fetchMock: typeof fetch = vi.fn(); - vi.mocked(fetchMock).mockResolvedValue({ - status: 200, - ok: true, - } as Response); - - await downloadFile({ - repo: DUMMY_REPO, - path: '/README.md', - hubUrl: 'http://dummy-hub', - fetch: fetchMock, - }); - - expect(fetchMock).toHaveBeenCalledWith('http://dummy-hub/hello-world/resolve/main//README.md', expect.anything()); - }); - - test("raw params should use raw url", async () => { - const fetchMock: typeof fetch = vi.fn(); - vi.mocked(fetchMock).mockResolvedValue({ - status: 200, - ok: true, - } as Response); - - await downloadFile({ - repo: DUMMY_REPO, - path: 'README.md', - raw: true, - fetch: fetchMock, - }); - - expect(fetchMock).toHaveBeenCalledWith('https://huggingface.co/hello-world/raw/main/README.md', expect.anything()); - }); - - test("internal server error should propagate the error", async () => { - const fetchMock: typeof fetch = vi.fn(); - vi.mocked(fetchMock).mockResolvedValue({ - status: 500, - ok: false, - headers: new Map([["Content-Type", "application/json"]]), - json: () => ({ - error: 'Dummy internal error', - }), - } as unknown as Response); - - await expect(async () => { - await downloadFile({ - repo: DUMMY_REPO, - path: 'README.md', - raw: true, - fetch: fetchMock, - }); - }).rejects.toThrowError('Dummy internal error'); - }); -}); \ No newline at end of file diff --git a/packages/blob/src/lib/download-file.ts b/packages/blob/src/lib/download-file.ts deleted file mode 100644 index 4f6ebde2e..000000000 --- a/packages/blob/src/lib/download-file.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; -import type { CredentialsParams, RepoDesignation } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; -import { toRepoId } from "../utils/toRepoId"; - -/** - * @returns null when the file doesn't exist - */ -export async function downloadFile( - params: { - repo: RepoDesignation; - path: string; - /** - * If true, will download the raw git file. - * - * For example, when calling on a file stored with Git LFS, the pointer file will be downloaded instead. - */ - raw?: boolean; - /** - * An optional Git revision id which can be a branch name, a tag, or a commit hash. - * - * @default "main" - */ - revision?: string; - /** - * Fetch only a specific part of the file - */ - range?: [number, number]; - hubUrl?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): Promise { - const accessToken = checkCredentials(params); - const repoId = toRepoId(params.repo); - const url = `${params.hubUrl ?? HUB_URL}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${ - params.raw ? "raw" : "resolve" - }/${encodeURIComponent(params.revision ?? "main")}/${params.path}`; - - const resp = await (params.fetch ?? fetch)(url, { - headers: { - ...(accessToken - ? { - Authorization: `Bearer ${accessToken}`, - } - : {}), - ...(params.range - ? { - Range: `bytes=${params.range[0]}-${params.range[1]}`, - } - : {}), - }, - }); - - if (resp.status === 404 && resp.headers.get("X-Error-Code") === "EntryNotFound") { - return null; - } else if (!resp.ok) { - throw await createApiError(resp); - } - - return resp; -} diff --git a/packages/blob/src/lib/file-download-info.spec.ts b/packages/blob/src/lib/file-download-info.spec.ts deleted file mode 100644 index bea4d6b71..000000000 --- a/packages/blob/src/lib/file-download-info.spec.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { assert, it, describe } from "vitest"; -import { fileDownloadInfo } from "./file-download-info"; - -describe("fileDownloadInfo", () => { - it("should fetch LFS file info", async () => { - const info = await fileDownloadInfo({ - repo: { - name: "bert-base-uncased", - type: "model", - }, - path: "tf_model.h5", - revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", - }); - - assert.strictEqual(info?.size, 536063208); - assert.strictEqual(info?.etag, '"41a0e56472bad33498744818c8b1ef2c-64"'); - assert(info?.downloadLink); - }); - - it("should fetch raw LFS pointer info", async () => { - const info = await fileDownloadInfo({ - repo: { - name: "bert-base-uncased", - type: "model", - }, - path: "tf_model.h5", - revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", - raw: true, - }); - - assert.strictEqual(info?.size, 134); - assert.strictEqual(info?.etag, '"9eb98c817f04b051b3bcca591bcd4e03cec88018"'); - assert(!info?.downloadLink); - }); - - it("should fetch non-LFS file info", async () => { - const info = await fileDownloadInfo({ - repo: { - name: "bert-base-uncased", - type: "model", - }, - path: "tokenizer_config.json", - revision: "1a7dd4986e3dab699c24ca19b2afd0f5e1a80f37", - }); - - assert.strictEqual(info?.size, 28); - assert.strictEqual(info?.etag, '"a661b1a138dac6dc5590367402d100765010ffd6"'); - }); -}); diff --git a/packages/blob/src/lib/file-download-info.ts b/packages/blob/src/lib/file-download-info.ts deleted file mode 100644 index 210bd11e7..000000000 --- a/packages/blob/src/lib/file-download-info.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError, InvalidApiResponseFormatError } from "../error"; -import type { CredentialsParams, RepoDesignation } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; -import { toRepoId } from "../utils/toRepoId"; - -export interface FileDownloadInfoOutput { - size: number; - etag: string; - /** - * In case of LFS file, link to download directly from cloud provider - */ - downloadLink: string | null; -} -/** - * @returns null when the file doesn't exist - */ -export async function fileDownloadInfo( - params: { - repo: RepoDesignation; - path: string; - revision?: string; - hubUrl?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - /** - * To get the raw pointer file behind a LFS file - */ - raw?: boolean; - /** - * To avoid the content-disposition header in the `downloadLink` for LFS files - * - * So that on browsers you can use the URL in an iframe for example - */ - noContentDisposition?: boolean; - } & Partial -): Promise { - const accessToken = checkCredentials(params); - const repoId = toRepoId(params.repo); - - const hubUrl = params.hubUrl ?? HUB_URL; - const url = - `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${ - params.raw ? "raw" : "resolve" - }/${encodeURIComponent(params.revision ?? "main")}/${params.path}` + - (params.noContentDisposition ? "?noContentDisposition=1" : ""); - - const resp = await (params.fetch ?? fetch)(url, { - method: "GET", - headers: { - ...(params.credentials && { - Authorization: `Bearer ${accessToken}`, - }), - Range: "bytes=0-0", - }, - }); - - if (resp.status === 404 && resp.headers.get("X-Error-Code") === "EntryNotFound") { - return null; - } - - if (!resp.ok) { - throw await createApiError(resp); - } - - const etag = resp.headers.get("ETag"); - - if (!etag) { - throw new InvalidApiResponseFormatError("Expected ETag"); - } - - const contentRangeHeader = resp.headers.get("content-range"); - - if (!contentRangeHeader) { - throw new InvalidApiResponseFormatError("Expected size information"); - } - - const [, parsedSize] = contentRangeHeader.split("/"); - const size = parseInt(parsedSize); - - if (isNaN(size)) { - throw new InvalidApiResponseFormatError("Invalid file size received"); - } - - return { - etag, - size, - downloadLink: new URL(resp.url).hostname !== new URL(hubUrl).hostname ? resp.url : null, - }; -} diff --git a/packages/blob/src/lib/file-exists.spec.ts b/packages/blob/src/lib/file-exists.spec.ts deleted file mode 100644 index e20acdf3b..000000000 --- a/packages/blob/src/lib/file-exists.spec.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { assert, it, describe } from "vitest"; -import { fileExists } from "./file-exists"; - -describe("fileExists", () => { - it("should return true for file that exists", async () => { - const info = await fileExists({ - repo: { - name: "bert-base-uncased", - type: "model", - }, - path: "tf_model.h5", - revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", - }); - - assert(info, "file should exist"); - }); - - it("should return false for file that does not exist", async () => { - const info = await fileExists({ - repo: { - name: "bert-base-uncased", - type: "model", - }, - path: "tf_model.h5dadazdzazd", - revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", - }); - - assert(!info, "file should not exist"); - }); -}); diff --git a/packages/blob/src/lib/file-exists.ts b/packages/blob/src/lib/file-exists.ts deleted file mode 100644 index 64acf1dd3..000000000 --- a/packages/blob/src/lib/file-exists.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; -import type { CredentialsParams, RepoDesignation } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; -import { toRepoId } from "../utils/toRepoId"; - -export async function fileExists( - params: { - repo: RepoDesignation; - path: string; - revision?: string; - hubUrl?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): Promise { - const accessToken = checkCredentials(params); - const repoId = toRepoId(params.repo); - - const hubUrl = params.hubUrl ?? HUB_URL; - const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/raw/${encodeURIComponent( - params.revision ?? "main" - )}/${params.path}`; - - const resp = await (params.fetch ?? fetch)(url, { - method: "HEAD", - headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {}, - }); - - if (resp.status === 404) { - return false; - } - - if (!resp.ok) { - throw await createApiError(resp); - } - - return true; -} diff --git a/packages/blob/src/lib/index.ts b/packages/blob/src/lib/index.ts deleted file mode 100644 index 24e239bdc..000000000 --- a/packages/blob/src/lib/index.ts +++ /dev/null @@ -1,29 +0,0 @@ -export * from "./cache-management"; -export * from "./check-repo-access"; -export * from "./commit"; -export * from "./count-commits"; -export * from "./create-repo"; -export * from "./dataset-info"; -export * from "./delete-file"; -export * from "./delete-files"; -export * from "./delete-repo"; -export * from "./download-file"; -export * from "./download-file-to-cache-dir"; -export * from "./file-download-info"; -export * from "./file-exists"; -export * from "./list-commits"; -export * from "./list-datasets"; -export * from "./list-files"; -export * from "./list-models"; -export * from "./list-spaces"; -export * from "./model-info"; -export * from "./oauth-handle-redirect"; -export * from "./oauth-login-url"; -export * from "./parse-safetensors-metadata"; -export * from "./paths-info"; -export * from "./snapshot-download"; -export * from "./space-info"; -export * from "./upload-file"; -export * from "./upload-files"; -export * from "./upload-files-with-progress"; -export * from "./who-am-i"; diff --git a/packages/blob/src/lib/list-commits.spec.ts b/packages/blob/src/lib/list-commits.spec.ts deleted file mode 100644 index a1f4dd5e5..000000000 --- a/packages/blob/src/lib/list-commits.spec.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { assert, it, describe } from "vitest"; -import type { CommitData } from "./list-commits"; -import { listCommits } from "./list-commits"; - -describe("listCommits", () => { - it("should fetch paginated commits from the repo", async () => { - const commits: CommitData[] = []; - for await (const commit of listCommits({ - repo: { - name: "openai-community/gpt2", - type: "model", - }, - revision: "607a30d783dfa663caf39e06633721c8d4cfcd7e", - batchSize: 5, - })) { - commits.push(commit); - } - - assert.equal(commits.length, 26); - assert.deepEqual(commits.slice(0, 6), [ - { - oid: "607a30d783dfa663caf39e06633721c8d4cfcd7e", - title: "Adds the tokenizer configuration file (#80)", - message: "\n\n\n- Adds tokenizer_config.json file (db6d57930088fb63e52c010bd9ac77c955ac55e7)\n\n", - authors: [ - { - username: "lysandre", - avatarUrl: - "https://cdn-avatars.huggingface.co/v1/production/uploads/5e3aec01f55e2b62848a5217/PMKS0NNB4MJQlTSFzh918.jpeg", - }, - ], - date: new Date("2024-02-19T10:57:45.000Z"), - }, - { - oid: "11c5a3d5811f50298f278a704980280950aedb10", - title: "Adding ONNX file of this model (#60)", - message: "\n\n\n- Adding ONNX file of this model (9411f419c589519e1a46c94ac7789ea20fd7c322)\n\n", - authors: [ - { - username: "fxmarty", - avatarUrl: - "https://cdn-avatars.huggingface.co/v1/production/uploads/1651743336129-624c60cba8ec93a7ac188b56.png", - }, - ], - date: new Date("2023-06-30T02:19:43.000Z"), - }, - { - oid: "e7da7f221d5bf496a48136c0cd264e630fe9fcc8", - title: "Update generation_config.json", - message: "", - authors: [ - { - username: "joaogante", - avatarUrl: "https://cdn-avatars.huggingface.co/v1/production/uploads/1641203017724-noauth.png", - }, - ], - date: new Date("2022-12-16T15:44:21.000Z"), - }, - { - oid: "f27b190eeac4c2302d24068eabf5e9d6044389ae", - title: "Add note that this is the smallest version of the model (#18)", - message: - "\n\n\n- Add note that this is the smallest version of the model (611838ef095a5bb35bf2027d05e1194b7c9d37ac)\n\n\nCo-authored-by: helen \n", - authors: [ - { - username: "sgugger", - avatarUrl: - "https://cdn-avatars.huggingface.co/v1/production/uploads/1593126474392-5ef50182b71947201082a4e5.jpeg", - }, - { - username: "mathemakitten", - avatarUrl: - "https://cdn-avatars.huggingface.co/v1/production/uploads/1658248499901-6079afe2d2cd8c150e6ae05e.jpeg", - }, - ], - date: new Date("2022-11-23T12:55:26.000Z"), - }, - { - oid: "0dd7bcc7a64e4350d8859c9a2813132fbf6ae591", - title: "Our very first generation_config.json (#17)", - message: - "\n\n\n- Our very first generation_config.json (671851b7e9d56ef062890732065d7bd5f4628bd6)\n\n\nCo-authored-by: Joao Gante \n", - authors: [ - { - username: "sgugger", - avatarUrl: - "https://cdn-avatars.huggingface.co/v1/production/uploads/1593126474392-5ef50182b71947201082a4e5.jpeg", - }, - { - username: "joaogante", - avatarUrl: "https://cdn-avatars.huggingface.co/v1/production/uploads/1641203017724-noauth.png", - }, - ], - date: new Date("2022-11-18T18:19:30.000Z"), - }, - { - oid: "75e09b43581151bd1d9ef6700faa605df408979f", - title: "Upload model.safetensors with huggingface_hub (#12)", - message: - "\n\n\n- Upload model.safetensors with huggingface_hub (ba2f794b2e4ea09ef932a6628fa0815dfaf09661)\n\n\nCo-authored-by: Nicolas Patry \n", - authors: [ - { - username: "julien-c", - avatarUrl: - "https://cdn-avatars.huggingface.co/v1/production/uploads/5dd96eb166059660ed1ee413/NQtzmrDdbG0H8qkZvRyGk.jpeg", - }, - { - username: "Narsil", - avatarUrl: - "https://cdn-avatars.huggingface.co/v1/production/uploads/1608285816082-5e2967b819407e3277369b95.png", - }, - ], - date: new Date("2022-10-20T09:34:54.000Z"), - }, - ]); - }); -}); diff --git a/packages/blob/src/lib/list-commits.ts b/packages/blob/src/lib/list-commits.ts deleted file mode 100644 index 2bbcc99f7..000000000 --- a/packages/blob/src/lib/list-commits.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; -import type { ApiCommitData } from "../types/api/api-commit"; -import type { CredentialsParams, RepoDesignation } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; -import { parseLinkHeader } from "../utils/parseLinkHeader"; -import { toRepoId } from "../utils/toRepoId"; - -export interface CommitData { - oid: string; - title: string; - message: string; - authors: Array<{ username: string; avatarUrl: string }>; - date: Date; -} - -export async function* listCommits( - params: { - repo: RepoDesignation; - /** - * Revision to list commits from. Defaults to the default branch. - */ - revision?: string; - hubUrl?: string; - /** - * Number of commits to fetch from the hub each http call. Defaults to 100. Can be set to 1000. - */ - batchSize?: number; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): AsyncGenerator { - const accessToken = checkCredentials(params); - const repoId = toRepoId(params.repo); - - // Could upgrade to 1000 commits per page - let url: string | undefined = `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commits/${ - params.revision ?? "main" - }?limit=${params.batchSize ?? 100}`; - - while (url) { - const res: Response = await (params.fetch ?? fetch)(url, { - headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {}, - }); - - if (!res.ok) { - throw await createApiError(res); - } - - const resJson: ApiCommitData[] = await res.json(); - for (const commit of resJson) { - yield { - oid: commit.id, - title: commit.title, - message: commit.message, - authors: commit.authors.map((author) => ({ - username: author.user, - avatarUrl: author.avatar, - })), - date: new Date(commit.date), - }; - } - - const linkHeader = res.headers.get("Link"); - - url = linkHeader ? parseLinkHeader(linkHeader).next : undefined; - } -} diff --git a/packages/blob/src/lib/list-datasets.spec.ts b/packages/blob/src/lib/list-datasets.spec.ts deleted file mode 100644 index 24993130b..000000000 --- a/packages/blob/src/lib/list-datasets.spec.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { DatasetEntry } from "./list-datasets"; -import { listDatasets } from "./list-datasets"; - -describe("listDatasets", () => { - it("should list datasets from hf-doc-builder", async () => { - const results: DatasetEntry[] = []; - - for await (const entry of listDatasets({ search: { owner: "hf-doc-build" } })) { - if (entry.name === "hf-doc-build/doc-build-dev-test") { - continue; - } - if (typeof entry.downloads === "number") { - entry.downloads = 0; - } - if (typeof entry.likes === "number") { - entry.likes = 0; - } - if (entry.updatedAt instanceof Date && !isNaN(entry.updatedAt.getTime())) { - entry.updatedAt = new Date(0); - } - - results.push(entry); - } - - expect(results).deep.equal([ - { - id: "6356b19985da6f13863228bd", - name: "hf-doc-build/doc-build", - private: false, - gated: false, - downloads: 0, - likes: 0, - updatedAt: new Date(0), - }, - { - id: "636a1b69f2f9ec4289c4c19e", - name: "hf-doc-build/doc-build-dev", - gated: false, - private: false, - downloads: 0, - likes: 0, - updatedAt: new Date(0), - }, - ]); - }); -}); diff --git a/packages/blob/src/lib/list-datasets.ts b/packages/blob/src/lib/list-datasets.ts deleted file mode 100644 index 5bfbc4a15..000000000 --- a/packages/blob/src/lib/list-datasets.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; -import type { ApiDatasetInfo } from "../types/api/api-dataset"; -import type { CredentialsParams } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; -import { parseLinkHeader } from "../utils/parseLinkHeader"; -import { pick } from "../utils/pick"; - -export const DATASET_EXPAND_KEYS = [ - "private", - "downloads", - "gated", - "likes", - "lastModified", -] as const satisfies readonly (keyof ApiDatasetInfo)[]; - -export const DATASET_EXPANDABLE_KEYS = [ - "author", - "cardData", - "citation", - "createdAt", - "disabled", - "description", - "downloads", - "downloadsAllTime", - "gated", - "gitalyUid", - "lastModified", - "likes", - "paperswithcode_id", - "private", - // "siblings", - "sha", - "tags", -] as const satisfies readonly (keyof ApiDatasetInfo)[]; - -export interface DatasetEntry { - id: string; - name: string; - private: boolean; - downloads: number; - gated: false | "auto" | "manual"; - likes: number; - updatedAt: Date; -} - -export async function* listDatasets< - const T extends Exclude<(typeof DATASET_EXPANDABLE_KEYS)[number], (typeof DATASET_EXPAND_KEYS)[number]> = never, ->( - params?: { - search?: { - /** - * Will search in the dataset name for matches - */ - query?: string; - owner?: string; - tags?: string[]; - }; - hubUrl?: string; - additionalFields?: T[]; - /** - * Set to limit the number of models returned. - */ - limit?: number; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): AsyncGenerator> { - const accessToken = params && checkCredentials(params); - let totalToFetch = params?.limit ?? Infinity; - const search = new URLSearchParams([ - ...Object.entries({ - limit: String(Math.min(totalToFetch, 500)), - ...(params?.search?.owner ? { author: params.search.owner } : undefined), - ...(params?.search?.query ? { search: params.search.query } : undefined), - }), - ...(params?.search?.tags?.map((tag) => ["filter", tag]) ?? []), - ...DATASET_EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), - ...(params?.additionalFields?.map((val) => ["expand", val] satisfies [string, string]) ?? []), - ]).toString(); - let url: string | undefined = `${params?.hubUrl || HUB_URL}/api/datasets` + (search ? "?" + search : ""); - - while (url) { - const res: Response = await (params?.fetch ?? fetch)(url, { - headers: { - accept: "application/json", - ...(params?.credentials ? { Authorization: `Bearer ${accessToken}` } : undefined), - }, - }); - - if (!res.ok) { - throw await createApiError(res); - } - - const items: ApiDatasetInfo[] = await res.json(); - - for (const item of items) { - yield { - ...(params?.additionalFields && pick(item, params.additionalFields)), - id: item._id, - name: item.id, - private: item.private, - downloads: item.downloads, - likes: item.likes, - gated: item.gated, - updatedAt: new Date(item.lastModified), - } as DatasetEntry & Pick; - totalToFetch--; - if (totalToFetch <= 0) { - return; - } - } - - const linkHeader = res.headers.get("Link"); - - url = linkHeader ? parseLinkHeader(linkHeader).next : undefined; - // Could update limit in url to fetch less items if not all items of next page are needed. - } -} diff --git a/packages/blob/src/lib/list-files.spec.ts b/packages/blob/src/lib/list-files.spec.ts deleted file mode 100644 index ee59a78a0..000000000 --- a/packages/blob/src/lib/list-files.spec.ts +++ /dev/null @@ -1,169 +0,0 @@ -import { assert, it, describe } from "vitest"; -import type { ListFileEntry } from "./list-files"; -import { listFiles } from "./list-files"; - -describe("listFiles", () => { - it("should fetch the list of files from the repo", async () => { - const cursor = listFiles({ - repo: { - name: "bert-base-uncased", - type: "model", - }, - revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", - }); - - const files: ListFileEntry[] = []; - - for await (const entry of cursor) { - files.push(entry); - } - - assert.deepStrictEqual(files, [ - { - oid: "dc08351d4dc0732d9c8af04070ced089b201ce2f", - path: ".gitattributes", - size: 345, - type: "file", - }, - { - oid: "fca794a5f07ff8f963fe8b61e3694b0fb7f955df", - path: "config.json", - size: 313, - type: "file", - }, - { - lfs: { - oid: "097417381d6c7230bd9e3557456d726de6e83245ec8b24f529f60198a67b203a", - size: 440473133, - pointerSize: 134, - }, - oid: "ba5d19791be1dd7992e33bd61f20207b0f7f50a5", - path: "pytorch_model.bin", - size: 440473133, - type: "file", - }, - { - lfs: { - oid: "a7a17d6d844b5de815ccab5f42cad6d24496db3850a2a43d8258221018ce87d2", - size: 536063208, - pointerSize: 134, - }, - oid: "9eb98c817f04b051b3bcca591bcd4e03cec88018", - path: "tf_model.h5", - size: 536063208, - type: "file", - }, - { - oid: "fb140275c155a9c7c5a3b3e0e77a9e839594a938", - path: "vocab.txt", - size: 231508, - type: "file", - }, - ]); - }); - - it("should fetch the list of files from the repo, including last commit", async () => { - const cursor = listFiles({ - repo: { - name: "bert-base-uncased", - type: "model", - }, - revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", - expand: true, - }); - - const files: ListFileEntry[] = []; - - for await (const entry of cursor) { - delete entry.securityFileStatus; // flaky - files.push(entry); - } - - assert.deepStrictEqual(files, [ - { - lastCommit: { - date: "2018-11-14T23:35:08.000Z", - id: "504939aa53e8ce310dba3dd2296dbe266c575de4", - title: "initial commit", - }, - oid: "dc08351d4dc0732d9c8af04070ced089b201ce2f", - path: ".gitattributes", - size: 345, - type: "file", - }, - { - lastCommit: { - date: "2019-06-18T09:06:51.000Z", - id: "bb3c1c3256d2598217df9889a14a2e811587891d", - title: "Update config.json", - }, - oid: "fca794a5f07ff8f963fe8b61e3694b0fb7f955df", - path: "config.json", - size: 313, - type: "file", - }, - { - lastCommit: { - date: "2019-06-18T09:06:34.000Z", - id: "3d2477d72b675a999d1b13ca822aaaf4908634ad", - title: "Update pytorch_model.bin", - }, - lfs: { - oid: "097417381d6c7230bd9e3557456d726de6e83245ec8b24f529f60198a67b203a", - size: 440473133, - pointerSize: 134, - }, - oid: "ba5d19791be1dd7992e33bd61f20207b0f7f50a5", - path: "pytorch_model.bin", - size: 440473133, - type: "file", - }, - { - lastCommit: { - date: "2019-09-23T19:48:44.000Z", - id: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", - title: "Update tf_model.h5", - }, - lfs: { - oid: "a7a17d6d844b5de815ccab5f42cad6d24496db3850a2a43d8258221018ce87d2", - size: 536063208, - pointerSize: 134, - }, - oid: "9eb98c817f04b051b3bcca591bcd4e03cec88018", - path: "tf_model.h5", - size: 536063208, - type: "file", - }, - { - lastCommit: { - date: "2018-11-14T23:35:08.000Z", - id: "2f07d813ca87c8c709147704c87210359ccf2309", - title: "Update vocab.txt", - }, - oid: "fb140275c155a9c7c5a3b3e0e77a9e839594a938", - path: "vocab.txt", - size: 231508, - type: "file", - }, - ]); - }); - - it("should fetch the list of files from the repo, including subfolders", async () => { - const cursor = listFiles({ - repo: { - name: "xsum", - type: "dataset", - }, - revision: "0f3ea2f2b55fcb11e71fb1e3aec6822e44ddcb0f", - recursive: true, - }); - - const files: ListFileEntry[] = []; - - for await (const entry of cursor) { - files.push(entry); - } - - assert(files.some((file) => file.path === "data/XSUM-EMNLP18-Summary-Data-Original.tar.gz")); - }); -}); diff --git a/packages/blob/src/lib/list-files.ts b/packages/blob/src/lib/list-files.ts deleted file mode 100644 index 1648bfe60..000000000 --- a/packages/blob/src/lib/list-files.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; -import type { ApiIndexTreeEntry } from "../types/api/api-index-tree"; -import type { CredentialsParams, RepoDesignation } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; -import { parseLinkHeader } from "../utils/parseLinkHeader"; -import { toRepoId } from "../utils/toRepoId"; - -export interface ListFileEntry { - type: "file" | "directory" | "unknown"; - size: number; - path: string; - oid: string; - lfs?: { - oid: string; - size: number; - /** Size of the raw pointer file, 100~200 bytes */ - pointerSize: number; - }; - /** - * Only fetched if `expand` is set to `true` in the `listFiles` call. - */ - lastCommit?: { - date: string; - id: string; - title: string; - }; - /** - * Only fetched if `expand` is set to `true` in the `listFiles` call. - */ - securityFileStatus?: unknown; -} - -/** - * List files in a folder. To list ALL files in the directory, call it - * with {@link params.recursive} set to `true`. - */ -export async function* listFiles( - params: { - repo: RepoDesignation; - /** - * Do we want to list files in subdirectories? - */ - recursive?: boolean; - /** - * Eg 'data' for listing all files in the 'data' folder. Leave it empty to list all - * files in the repo. - */ - path?: string; - /** - * Fetch `lastCommit` and `securityFileStatus` for each file. - */ - expand?: boolean; - revision?: string; - hubUrl?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): AsyncGenerator { - const accessToken = checkCredentials(params); - const repoId = toRepoId(params.repo); - let url: string | undefined = `${params.hubUrl || HUB_URL}/api/${repoId.type}s/${repoId.name}/tree/${ - params.revision || "main" - }${params.path ? "/" + params.path : ""}?recursive=${!!params.recursive}&expand=${!!params.expand}`; - - while (url) { - const res: Response = await (params.fetch ?? fetch)(url, { - headers: { - accept: "application/json", - ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : undefined), - }, - }); - - if (!res.ok) { - throw await createApiError(res); - } - - const items: ApiIndexTreeEntry[] = await res.json(); - - for (const item of items) { - yield item; - } - - const linkHeader = res.headers.get("Link"); - - url = linkHeader ? parseLinkHeader(linkHeader).next : undefined; - } -} diff --git a/packages/blob/src/lib/list-models.spec.ts b/packages/blob/src/lib/list-models.spec.ts deleted file mode 100644 index e3da993c6..000000000 --- a/packages/blob/src/lib/list-models.spec.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { ModelEntry } from "./list-models"; -import { listModels } from "./list-models"; - -describe("listModels", () => { - it("should list models for depth estimation", async () => { - const results: ModelEntry[] = []; - - for await (const entry of listModels({ - search: { owner: "Intel", task: "depth-estimation" }, - })) { - if (typeof entry.downloads === "number") { - entry.downloads = 0; - } - if (typeof entry.likes === "number") { - entry.likes = 0; - } - if (entry.updatedAt instanceof Date && !isNaN(entry.updatedAt.getTime())) { - entry.updatedAt = new Date(0); - } - - if (!["Intel/dpt-large", "Intel/dpt-hybrid-midas"].includes(entry.name)) { - expect(entry.task).to.equal("depth-estimation"); - continue; - } - - results.push(entry); - } - - results.sort((a, b) => a.id.localeCompare(b.id)); - - expect(results).deep.equal([ - { - id: "621ffdc136468d709f17e709", - name: "Intel/dpt-large", - private: false, - gated: false, - downloads: 0, - likes: 0, - task: "depth-estimation", - updatedAt: new Date(0), - }, - { - id: "638f07977559bf9a2b2b04ac", - name: "Intel/dpt-hybrid-midas", - gated: false, - private: false, - downloads: 0, - likes: 0, - task: "depth-estimation", - updatedAt: new Date(0), - }, - ]); - }); - - it("should list indonesian models with gguf format", async () => { - let count = 0; - for await (const entry of listModels({ - search: { tags: ["gguf", "id"] }, - additionalFields: ["tags"], - limit: 2, - })) { - count++; - expect(entry.tags).to.include("gguf"); - expect(entry.tags).to.include("id"); - } - - expect(count).to.equal(2); - }); - - it("should search model by name", async () => { - let count = 0; - for await (const entry of listModels({ - search: { query: "t5" }, - limit: 10, - })) { - count++; - expect(entry.name.toLocaleLowerCase()).to.include("t5"); - } - - expect(count).to.equal(10); - }); -}); diff --git a/packages/blob/src/lib/list-models.ts b/packages/blob/src/lib/list-models.ts deleted file mode 100644 index 0e62c8caf..000000000 --- a/packages/blob/src/lib/list-models.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; -import type { ApiModelInfo } from "../types/api/api-model"; -import type { CredentialsParams, PipelineType } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; -import { parseLinkHeader } from "../utils/parseLinkHeader"; -import { pick } from "../utils/pick"; - -export const MODEL_EXPAND_KEYS = [ - "pipeline_tag", - "private", - "gated", - "downloads", - "likes", - "lastModified", -] as const satisfies readonly (keyof ApiModelInfo)[]; - -export const MODEL_EXPANDABLE_KEYS = [ - "author", - "cardData", - "config", - "createdAt", - "disabled", - "downloads", - "downloadsAllTime", - "gated", - "gitalyUid", - "lastModified", - "library_name", - "likes", - "model-index", - "pipeline_tag", - "private", - "safetensors", - "sha", - // "siblings", - "spaces", - "tags", - "transformersInfo", -] as const satisfies readonly (keyof ApiModelInfo)[]; - -export interface ModelEntry { - id: string; - name: string; - private: boolean; - gated: false | "auto" | "manual"; - task?: PipelineType; - likes: number; - downloads: number; - updatedAt: Date; -} - -export async function* listModels< - const T extends Exclude<(typeof MODEL_EXPANDABLE_KEYS)[number], (typeof MODEL_EXPAND_KEYS)[number]> = never, ->( - params?: { - search?: { - /** - * Will search in the model name for matches - */ - query?: string; - owner?: string; - task?: PipelineType; - tags?: string[]; - }; - hubUrl?: string; - additionalFields?: T[]; - /** - * Set to limit the number of models returned. - */ - limit?: number; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): AsyncGenerator> { - const accessToken = params && checkCredentials(params); - let totalToFetch = params?.limit ?? Infinity; - const search = new URLSearchParams([ - ...Object.entries({ - limit: String(Math.min(totalToFetch, 500)), - ...(params?.search?.owner ? { author: params.search.owner } : undefined), - ...(params?.search?.task ? { pipeline_tag: params.search.task } : undefined), - ...(params?.search?.query ? { search: params.search.query } : undefined), - }), - ...(params?.search?.tags?.map((tag) => ["filter", tag]) ?? []), - ...MODEL_EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), - ...(params?.additionalFields?.map((val) => ["expand", val] satisfies [string, string]) ?? []), - ]).toString(); - let url: string | undefined = `${params?.hubUrl || HUB_URL}/api/models?${search}`; - - while (url) { - const res: Response = await (params?.fetch ?? fetch)(url, { - headers: { - accept: "application/json", - ...(params?.credentials ? { Authorization: `Bearer ${accessToken}` } : undefined), - }, - }); - - if (!res.ok) { - throw await createApiError(res); - } - - const items: ApiModelInfo[] = await res.json(); - - for (const item of items) { - yield { - ...(params?.additionalFields && pick(item, params.additionalFields)), - id: item._id, - name: item.id, - private: item.private, - task: item.pipeline_tag, - downloads: item.downloads, - gated: item.gated, - likes: item.likes, - updatedAt: new Date(item.lastModified), - } as ModelEntry & Pick; - totalToFetch--; - - if (totalToFetch <= 0) { - return; - } - } - - const linkHeader = res.headers.get("Link"); - - url = linkHeader ? parseLinkHeader(linkHeader).next : undefined; - // Could update url to reduce the limit if we don't need the whole 500 of the next batch. - } -} diff --git a/packages/blob/src/lib/list-spaces.spec.ts b/packages/blob/src/lib/list-spaces.spec.ts deleted file mode 100644 index 3cc599913..000000000 --- a/packages/blob/src/lib/list-spaces.spec.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { SpaceEntry } from "./list-spaces"; -import { listSpaces } from "./list-spaces"; - -describe("listSpaces", () => { - it("should list spaces for Microsoft", async () => { - const results: SpaceEntry[] = []; - - for await (const entry of listSpaces({ - search: { owner: "microsoft" }, - additionalFields: ["subdomain"], - })) { - if (entry.name !== "microsoft/visual_chatgpt") { - continue; - } - if (typeof entry.likes === "number") { - entry.likes = 0; - } - if (entry.updatedAt instanceof Date && !isNaN(entry.updatedAt.getTime())) { - entry.updatedAt = new Date(0); - } - - results.push(entry); - } - - results.sort((a, b) => a.id.localeCompare(b.id)); - - expect(results).deep.equal([ - { - id: "6409a392bbc73d022c58c980", - name: "microsoft/visual_chatgpt", - private: false, - likes: 0, - sdk: "gradio", - subdomain: "microsoft-visual-chatgpt", - updatedAt: new Date(0), - }, - ]); - }); -}); diff --git a/packages/blob/src/lib/list-spaces.ts b/packages/blob/src/lib/list-spaces.ts deleted file mode 100644 index 36a251afd..000000000 --- a/packages/blob/src/lib/list-spaces.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; -import type { ApiSpaceInfo } from "../types/api/api-space"; -import type { CredentialsParams, SpaceSdk } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; -import { parseLinkHeader } from "../utils/parseLinkHeader"; -import { pick } from "../utils/pick"; - -export const SPACE_EXPAND_KEYS = [ - "sdk", - "likes", - "private", - "lastModified", -] as const satisfies readonly (keyof ApiSpaceInfo)[]; -export const SPACE_EXPANDABLE_KEYS = [ - "author", - "cardData", - "datasets", - "disabled", - "gitalyUid", - "lastModified", - "createdAt", - "likes", - "private", - "runtime", - "sdk", - // "siblings", - "sha", - "subdomain", - "tags", - "models", -] as const satisfies readonly (keyof ApiSpaceInfo)[]; - -export interface SpaceEntry { - id: string; - name: string; - sdk?: SpaceSdk; - likes: number; - private: boolean; - updatedAt: Date; - // Use additionalFields to fetch the fields from ApiSpaceInfo -} - -export async function* listSpaces< - const T extends Exclude<(typeof SPACE_EXPANDABLE_KEYS)[number], (typeof SPACE_EXPAND_KEYS)[number]> = never, ->( - params?: { - search?: { - /** - * Will search in the space name for matches - */ - query?: string; - owner?: string; - tags?: string[]; - }; - hubUrl?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - /** - * Additional fields to fetch from huggingface.co. - */ - additionalFields?: T[]; - } & Partial -): AsyncGenerator> { - const accessToken = params && checkCredentials(params); - const search = new URLSearchParams([ - ...Object.entries({ - limit: "500", - ...(params?.search?.owner ? { author: params.search.owner } : undefined), - ...(params?.search?.query ? { search: params.search.query } : undefined), - }), - ...(params?.search?.tags?.map((tag) => ["filter", tag]) ?? []), - ...[...SPACE_EXPAND_KEYS, ...(params?.additionalFields ?? [])].map( - (val) => ["expand", val] satisfies [string, string] - ), - ]).toString(); - let url: string | undefined = `${params?.hubUrl || HUB_URL}/api/spaces?${search}`; - - while (url) { - const res: Response = await (params?.fetch ?? fetch)(url, { - headers: { - accept: "application/json", - ...(params?.credentials ? { Authorization: `Bearer ${accessToken}` } : undefined), - }, - }); - - if (!res.ok) { - throw await createApiError(res); - } - - const items: ApiSpaceInfo[] = await res.json(); - - for (const item of items) { - yield { - ...(params?.additionalFields && pick(item, params.additionalFields)), - id: item._id, - name: item.id, - sdk: item.sdk, - likes: item.likes, - private: item.private, - updatedAt: new Date(item.lastModified), - } as SpaceEntry & Pick; - } - - const linkHeader = res.headers.get("Link"); - - url = linkHeader ? parseLinkHeader(linkHeader).next : undefined; - } -} diff --git a/packages/blob/src/lib/model-info.spec.ts b/packages/blob/src/lib/model-info.spec.ts deleted file mode 100644 index 3886f3039..000000000 --- a/packages/blob/src/lib/model-info.spec.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { modelInfo } from "./model-info"; -import type { ModelEntry } from "./list-models"; -import type { ApiModelInfo } from "../types/api/api-model"; - -describe("modelInfo", () => { - it("should return the model info", async () => { - const info = await modelInfo({ - name: "openai-community/gpt2", - }); - expect(info).toEqual({ - id: "621ffdc036468d709f17434d", - downloads: expect.any(Number), - gated: false, - name: "openai-community/gpt2", - updatedAt: expect.any(Date), - likes: expect.any(Number), - task: "text-generation", - private: false, - }); - }); - - it("should return the model info with author", async () => { - const info: ModelEntry & Pick = await modelInfo({ - name: "openai-community/gpt2", - additionalFields: ["author"], - }); - expect(info).toEqual({ - id: "621ffdc036468d709f17434d", - downloads: expect.any(Number), - author: "openai-community", - gated: false, - name: "openai-community/gpt2", - updatedAt: expect.any(Date), - likes: expect.any(Number), - task: "text-generation", - private: false, - }); - }); - - it("should return the model info for a specific revision", async () => { - const info: ModelEntry & Pick = await modelInfo({ - name: "openai-community/gpt2", - additionalFields: ["sha"], - revision: 'f27b190eeac4c2302d24068eabf5e9d6044389ae', - }); - expect(info).toEqual({ - id: "621ffdc036468d709f17434d", - downloads: expect.any(Number), - gated: false, - name: "openai-community/gpt2", - updatedAt: expect.any(Date), - likes: expect.any(Number), - task: "text-generation", - private: false, - sha: "f27b190eeac4c2302d24068eabf5e9d6044389ae", - }); - }); -}); diff --git a/packages/blob/src/lib/model-info.ts b/packages/blob/src/lib/model-info.ts deleted file mode 100644 index 828322e67..000000000 --- a/packages/blob/src/lib/model-info.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; -import type { ApiModelInfo } from "../types/api/api-model"; -import type { CredentialsParams } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; -import { pick } from "../utils/pick"; -import { MODEL_EXPAND_KEYS, type MODEL_EXPANDABLE_KEYS, type ModelEntry } from "./list-models"; - -export async function modelInfo< - const T extends Exclude<(typeof MODEL_EXPANDABLE_KEYS)[number], (typeof MODEL_EXPAND_KEYS)[number]> = never, ->( - params: { - name: string; - hubUrl?: string; - additionalFields?: T[]; - /** - * An optional Git revision id which can be a branch name, a tag, or a commit hash. - */ - revision?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): Promise> { - const accessToken = params && checkCredentials(params); - - const search = new URLSearchParams([ - ...MODEL_EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), - ...(params?.additionalFields?.map((val) => ["expand", val] satisfies [string, string]) ?? []), - ]).toString(); - - const response = await (params.fetch || fetch)( - `${params?.hubUrl || HUB_URL}/api/models/${params.name}/revision/${encodeURIComponent(params.revision ?? "HEAD")}?${search.toString()}`, - { - headers: { - ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), - Accepts: "application/json", - }, - } - ); - - if (!response.ok) { - throw await createApiError(response); - } - - const data = await response.json(); - - return { - ...(params?.additionalFields && pick(data, params.additionalFields)), - id: data._id, - name: data.id, - private: data.private, - task: data.pipeline_tag, - downloads: data.downloads, - gated: data.gated, - likes: data.likes, - updatedAt: new Date(data.lastModified), - } as ModelEntry & Pick; -} diff --git a/packages/blob/src/lib/oauth-handle-redirect.ts b/packages/blob/src/lib/oauth-handle-redirect.ts deleted file mode 100644 index 771b6d439..000000000 --- a/packages/blob/src/lib/oauth-handle-redirect.ts +++ /dev/null @@ -1,233 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; - -export interface OAuthResult { - accessToken: string; - accessTokenExpiresAt: Date; - userInfo: { - id: string; - name: string; - fullname: string; - email?: string; - emailVerified?: boolean; - avatarUrl: string; - websiteUrl?: string; - isPro: boolean; - canPay?: boolean; - orgs: Array<{ - id: string; - name: string; - isEnterprise: boolean; - canPay?: boolean; - avatarUrl: string; - roleInOrg?: string; - }>; - }; - /** - * State passed to the OAuth provider in the original request to the OAuth provider. - */ - state?: string; - /** - * Granted scope - */ - scope: string; -} - -/** - * To call after the OAuth provider redirects back to the app. - * - * There is also a helper function {@link oauthHandleRedirectIfPresent}, which will call `oauthHandleRedirect` if the URL contains an oauth code - * in the query parameters and return `false` otherwise. - */ -export async function oauthHandleRedirect(opts?: { hubUrl?: string }): Promise { - if (typeof window === "undefined") { - throw new Error("oauthHandleRedirect is only available in the browser"); - } - - const searchParams = new URLSearchParams(window.location.search); - - const [error, errorDescription] = [searchParams.get("error"), searchParams.get("error_description")]; - - if (error) { - throw new Error(`${error}: ${errorDescription}`); - } - - const code = searchParams.get("code"); - const nonce = localStorage.getItem("huggingface.co:oauth:nonce"); - - if (!code) { - throw new Error("Missing oauth code from query parameters in redirected URL"); - } - - if (!nonce) { - throw new Error("Missing oauth nonce from localStorage"); - } - - const codeVerifier = localStorage.getItem("huggingface.co:oauth:code_verifier"); - - if (!codeVerifier) { - throw new Error("Missing oauth code_verifier from localStorage"); - } - - const state = searchParams.get("state"); - - if (!state) { - throw new Error("Missing oauth state from query parameters in redirected URL"); - } - - let parsedState: { nonce: string; redirectUri: string; state?: string }; - - try { - parsedState = JSON.parse(state); - } catch { - throw new Error("Invalid oauth state in redirected URL, unable to parse JSON: " + state); - } - - if (parsedState.nonce !== nonce) { - throw new Error("Invalid oauth state in redirected URL"); - } - - const hubUrl = opts?.hubUrl || HUB_URL; - - const openidConfigUrl = `${new URL(hubUrl).origin}/.well-known/openid-configuration`; - const openidConfigRes = await fetch(openidConfigUrl, { - headers: { - Accept: "application/json", - }, - }); - - if (!openidConfigRes.ok) { - throw await createApiError(openidConfigRes); - } - - const opendidConfig: { - authorization_endpoint: string; - token_endpoint: string; - userinfo_endpoint: string; - } = await openidConfigRes.json(); - - const tokenRes = await fetch(opendidConfig.token_endpoint, { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - }, - body: new URLSearchParams({ - grant_type: "authorization_code", - code, - redirect_uri: parsedState.redirectUri, - code_verifier: codeVerifier, - }).toString(), - }); - - localStorage.removeItem("huggingface.co:oauth:code_verifier"); - localStorage.removeItem("huggingface.co:oauth:nonce"); - - if (!tokenRes.ok) { - throw await createApiError(tokenRes); - } - - const token: { - access_token: string; - expires_in: number; - id_token: string; - // refresh_token: string; - scope: string; - token_type: string; - } = await tokenRes.json(); - - const accessTokenExpiresAt = new Date(Date.now() + token.expires_in * 1000); - - const userInfoRes = await fetch(opendidConfig.userinfo_endpoint, { - headers: { - Authorization: `Bearer ${token.access_token}`, - }, - }); - - if (!userInfoRes.ok) { - throw await createApiError(userInfoRes); - } - - const userInfo: { - sub: string; - name: string; - preferred_username: string; - email_verified?: boolean; - email?: string; - picture: string; - website?: string; - isPro: boolean; - canPay?: boolean; - orgs?: Array<{ - sub: string; - name: string; - picture: string; - isEnterprise: boolean; - canPay?: boolean; - roleInOrg?: string; - }>; - } = await userInfoRes.json(); - - return { - accessToken: token.access_token, - accessTokenExpiresAt, - userInfo: { - id: userInfo.sub, - name: userInfo.name, - fullname: userInfo.preferred_username, - email: userInfo.email, - emailVerified: userInfo.email_verified, - avatarUrl: userInfo.picture, - websiteUrl: userInfo.website, - isPro: userInfo.isPro, - orgs: - userInfo.orgs?.map((org) => ({ - id: org.sub, - name: org.name, - fullname: org.name, - isEnterprise: org.isEnterprise, - canPay: org.canPay, - avatarUrl: org.picture, - roleInOrg: org.roleInOrg, - })) ?? [], - }, - state: parsedState.state, - scope: token.scope, - }; -} - -// if (code && !nonce) { -// console.warn("Missing oauth nonce from localStorage"); -// } - -/** - * To call after the OAuth provider redirects back to the app. - * - * It returns false if the URL does not contain an oauth code in the query parameters, otherwise - * it calls {@link oauthHandleRedirect}. - * - * Depending on your app, you may want to call {@link oauthHandleRedirect} directly instead. - */ -export async function oauthHandleRedirectIfPresent(opts?: { hubUrl?: string }): Promise { - if (typeof window === "undefined") { - throw new Error("oauthHandleRedirect is only available in the browser"); - } - - const searchParams = new URLSearchParams(window.location.search); - - if (searchParams.has("error")) { - return oauthHandleRedirect(opts); - } - - if (searchParams.has("code")) { - if (!localStorage.getItem("huggingface.co:oauth:nonce")) { - console.warn( - "Missing oauth nonce from localStorage. This can happen when the user refreshes the page after logging in, without changing the URL." - ); - return false; - } - - return oauthHandleRedirect(opts); - } - - return false; -} diff --git a/packages/blob/src/lib/oauth-login-url.ts b/packages/blob/src/lib/oauth-login-url.ts deleted file mode 100644 index 9067ba994..000000000 --- a/packages/blob/src/lib/oauth-login-url.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; -import { base64FromBytes } from "../utils/base64FromBytes"; - -/** - * Use "Sign in with Hub" to authenticate a user, and get oauth user info / access token. - * - * Returns an url to redirect to. After the user is redirected back to your app, call `oauthHandleRedirect` to get the oauth user info / access token. - * - * When called from inside a static Space with OAuth enabled, it will load the config from the space, otherwise you need to at least specify - * the client ID of your OAuth App. - * - * @example - * ```ts - * import { oauthLoginUrl, oauthHandleRedirectIfPresent } from "@huggingface/hub"; - * - * const oauthResult = await oauthHandleRedirectIfPresent(); - * - * if (!oauthResult) { - * // If the user is not logged in, redirect to the login page - * window.location.href = await oauthLoginUrl(); - * } - * - * // You can use oauthResult.accessToken, oauthResult.accessTokenExpiresAt and oauthResult.userInfo - * console.log(oauthResult); - * ``` - * - * (Theoretically, this function could be used to authenticate a user for any OAuth provider supporting PKCE and OpenID Connect by changing `hubUrl`, - * but it is currently only tested with the Hugging Face Hub.) - */ -export async function oauthLoginUrl(opts?: { - /** - * OAuth client ID. - * - * For static Spaces, you can omit this and it will be loaded from the Space config, as long as `hf_oauth: true` is present in the README.md's metadata. - * For other Spaces, it is available to the backend in the OAUTH_CLIENT_ID environment variable, as long as `hf_oauth: true` is present in the README.md's metadata. - * - * You can also create a Developer Application at https://huggingface.co/settings/connected-applications and use its client ID. - */ - clientId?: string; - hubUrl?: string; - /** - * OAuth scope, a list of space separate scopes. - * - * For static Spaces, you can omit this and it will be loaded from the Space config, as long as `hf_oauth: true` is present in the README.md's metadata. - * For other Spaces, it is available to the backend in the OAUTH_SCOPES environment variable, as long as `hf_oauth: true` is present in the README.md's metadata. - * - * Defaults to "openid profile". - * - * You can also create a Developer Application at https://huggingface.co/settings/connected-applications and use its scopes. - * - * See https://huggingface.co/docs/hub/oauth for a list of available scopes. - */ - scopes?: string; - /** - * Redirect URI, defaults to the current URL. - * - * For Spaces, any URL within the Space is allowed. - * - * For Developer Applications, you can add any URL you want to the list of allowed redirect URIs at https://huggingface.co/settings/connected-applications. - */ - redirectUrl?: string; - /** - * State to pass to the OAuth provider, which will be returned in the call to `oauthLogin` after the redirect. - */ - state?: string; -}): Promise { - if (typeof window === "undefined") { - throw new Error("oauthLogin is only available in the browser"); - } - - const hubUrl = opts?.hubUrl || HUB_URL; - const openidConfigUrl = `${new URL(hubUrl).origin}/.well-known/openid-configuration`; - const openidConfigRes = await fetch(openidConfigUrl, { - headers: { - Accept: "application/json", - }, - }); - - if (!openidConfigRes.ok) { - throw await createApiError(openidConfigRes); - } - - const opendidConfig: { - authorization_endpoint: string; - token_endpoint: string; - userinfo_endpoint: string; - } = await openidConfigRes.json(); - - const newNonce = globalThis.crypto.randomUUID(); - // Two random UUIDs concatenated together, because min length is 43 and max length is 128 - const newCodeVerifier = globalThis.crypto.randomUUID() + globalThis.crypto.randomUUID(); - - localStorage.setItem("huggingface.co:oauth:nonce", newNonce); - localStorage.setItem("huggingface.co:oauth:code_verifier", newCodeVerifier); - - const redirectUri = opts?.redirectUrl || window.location.href; - const state = JSON.stringify({ - nonce: newNonce, - redirectUri, - state: opts?.state, - }); - - // @ts-expect-error window.huggingface is defined inside static Spaces. - const variables: Record | null = window?.huggingface?.variables ?? null; - - const clientId = opts?.clientId || variables?.OAUTH_CLIENT_ID; - - if (!clientId) { - if (variables) { - throw new Error("Missing clientId, please add hf_oauth: true to the README.md's metadata in your static Space"); - } - throw new Error("Missing clientId"); - } - - const challenge = base64FromBytes( - new Uint8Array(await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(newCodeVerifier))) - ) - .replace(/[+]/g, "-") - .replace(/[/]/g, "_") - .replace(/=/g, ""); - - return `${opendidConfig.authorization_endpoint}?${new URLSearchParams({ - client_id: clientId, - scope: opts?.scopes || variables?.OAUTH_SCOPES || "openid profile", - response_type: "code", - redirect_uri: redirectUri, - state, - code_challenge: challenge, - code_challenge_method: "S256", - }).toString()}`; -} diff --git a/packages/blob/src/lib/parse-safetensors-metadata.spec.ts b/packages/blob/src/lib/parse-safetensors-metadata.spec.ts deleted file mode 100644 index 71077e3bb..000000000 --- a/packages/blob/src/lib/parse-safetensors-metadata.spec.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { assert, it, describe } from "vitest"; -import { parseSafetensorsMetadata, parseSafetensorsShardFilename } from "./parse-safetensors-metadata"; -import { sum } from "../utils/sum"; - -describe("parseSafetensorsMetadata", () => { - it("fetch info for single-file (with the default conventional filename)", async () => { - const parse = await parseSafetensorsMetadata({ - repo: "bert-base-uncased", - computeParametersCount: true, - revision: "86b5e0934494bd15c9632b12f734a8a67f723594", - }); - - assert(!parse.sharded); - assert.deepStrictEqual(parse.header.__metadata__, { format: "pt" }); - - // Example of one tensor (the header contains many tensors) - - assert.deepStrictEqual(parse.header["bert.embeddings.LayerNorm.beta"], { - dtype: "F32", - shape: [768], - data_offsets: [0, 3072], - }); - - assert.deepStrictEqual(parse.parameterCount, { F32: 110_106_428 }); - assert.deepStrictEqual(sum(Object.values(parse.parameterCount)), 110_106_428); - // total params = 110m - }); - - it("fetch info for sharded (with the default conventional filename)", async () => { - const parse = await parseSafetensorsMetadata({ - repo: "bigscience/bloom", - computeParametersCount: true, - revision: "053d9cd9fbe814e091294f67fcfedb3397b954bb", - }); - - assert(parse.sharded); - - assert.strictEqual(Object.keys(parse.headers).length, 72); - // This model has 72 shards! - - // Example of one tensor inside one file - - assert.deepStrictEqual(parse.headers["model_00012-of-00072.safetensors"]["h.10.input_layernorm.weight"], { - dtype: "BF16", - shape: [14336], - data_offsets: [3288649728, 3288678400], - }); - - assert.deepStrictEqual(parse.parameterCount, { BF16: 176_247_271_424 }); - assert.deepStrictEqual(sum(Object.values(parse.parameterCount)), 176_247_271_424); - // total params = 176B - }, 30_000); - - it("fetch info for single-file with multiple dtypes", async () => { - const parse = await parseSafetensorsMetadata({ - repo: "roberta-base", - computeParametersCount: true, - revision: "e2da8e2f811d1448a5b465c236feacd80ffbac7b", - }); - - assert(!parse.sharded); - - assert.deepStrictEqual(parse.parameterCount, { F32: 124_697_433, I64: 514 }); - assert.deepStrictEqual(sum(Object.values(parse.parameterCount)), 124_697_947); - // total params = 124m - }); - - it("fetch info for single-file with file path", async () => { - const parse = await parseSafetensorsMetadata({ - repo: "CompVis/stable-diffusion-v1-4", - computeParametersCount: true, - path: "unet/diffusion_pytorch_model.safetensors", - revision: "133a221b8aa7292a167afc5127cb63fb5005638b", - }); - - assert(!parse.sharded); - assert.deepStrictEqual(parse.header.__metadata__, { format: "pt" }); - - // Example of one tensor (the header contains many tensors) - - assert.deepStrictEqual(parse.header["up_blocks.3.resnets.0.norm2.bias"], { - dtype: "F32", - shape: [320], - data_offsets: [3_409_382_416, 3_409_383_696], - }); - - assert.deepStrictEqual(parse.parameterCount, { F32: 859_520_964 }); - assert.deepStrictEqual(sum(Object.values(parse.parameterCount)), 859_520_964); - }); - - it("fetch info for sharded (with the default conventional filename) with file path", async () => { - const parse = await parseSafetensorsMetadata({ - repo: "Alignment-Lab-AI/ALAI-gemma-7b", - computeParametersCount: true, - path: "7b/1/model.safetensors.index.json", - revision: "37e307261fe97bbf8b2463d61dbdd1a10daa264c", - }); - - assert(parse.sharded); - - assert.strictEqual(Object.keys(parse.headers).length, 4); - - assert.deepStrictEqual(parse.headers["model-00004-of-00004.safetensors"]["model.layers.24.mlp.up_proj.weight"], { - dtype: "BF16", - shape: [24576, 3072], - data_offsets: [301996032, 452990976], - }); - - assert.deepStrictEqual(parse.parameterCount, { BF16: 8_537_680_896 }); - assert.deepStrictEqual(sum(Object.values(parse.parameterCount)), 8_537_680_896); - }); - - it("should detect sharded safetensors filename", async () => { - const safetensorsFilename = "model_00005-of-00072.safetensors"; // https://huggingface.co/bigscience/bloom/blob/4d8e28c67403974b0f17a4ac5992e4ba0b0dbb6f/model_00005-of-00072.safetensors - const safetensorsShardFileInfo = parseSafetensorsShardFilename(safetensorsFilename); - - assert.strictEqual(safetensorsShardFileInfo?.prefix, "model_"); - assert.strictEqual(safetensorsShardFileInfo?.basePrefix, "model"); - assert.strictEqual(safetensorsShardFileInfo?.shard, "00005"); - assert.strictEqual(safetensorsShardFileInfo?.total, "00072"); - }); -}); diff --git a/packages/blob/src/lib/parse-safetensors-metadata.ts b/packages/blob/src/lib/parse-safetensors-metadata.ts deleted file mode 100644 index 063a503c9..000000000 --- a/packages/blob/src/lib/parse-safetensors-metadata.ts +++ /dev/null @@ -1,285 +0,0 @@ -import type { CredentialsParams, RepoDesignation } from "../types/public"; -import { omit } from "../utils/omit"; -import { toRepoId } from "../utils/toRepoId"; -import { typedEntries } from "../utils/typedEntries"; -import { downloadFile } from "./download-file"; -import { fileExists } from "./file-exists"; -import { promisesQueue } from "../utils/promisesQueue"; -import type { SetRequired } from "../vendor/type-fest/set-required"; - -export const SAFETENSORS_FILE = "model.safetensors"; -export const SAFETENSORS_INDEX_FILE = "model.safetensors.index.json"; -/// We advise model/library authors to use the filenames above for convention inside model repos, -/// but in some situations safetensors weights have different filenames. -export const RE_SAFETENSORS_FILE = /\.safetensors$/; -export const RE_SAFETENSORS_INDEX_FILE = /\.safetensors\.index\.json$/; -export const RE_SAFETENSORS_SHARD_FILE = - /^(?(?.*?)[_-])(?\d{5})-of-(?\d{5})\.safetensors$/; -export interface SafetensorsShardFileInfo { - prefix: string; - basePrefix: string; - shard: string; - total: string; -} -export function parseSafetensorsShardFilename(filename: string): SafetensorsShardFileInfo | null { - const match = RE_SAFETENSORS_SHARD_FILE.exec(filename); - if (match && match.groups) { - return { - prefix: match.groups["prefix"], - basePrefix: match.groups["basePrefix"], - shard: match.groups["shard"], - total: match.groups["total"], - }; - } - return null; -} - -const PARALLEL_DOWNLOADS = 20; -const MAX_HEADER_LENGTH = 25_000_000; - -class SafetensorParseError extends Error {} - -type FileName = string; - -export type TensorName = string; -export type Dtype = "F64" | "F32" | "F16" | "BF16" | "I64" | "I32" | "I16" | "I8" | "U8" | "BOOL"; - -export interface TensorInfo { - dtype: Dtype; - shape: number[]; - data_offsets: [number, number]; -} - -export type SafetensorsFileHeader = Record & { - __metadata__: Record; -}; - -export interface SafetensorsIndexJson { - dtype?: string; - /// ^there's sometimes a dtype but it looks inconsistent. - metadata?: Record; - /// ^ why the naming inconsistency? - weight_map: Record; -} - -export type SafetensorsShardedHeaders = Record; - -export type SafetensorsParseFromRepo = - | { - sharded: false; - header: SafetensorsFileHeader; - parameterCount?: Partial>; - } - | { - sharded: true; - index: SafetensorsIndexJson; - headers: SafetensorsShardedHeaders; - parameterCount?: Partial>; - }; - -async function parseSingleFile( - path: string, - params: { - repo: RepoDesignation; - revision?: string; - hubUrl?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): Promise { - const firstResp = await downloadFile({ - ...params, - path, - range: [0, 7], - }); - - if (!firstResp) { - throw new SafetensorParseError(`Failed to parse file ${path}: failed to fetch safetensors header length.`); - } - - const bufLengthOfHeaderLE = await firstResp.arrayBuffer(); - const lengthOfHeader = new DataView(bufLengthOfHeaderLE).getBigUint64(0, true); - // ^little-endian - if (lengthOfHeader <= 0) { - throw new SafetensorParseError(`Failed to parse file ${path}: safetensors header is malformed.`); - } - if (lengthOfHeader > MAX_HEADER_LENGTH) { - throw new SafetensorParseError( - `Failed to parse file ${path}: safetensor header is too big. Maximum supported size is ${MAX_HEADER_LENGTH} bytes.` - ); - } - - const secondResp = await downloadFile({ ...params, path, range: [8, 7 + Number(lengthOfHeader)] }); - - if (!secondResp) { - throw new SafetensorParseError(`Failed to parse file ${path}: failed to fetch safetensors header.`); - } - - try { - // no validation for now, we assume it's a valid FileHeader. - const header: SafetensorsFileHeader = await secondResp.json(); - return header; - } catch (err) { - throw new SafetensorParseError(`Failed to parse file ${path}: safetensors header is not valid JSON.`); - } -} - -async function parseShardedIndex( - path: string, - params: { - repo: RepoDesignation; - revision?: string; - hubUrl?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): Promise<{ index: SafetensorsIndexJson; headers: SafetensorsShardedHeaders }> { - const indexResp = await downloadFile({ - ...params, - path, - range: [0, 10_000_000], - }); - - if (!indexResp) { - throw new SafetensorParseError(`Failed to parse file ${path}: failed to fetch safetensors index.`); - } - - // no validation for now, we assume it's a valid IndexJson. - let index: SafetensorsIndexJson; - try { - index = await indexResp.json(); - } catch (error) { - throw new SafetensorParseError(`Failed to parse file ${path}: not a valid JSON.`); - } - - const pathPrefix = path.slice(0, path.lastIndexOf("/") + 1); - const filenames = [...new Set(Object.values(index.weight_map))]; - const shardedMap: SafetensorsShardedHeaders = Object.fromEntries( - await promisesQueue( - filenames.map( - (filename) => async () => - [filename, await parseSingleFile(pathPrefix + filename, params)] satisfies [string, SafetensorsFileHeader] - ), - PARALLEL_DOWNLOADS - ) - ); - return { index, headers: shardedMap }; -} - -/** - * Analyze model.safetensors.index.json or model.safetensors from a model hosted - * on Hugging Face using smart range requests to extract its metadata. - */ -export async function parseSafetensorsMetadata( - params: { - /** Only models are supported */ - repo: RepoDesignation; - /** - * Relative file path to safetensors file inside `repo`. Defaults to `SAFETENSORS_FILE` or `SAFETENSORS_INDEX_FILE` (whichever one exists). - */ - path?: string; - /** - * Will include SafetensorsParseFromRepo["parameterCount"], an object containing the number of parameters for each DType - * - * @default false - */ - computeParametersCount: true; - hubUrl?: string; - revision?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): Promise>; -export async function parseSafetensorsMetadata( - params: { - /** Only models are supported */ - repo: RepoDesignation; - /** - * Will include SafetensorsParseFromRepo["parameterCount"], an object containing the number of parameters for each DType - * - * @default false - */ - path?: string; - computeParametersCount?: boolean; - hubUrl?: string; - revision?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): Promise; -export async function parseSafetensorsMetadata( - params: { - repo: RepoDesignation; - path?: string; - computeParametersCount?: boolean; - hubUrl?: string; - revision?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): Promise { - const repoId = toRepoId(params.repo); - - if (repoId.type !== "model") { - throw new TypeError("Only model repos should contain safetensors files."); - } - - if (RE_SAFETENSORS_FILE.test(params.path ?? "") || (await fileExists({ ...params, path: SAFETENSORS_FILE }))) { - const header = await parseSingleFile(params.path ?? SAFETENSORS_FILE, params); - return { - sharded: false, - header, - ...(params.computeParametersCount && { - parameterCount: computeNumOfParamsByDtypeSingleFile(header), - }), - }; - } else if ( - RE_SAFETENSORS_INDEX_FILE.test(params.path ?? "") || - (await fileExists({ ...params, path: SAFETENSORS_INDEX_FILE })) - ) { - const { index, headers } = await parseShardedIndex(params.path ?? SAFETENSORS_INDEX_FILE, params); - return { - sharded: true, - index, - headers, - ...(params.computeParametersCount && { - parameterCount: computeNumOfParamsByDtypeSharded(headers), - }), - }; - } else { - throw new Error("model id does not seem to contain safetensors weights"); - } -} - -function computeNumOfParamsByDtypeSingleFile(header: SafetensorsFileHeader): Partial> { - const counter: Partial> = {}; - const tensors = omit(header, "__metadata__"); - - for (const [, v] of typedEntries(tensors)) { - if (v.shape.length === 0) { - continue; - } - counter[v.dtype] = (counter[v.dtype] ?? 0) + v.shape.reduce((a, b) => a * b); - } - return counter; -} - -function computeNumOfParamsByDtypeSharded(shardedMap: SafetensorsShardedHeaders): Partial> { - const counter: Partial> = {}; - for (const header of Object.values(shardedMap)) { - for (const [k, v] of typedEntries(computeNumOfParamsByDtypeSingleFile(header))) { - counter[k] = (counter[k] ?? 0) + (v ?? 0); - } - } - return counter; -} diff --git a/packages/blob/src/lib/paths-info.spec.ts b/packages/blob/src/lib/paths-info.spec.ts deleted file mode 100644 index 994d623ae..000000000 --- a/packages/blob/src/lib/paths-info.spec.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { expect, it, describe } from "vitest"; -import type { CommitInfo, PathInfo, SecurityFileStatus } from "./paths-info"; -import { pathsInfo } from "./paths-info"; - -describe("pathsInfo", () => { - it("should fetch LFS path info", async () => { - const result: PathInfo[] = await pathsInfo({ - repo: { - name: "bert-base-uncased", - type: "model", - }, - paths: ["tf_model.h5"], - revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", - }); - - expect(result).toHaveLength(1); - - const modelPathInfo = result[0]; - expect(modelPathInfo.path).toBe('tf_model.h5'); - expect(modelPathInfo.type).toBe('file'); - // lfs pointer, therefore lfs should be defined - expect(modelPathInfo?.lfs).toBeDefined(); - expect(modelPathInfo?.lfs?.oid).toBe("a7a17d6d844b5de815ccab5f42cad6d24496db3850a2a43d8258221018ce87d2"); - expect(modelPathInfo?.lfs?.size).toBe(536063208); - expect(modelPathInfo?.lfs?.pointerSize).toBe(134); - - // should not include expand info - expect(modelPathInfo.lastCommit).toBeUndefined(); - expect(modelPathInfo.securityFileStatus).toBeUndefined(); - }); - - it("expand parmas should fetch lastCommit and securityFileStatus", async () => { - const result: (PathInfo & { - lastCommit: CommitInfo, - securityFileStatus: SecurityFileStatus, - })[] = await pathsInfo({ - repo: { - name: "bert-base-uncased", - type: "model", - }, - paths: ["tf_model.h5"], - revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", - expand: true, // include - }); - - expect(result).toHaveLength(1); - - const modelPathInfo = result[0]; - - // should include expand info - expect(modelPathInfo.lastCommit).toBeDefined(); - expect(modelPathInfo.securityFileStatus).toBeDefined(); - - expect(modelPathInfo.lastCommit.id).toBe("dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7"); - expect(modelPathInfo.lastCommit.title).toBe("Update tf_model.h5"); - expect(modelPathInfo.lastCommit.date.getTime()).toBe(1569268124000); // 2019-09-23T19:48:44.000Z - }); - - it("non-LFS pointer should have lfs undefined", async () => { - const result: (PathInfo)[] = await pathsInfo({ - repo: { - name: "bert-base-uncased", - type: "model", - }, - paths: ["config.json"], - revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", - }); - - expect(result).toHaveLength(1); - - const modelPathInfo = result[0]; - expect(modelPathInfo.path).toBe("config.json"); - expect(modelPathInfo.lfs).toBeUndefined(); - }); -}); diff --git a/packages/blob/src/lib/paths-info.ts b/packages/blob/src/lib/paths-info.ts deleted file mode 100644 index 4c9a1de20..000000000 --- a/packages/blob/src/lib/paths-info.ts +++ /dev/null @@ -1,120 +0,0 @@ -import type { CredentialsParams, RepoDesignation } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; -import { toRepoId } from "../utils/toRepoId"; -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; - -export interface LfsPathInfo { - "oid": string, - "size": number, - "pointerSize": number -} - -export interface CommitInfo { - "id": string, - "title": string, - "date": Date, -} - -export interface SecurityFileStatus { - "status": string, -} - -export interface PathInfo { - path: string, - type: string, - oid: string, - size: number, - /** - * Only defined when path is LFS pointer - */ - lfs?: LfsPathInfo, - lastCommit?: CommitInfo, - securityFileStatus?: SecurityFileStatus -} - -// Define the overloaded signatures -export function pathsInfo( - params: { - repo: RepoDesignation; - paths: string[]; - expand: true; // if expand true - revision?: string; - hubUrl?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): Promise<(PathInfo & {lastCommit: CommitInfo, securityFileStatus: SecurityFileStatus })[]>; -export function pathsInfo( - params: { - repo: RepoDesignation; - paths: string[]; - expand?: boolean; - revision?: string; - hubUrl?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): Promise<(PathInfo)[]>; - -export async function pathsInfo( - params: { - repo: RepoDesignation; - paths: string[]; - expand?: boolean; - revision?: string; - hubUrl?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): Promise { - const accessToken = checkCredentials(params); - const repoId = toRepoId(params.repo); - - const hubUrl = params.hubUrl ?? HUB_URL; - - const url = `${hubUrl}/api/${repoId.type}s/${repoId.name}/paths-info/${encodeURIComponent(params.revision ?? "main")}`; - - const resp = await (params.fetch ?? fetch)(url, { - method: "POST", - headers: { - ...(params.credentials && { - Authorization: `Bearer ${accessToken}`, - }), - 'Accept': 'application/json', - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - paths: params.paths, - expand: params.expand, - }), - }); - - if (!resp.ok) { - throw await createApiError(resp); - } - - const json: unknown = await resp.json(); - if(!Array.isArray(json)) throw new Error('malformed response: expected array'); - - return json.map((item: PathInfo) => ({ - path: item.path, - lfs: item.lfs, - type: item.type, - oid: item.oid, - size: item.size, - // expand fields - securityFileStatus: item.securityFileStatus, - lastCommit: item.lastCommit ? { - date: new Date(item.lastCommit.date), - title: item.lastCommit.title, - id: item.lastCommit.id, - }: undefined, - })); -} diff --git a/packages/blob/src/lib/snapshot-download.spec.ts b/packages/blob/src/lib/snapshot-download.spec.ts deleted file mode 100644 index 0c44cc842..000000000 --- a/packages/blob/src/lib/snapshot-download.spec.ts +++ /dev/null @@ -1,275 +0,0 @@ -import { expect, test, describe, vi, beforeEach } from "vitest"; -import { dirname, join } from "node:path"; -import { mkdir, writeFile } from "node:fs/promises"; -import { getHFHubCachePath } from "./cache-management"; -import { downloadFileToCacheDir } from "./download-file-to-cache-dir"; -import { snapshotDownload } from "./snapshot-download"; -import type { ListFileEntry } from "./list-files"; -import { listFiles } from "./list-files"; -import { modelInfo } from "./model-info"; -import type { ModelEntry } from "./list-models"; -import type { ApiModelInfo } from "../types/api/api-model"; -import { datasetInfo } from "./dataset-info"; -import type { DatasetEntry } from "./list-datasets"; -import type { ApiDatasetInfo } from "../types/api/api-dataset"; -import { spaceInfo } from "./space-info"; -import type { SpaceEntry } from "./list-spaces"; -import type { ApiSpaceInfo } from "../types/api/api-space"; - -vi.mock("node:fs/promises", () => ({ - writeFile: vi.fn(), - mkdir: vi.fn(), -})); - -vi.mock("./space-info", () => ({ - spaceInfo: vi.fn(), -})); - -vi.mock("./dataset-info", () => ({ - datasetInfo: vi.fn(), -})); - -vi.mock("./model-info", () => ({ - modelInfo: vi.fn(), -})); - -vi.mock("./list-files", () => ({ - listFiles: vi.fn(), -})); - -vi.mock("./download-file-to-cache-dir", () => ({ - downloadFileToCacheDir: vi.fn(), -})); - -const DUMMY_SHA = "dummy-sha"; - -// utility method to transform an array of ListFileEntry to an AsyncGenerator -async function* toAsyncGenerator(content: ListFileEntry[]): AsyncGenerator { - for (const entry of content) { - yield Promise.resolve(entry); - } -} - -beforeEach(() => { - vi.resetAllMocks(); - vi.mocked(listFiles).mockReturnValue(toAsyncGenerator([])); - - // mock repo info - vi.mocked(modelInfo).mockResolvedValue({ - sha: DUMMY_SHA, - } as ModelEntry & ApiModelInfo); - vi.mocked(datasetInfo).mockResolvedValue({ - sha: DUMMY_SHA, - } as DatasetEntry & ApiDatasetInfo); - vi.mocked(spaceInfo).mockResolvedValue({ - sha: DUMMY_SHA, - } as SpaceEntry & ApiSpaceInfo); -}); - -describe("snapshotDownload", () => { - test("empty AsyncGenerator should not call downloadFileToCacheDir", async () => { - await snapshotDownload({ - repo: { - name: "foo/bar", - type: "space", - }, - }); - - expect(downloadFileToCacheDir).not.toHaveBeenCalled(); - }); - - test("repo type model should use modelInfo", async () => { - await snapshotDownload({ - repo: { - name: "foo/bar", - type: "model", - }, - }); - expect(modelInfo).toHaveBeenCalledOnce(); - expect(modelInfo).toHaveBeenCalledWith({ - name: "foo/bar", - additionalFields: ["sha"], - revision: "main", - repo: { - name: "foo/bar", - type: "model", - }, - }); - }); - - test("repo type dataset should use datasetInfo", async () => { - await snapshotDownload({ - repo: { - name: "foo/bar", - type: "dataset", - }, - }); - expect(datasetInfo).toHaveBeenCalledOnce(); - expect(datasetInfo).toHaveBeenCalledWith({ - name: "foo/bar", - additionalFields: ["sha"], - revision: "main", - repo: { - name: "foo/bar", - type: "dataset", - }, - }); - }); - - test("repo type space should use spaceInfo", async () => { - await snapshotDownload({ - repo: { - name: "foo/bar", - type: "space", - }, - }); - expect(spaceInfo).toHaveBeenCalledOnce(); - expect(spaceInfo).toHaveBeenCalledWith({ - name: "foo/bar", - additionalFields: ["sha"], - revision: "main", - repo: { - name: "foo/bar", - type: "space", - }, - }); - }); - - test("commitHash should be saved to ref folder", async () => { - await snapshotDownload({ - repo: { - name: "foo/bar", - type: "space", - }, - revision: "dummy-revision", - }); - - // cross-platform testing - const expectedPath = join(getHFHubCachePath(), "spaces--foo--bar", "refs", "dummy-revision"); - expect(mkdir).toHaveBeenCalledWith(dirname(expectedPath), { recursive: true }); - expect(writeFile).toHaveBeenCalledWith(expectedPath, DUMMY_SHA); - }); - - test("directory ListFileEntry should mkdir it", async () => { - vi.mocked(listFiles).mockReturnValue( - toAsyncGenerator([ - { - oid: "dummy-etag", - type: "directory", - path: "potatoes", - size: 0, - lastCommit: { - date: new Date().toISOString(), - id: DUMMY_SHA, - title: "feat: best commit", - }, - }, - ]) - ); - - await snapshotDownload({ - repo: { - name: "foo/bar", - type: "space", - }, - }); - - // cross-platform testing - const expectedPath = join(getHFHubCachePath(), "spaces--foo--bar", "snapshots", DUMMY_SHA, "potatoes"); - expect(mkdir).toHaveBeenCalledWith(expectedPath, { recursive: true }); - }); - - test("files in ListFileEntry should download them", async () => { - const entries: ListFileEntry[] = Array.from({ length: 10 }, (_, i) => ({ - oid: `dummy-etag-${i}`, - type: "file", - path: `file-${i}.txt`, - size: i, - lastCommit: { - date: new Date().toISOString(), - id: DUMMY_SHA, - title: "feat: best commit", - }, - })); - vi.mocked(listFiles).mockReturnValue(toAsyncGenerator(entries)); - - await snapshotDownload({ - repo: { - name: "foo/bar", - type: "space", - }, - }); - - for (const entry of entries) { - expect(downloadFileToCacheDir).toHaveBeenCalledWith( - expect.objectContaining({ - repo: { - name: "foo/bar", - type: "space", - }, - path: entry.path, - revision: DUMMY_SHA, - }) - ); - } - }); - - test("custom params should be propagated", async () => { - // fetch mock - const fetchMock: typeof fetch = vi.fn(); - const hubMock = "https://foor.bar"; - const accessTokenMock = "dummy-access-token"; - - vi.mocked(listFiles).mockReturnValue( - toAsyncGenerator([ - { - oid: `dummy-etag`, - type: "file", - path: `file.txt`, - size: 10, - lastCommit: { - date: new Date().toISOString(), - id: DUMMY_SHA, - title: "feat: best commit", - }, - }, - ]) - ); - - await snapshotDownload({ - repo: { - name: "foo/bar", - type: "space", - }, - hubUrl: hubMock, - fetch: fetchMock, - accessToken: accessTokenMock, - }); - - expect(spaceInfo).toHaveBeenCalledWith( - expect.objectContaining({ - fetch: fetchMock, - hubUrl: hubMock, - accessToken: accessTokenMock, - }) - ); - - // list files should receive custom fetch - expect(listFiles).toHaveBeenCalledWith( - expect.objectContaining({ - fetch: fetchMock, - hubUrl: hubMock, - accessToken: accessTokenMock, - }) - ); - - // download file to cache should receive custom fetch - expect(downloadFileToCacheDir).toHaveBeenCalledWith( - expect.objectContaining({ - fetch: fetchMock, - hubUrl: hubMock, - accessToken: accessTokenMock, - }) - ); - }); -}); diff --git a/packages/blob/src/lib/snapshot-download.ts b/packages/blob/src/lib/snapshot-download.ts deleted file mode 100644 index b3e30c13f..000000000 --- a/packages/blob/src/lib/snapshot-download.ts +++ /dev/null @@ -1,124 +0,0 @@ -import type { CredentialsParams, RepoDesignation } from "../types/public"; -import { listFiles } from "./list-files"; -import { getHFHubCachePath, getRepoFolderName } from "./cache-management"; -import { spaceInfo } from "./space-info"; -import { datasetInfo } from "./dataset-info"; -import { modelInfo } from "./model-info"; -import { toRepoId } from "../utils/toRepoId"; -import { join, dirname } from "node:path"; -import { mkdir, writeFile } from "node:fs/promises"; -import { downloadFileToCacheDir } from "./download-file-to-cache-dir"; - -export const DEFAULT_REVISION = "main"; - -/** - * Downloads an entire repository at a given revision in the cache directory {@link getHFHubCachePath}. - * You can list all cached repositories using {@link scanCachedRepo} - * @remarks It uses internally {@link downloadFileToCacheDir}. - */ -export async function snapshotDownload( - params: { - repo: RepoDesignation; - cacheDir?: string; - /** - * An optional Git revision id which can be a branch name, a tag, or a commit hash. - * - * @default "main" - */ - revision?: string; - hubUrl?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): Promise { - let cacheDir: string; - if (params.cacheDir) { - cacheDir = params.cacheDir; - } else { - cacheDir = getHFHubCachePath(); - } - - let revision: string; - if (params.revision) { - revision = params.revision; - } else { - revision = DEFAULT_REVISION; - } - - const repoId = toRepoId(params.repo); - - // get repository revision value (sha) - let repoInfo: { sha: string }; - switch (repoId.type) { - case "space": - repoInfo = await spaceInfo({ - ...params, - name: repoId.name, - additionalFields: ["sha"], - revision: revision, - }); - break; - case "dataset": - repoInfo = await datasetInfo({ - ...params, - name: repoId.name, - additionalFields: ["sha"], - revision: revision, - }); - break; - case "model": - repoInfo = await modelInfo({ - ...params, - name: repoId.name, - additionalFields: ["sha"], - revision: revision, - }); - break; - default: - throw new Error(`invalid repository type ${repoId.type}`); - } - - const commitHash: string = repoInfo.sha; - - // get storage folder - const storageFolder = join(cacheDir, getRepoFolderName(repoId)); - const snapshotFolder = join(storageFolder, "snapshots", commitHash); - - // if passed revision is not identical to commit_hash - // then revision has to be a branch name or tag name. - // In that case store a ref. - if (revision !== commitHash) { - const refPath = join(storageFolder, "refs", revision); - await mkdir(dirname(refPath), { recursive: true }); - await writeFile(refPath, commitHash); - } - - const cursor = listFiles({ - ...params, - repo: params.repo, - recursive: true, - revision: repoInfo.sha, - }); - - for await (const entry of cursor) { - switch (entry.type) { - case "file": - await downloadFileToCacheDir({ - ...params, - path: entry.path, - revision: commitHash, - cacheDir: cacheDir, - }); - break; - case "directory": - await mkdir(join(snapshotFolder, entry.path), { recursive: true }); - break; - default: - throw new Error(`unknown entry type: ${entry.type}`); - } - } - - return snapshotFolder; -} diff --git a/packages/blob/src/lib/space-info.spec.ts b/packages/blob/src/lib/space-info.spec.ts deleted file mode 100644 index aafa9b88f..000000000 --- a/packages/blob/src/lib/space-info.spec.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { spaceInfo } from "./space-info"; -import type { SpaceEntry } from "./list-spaces"; -import type { ApiSpaceInfo } from "../types/api/api-space"; - -describe("spaceInfo", () => { - it("should return the space info", async () => { - const info = await spaceInfo({ - name: "huggingfacejs/client-side-oauth", - }); - expect(info).toEqual({ - id: "659835e689010f9c7aed608d", - name: "huggingfacejs/client-side-oauth", - updatedAt: expect.any(Date), - likes: expect.any(Number), - private: false, - sdk: "static", - }); - }); - - it("should return the space info with author", async () => { - const info: SpaceEntry & Pick = await spaceInfo({ - name: "huggingfacejs/client-side-oauth", - additionalFields: ['author'], - }); - expect(info).toEqual({ - id: "659835e689010f9c7aed608d", - name: "huggingfacejs/client-side-oauth", - updatedAt: expect.any(Date), - likes: expect.any(Number), - private: false, - sdk: "static", - author: 'huggingfacejs', - }); - }); - - it("should return the space info for a given revision", async () => { - const info: SpaceEntry & Pick = await spaceInfo({ - name: "huggingfacejs/client-side-oauth", - additionalFields: ['sha'], - revision: 'e410a9ff348e6bed393b847711e793282d7c672e' - }); - expect(info).toEqual({ - id: "659835e689010f9c7aed608d", - name: "huggingfacejs/client-side-oauth", - updatedAt: expect.any(Date), - likes: expect.any(Number), - private: false, - sdk: "static", - sha: 'e410a9ff348e6bed393b847711e793282d7c672e', - }); - }); -}); diff --git a/packages/blob/src/lib/space-info.ts b/packages/blob/src/lib/space-info.ts deleted file mode 100644 index fcbfee60d..000000000 --- a/packages/blob/src/lib/space-info.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; -import type { ApiSpaceInfo } from "../types/api/api-space"; -import type { CredentialsParams } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; -import { pick } from "../utils/pick"; -import type { SPACE_EXPANDABLE_KEYS, SpaceEntry } from "./list-spaces"; -import { SPACE_EXPAND_KEYS } from "./list-spaces"; - -export async function spaceInfo< - const T extends Exclude<(typeof SPACE_EXPANDABLE_KEYS)[number], (typeof SPACE_EXPAND_KEYS)[number]> = never, ->( - params: { - name: string; - hubUrl?: string; - additionalFields?: T[]; - /** - * An optional Git revision id which can be a branch name, a tag, or a commit hash. - */ - revision?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & Partial -): Promise> { - const accessToken = params && checkCredentials(params); - - const search = new URLSearchParams([ - ...SPACE_EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), - ...(params?.additionalFields?.map((val) => ["expand", val] satisfies [string, string]) ?? []), - ]).toString(); - - const response = await (params.fetch || fetch)( - `${params?.hubUrl || HUB_URL}/api/spaces/${params.name}/revision/${encodeURIComponent(params.revision ?? "HEAD")}?${search.toString()}`, - { - headers: { - ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), - Accepts: "application/json", - }, - } - ); - - if (!response.ok) { - throw await createApiError(response); - } - - const data = await response.json(); - - return { - ...(params?.additionalFields && pick(data, params.additionalFields)), - id: data._id, - name: data.id, - sdk: data.sdk, - likes: data.likes, - private: data.private, - updatedAt: new Date(data.lastModified), - } as SpaceEntry & Pick; -} diff --git a/packages/blob/src/lib/upload-file.spec.ts b/packages/blob/src/lib/upload-file.spec.ts deleted file mode 100644 index af7534963..000000000 --- a/packages/blob/src/lib/upload-file.spec.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { assert, it, describe } from "vitest"; - -import { TEST_ACCESS_TOKEN, TEST_HUB_URL, TEST_USER } from "../test/consts"; -import type { RepoId } from "../types/public"; -import { insecureRandomString } from "../utils/insecureRandomString"; -import { createRepo } from "./create-repo"; -import { deleteRepo } from "./delete-repo"; -import { downloadFile } from "./download-file"; -import { uploadFile } from "./upload-file"; - -describe("uploadFile", () => { - it("should upload a file", async () => { - const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; - const repo = { type: "model", name: repoName } satisfies RepoId; - - try { - const result = await createRepo({ - accessToken: TEST_ACCESS_TOKEN, - repo, - hubUrl: TEST_HUB_URL, - }); - - assert.deepStrictEqual(result, { - repoUrl: `${TEST_HUB_URL}/${repoName}`, - }); - - await uploadFile({ - accessToken: TEST_ACCESS_TOKEN, - repo, - file: { content: new Blob(["file1"]), path: "file1" }, - hubUrl: TEST_HUB_URL, - }); - await uploadFile({ - accessToken: TEST_ACCESS_TOKEN, - repo, - file: new URL("https://huggingface.co/gpt2/raw/main/config.json"), - hubUrl: TEST_HUB_URL, - }); - - let content = await downloadFile({ - repo, - path: "file1", - hubUrl: TEST_HUB_URL, - }); - - assert.strictEqual(await content?.text(), "file1"); - - content = await downloadFile({ - repo, - path: "config.json", - hubUrl: TEST_HUB_URL, - }); - - assert.strictEqual( - (await content?.text())?.trim(), - ` -{ - "activation_function": "gelu_new", - "architectures": [ - "GPT2LMHeadModel" - ], - "attn_pdrop": 0.1, - "bos_token_id": 50256, - "embd_pdrop": 0.1, - "eos_token_id": 50256, - "initializer_range": 0.02, - "layer_norm_epsilon": 1e-05, - "model_type": "gpt2", - "n_ctx": 1024, - "n_embd": 768, - "n_head": 12, - "n_layer": 12, - "n_positions": 1024, - "resid_pdrop": 0.1, - "summary_activation": null, - "summary_first_dropout": 0.1, - "summary_proj_to_labels": true, - "summary_type": "cls_index", - "summary_use_proj": true, - "task_specific_params": { - "text-generation": { - "do_sample": true, - "max_length": 50 - } - }, - "vocab_size": 50257 -} - `.trim() - ); - } finally { - await deleteRepo({ - repo, - accessToken: TEST_ACCESS_TOKEN, - hubUrl: TEST_HUB_URL, - }); - } - }); -}); diff --git a/packages/blob/src/lib/upload-file.ts b/packages/blob/src/lib/upload-file.ts deleted file mode 100644 index 290cfcb03..000000000 --- a/packages/blob/src/lib/upload-file.ts +++ /dev/null @@ -1,47 +0,0 @@ -import type { CredentialsParams } from "../types/public"; -import type { CommitOutput, CommitParams, ContentSource } from "./commit"; -import { commit } from "./commit"; - -export function uploadFile( - params: { - repo: CommitParams["repo"]; - file: URL | File | { path: string; content: ContentSource }; - commitTitle?: CommitParams["title"]; - commitDescription?: CommitParams["description"]; - hubUrl?: CommitParams["hubUrl"]; - branch?: CommitParams["branch"]; - isPullRequest?: CommitParams["isPullRequest"]; - parentCommit?: CommitParams["parentCommit"]; - fetch?: CommitParams["fetch"]; - useWebWorkers?: CommitParams["useWebWorkers"]; - abortSignal?: CommitParams["abortSignal"]; - } & Partial -): Promise { - const path = - params.file instanceof URL - ? params.file.pathname.split("/").at(-1) ?? "file" - : "path" in params.file - ? params.file.path - : params.file.name; - - return commit({ - ...(params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }), - repo: params.repo, - operations: [ - { - operation: "addOrUpdate", - path, - content: "content" in params.file ? params.file.content : params.file, - }, - ], - title: params.commitTitle ?? `Add ${path}`, - description: params.commitDescription, - hubUrl: params.hubUrl, - branch: params.branch, - isPullRequest: params.isPullRequest, - parentCommit: params.parentCommit, - fetch: params.fetch, - useWebWorkers: params.useWebWorkers, - abortSignal: params.abortSignal, - }); -} diff --git a/packages/blob/src/lib/upload-files-with-progress.spec.ts b/packages/blob/src/lib/upload-files-with-progress.spec.ts deleted file mode 100644 index b088e5597..000000000 --- a/packages/blob/src/lib/upload-files-with-progress.spec.ts +++ /dev/null @@ -1,168 +0,0 @@ -import { assert, it, describe } from "vitest"; - -import { TEST_HUB_URL, TEST_ACCESS_TOKEN, TEST_USER } from "../test/consts"; -import type { RepoId } from "../types/public"; -import { insecureRandomString } from "../utils/insecureRandomString"; -import { createRepo } from "./create-repo"; -import { deleteRepo } from "./delete-repo"; -import { downloadFile } from "./download-file"; -import { uploadFilesWithProgress } from "./upload-files-with-progress"; -import type { CommitOutput, CommitProgressEvent } from "./commit"; - -describe("uploadFilesWithProgress", () => { - it("should upload files", async () => { - const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; - const repo = { type: "model", name: repoName } satisfies RepoId; - const lfsContent = "O123456789".repeat(100_000); - - try { - const result = await createRepo({ - accessToken: TEST_ACCESS_TOKEN, - repo, - hubUrl: TEST_HUB_URL, - }); - - assert.deepStrictEqual(result, { - repoUrl: `${TEST_HUB_URL}/${repoName}`, - }); - - const it = uploadFilesWithProgress({ - accessToken: TEST_ACCESS_TOKEN, - repo, - files: [ - { content: new Blob(["file1"]), path: "file1" }, - new URL("https://huggingface.co/gpt2/raw/main/config.json"), - // Large file - { - content: new Blob([lfsContent]), - path: "test.lfs.txt", - }, - ], - useWebWorkers: { - minSize: 1_000, - }, - hubUrl: TEST_HUB_URL, - }); - - let res: IteratorResult; - let progressEvents: CommitProgressEvent[] = []; - - do { - res = await it.next(); - if (!res.done) { - progressEvents.push(res.value); - } - } while (!res.done); - - // const intermediateHashingEvents = progressEvents.filter( - // (e) => e.event === "fileProgress" && e.type === "hashing" && e.progress !== 0 && e.progress !== 1 - // ); - // if (isFrontend) { - // assert(intermediateHashingEvents.length > 0); - // } - // const intermediateUploadEvents = progressEvents.filter( - // (e) => e.event === "fileProgress" && e.type === "uploading" && e.progress !== 0 && e.progress !== 1 - // ); - // if (isFrontend) { - // assert(intermediateUploadEvents.length > 0, "There should be at least one intermediate upload event"); - // } - progressEvents = progressEvents.filter((e) => e.event !== "fileProgress" || e.progress === 0 || e.progress === 1); - - assert.deepStrictEqual(progressEvents, [ - { - event: "phase", - phase: "preuploading", - }, - { - event: "phase", - phase: "uploadingLargeFiles", - }, - { - event: "fileProgress", - path: "test.lfs.txt", - progress: 0, - state: "hashing", - }, - { - event: "fileProgress", - path: "test.lfs.txt", - progress: 1, - state: "hashing", - }, - { - event: "fileProgress", - path: "test.lfs.txt", - progress: 0, - state: "uploading", - }, - { - event: "fileProgress", - path: "test.lfs.txt", - progress: 1, - state: "uploading", - }, - { - event: "phase", - phase: "committing", - }, - ]); - - let content = await downloadFile({ - repo, - path: "file1", - hubUrl: TEST_HUB_URL, - }); - - assert.strictEqual(await content?.text(), "file1"); - - content = await downloadFile({ - repo, - path: "config.json", - hubUrl: TEST_HUB_URL, - }); - - assert.strictEqual( - (await content?.text())?.trim(), - ` -{ - "activation_function": "gelu_new", - "architectures": [ - "GPT2LMHeadModel" - ], - "attn_pdrop": 0.1, - "bos_token_id": 50256, - "embd_pdrop": 0.1, - "eos_token_id": 50256, - "initializer_range": 0.02, - "layer_norm_epsilon": 1e-05, - "model_type": "gpt2", - "n_ctx": 1024, - "n_embd": 768, - "n_head": 12, - "n_layer": 12, - "n_positions": 1024, - "resid_pdrop": 0.1, - "summary_activation": null, - "summary_first_dropout": 0.1, - "summary_proj_to_labels": true, - "summary_type": "cls_index", - "summary_use_proj": true, - "task_specific_params": { - "text-generation": { - "do_sample": true, - "max_length": 50 - } - }, - "vocab_size": 50257 -} - `.trim() - ); - } finally { - await deleteRepo({ - repo, - accessToken: TEST_ACCESS_TOKEN, - hubUrl: TEST_HUB_URL, - }); - } - }, 60_000); -}); diff --git a/packages/blob/src/lib/upload-files-with-progress.ts b/packages/blob/src/lib/upload-files-with-progress.ts deleted file mode 100644 index e0e4c9d7f..000000000 --- a/packages/blob/src/lib/upload-files-with-progress.ts +++ /dev/null @@ -1,153 +0,0 @@ -import type { CredentialsParams } from "../types/public"; -import { typedInclude } from "../utils/typedInclude"; -import type { CommitOutput, CommitParams, CommitProgressEvent, ContentSource } from "./commit"; -import { commitIter } from "./commit"; - -const multipartUploadTracking = new WeakMap< - (progress: number) => void, - { - numParts: number; - partsProgress: Record; - } ->(); - -/** - * Uploads with progress - * - * Needs XMLHttpRequest to be available for progress events for uploads - * Set useWebWorkers to true in order to have progress events for hashing - */ -export async function* uploadFilesWithProgress( - params: { - repo: CommitParams["repo"]; - files: Array; - commitTitle?: CommitParams["title"]; - commitDescription?: CommitParams["description"]; - hubUrl?: CommitParams["hubUrl"]; - branch?: CommitParams["branch"]; - isPullRequest?: CommitParams["isPullRequest"]; - parentCommit?: CommitParams["parentCommit"]; - abortSignal?: CommitParams["abortSignal"]; - /** - * Set this to true in order to have progress events for hashing - */ - useWebWorkers?: CommitParams["useWebWorkers"]; - } & Partial -): AsyncGenerator { - return yield* commitIter({ - ...(params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }), - repo: params.repo, - operations: params.files.map((file) => ({ - operation: "addOrUpdate", - path: file instanceof URL ? file.pathname.split("/").at(-1) ?? "file" : "path" in file ? file.path : file.name, - content: "content" in file ? file.content : file, - })), - title: params.commitTitle ?? `Add ${params.files.length} files`, - description: params.commitDescription, - hubUrl: params.hubUrl, - branch: params.branch, - isPullRequest: params.isPullRequest, - parentCommit: params.parentCommit, - useWebWorkers: params.useWebWorkers, - abortSignal: params.abortSignal, - fetch: async (input, init) => { - if (!init) { - return fetch(input); - } - - if ( - !typedInclude(["PUT", "POST"], init.method) || - !("progressHint" in init) || - !init.progressHint || - typeof XMLHttpRequest === "undefined" || - typeof input !== "string" || - (!(init.body instanceof ArrayBuffer) && - !(init.body instanceof Blob) && - !(init.body instanceof File) && - typeof init.body !== "string") - ) { - return fetch(input, init); - } - - const progressHint = init.progressHint as { - progressCallback: (progress: number) => void; - } & (Record | { part: number; numParts: number }); - const progressCallback = progressHint.progressCallback; - - const xhr = new XMLHttpRequest(); - - xhr.upload.addEventListener("progress", (event) => { - if (event.lengthComputable) { - if (progressHint.part !== undefined) { - let tracking = multipartUploadTracking.get(progressCallback); - if (!tracking) { - tracking = { numParts: progressHint.numParts, partsProgress: {} }; - multipartUploadTracking.set(progressCallback, tracking); - } - tracking.partsProgress[progressHint.part] = event.loaded / event.total; - let totalProgress = 0; - for (const partProgress of Object.values(tracking.partsProgress)) { - totalProgress += partProgress; - } - if (totalProgress === tracking.numParts) { - progressCallback(0.9999999999); - } else { - progressCallback(totalProgress / tracking.numParts); - } - } else { - if (event.loaded === event.total) { - progressCallback(0.9999999999); - } else { - progressCallback(event.loaded / event.total); - } - } - } - }); - - xhr.open(init.method, input, true); - - if (init.headers) { - const headers = new Headers(init.headers); - headers.forEach((value, key) => { - xhr.setRequestHeader(key, value); - }); - } - - init.signal?.throwIfAborted(); - xhr.send(init.body); - - return new Promise((resolve, reject) => { - xhr.addEventListener("load", () => { - resolve( - new Response(xhr.responseText, { - status: xhr.status, - statusText: xhr.statusText, - headers: Object.fromEntries( - xhr - .getAllResponseHeaders() - .trim() - .split("\n") - .map((header) => [header.slice(0, header.indexOf(":")), header.slice(header.indexOf(":") + 1).trim()]) - ), - }) - ); - }); - xhr.addEventListener("error", () => { - reject(new Error(xhr.statusText)); - }); - - if (init.signal) { - init.signal.addEventListener("abort", () => { - xhr.abort(); - - try { - init.signal?.throwIfAborted(); - } catch (err) { - reject(err); - } - }); - } - }); - }, - }); -} diff --git a/packages/blob/src/lib/upload-files.spec.ts b/packages/blob/src/lib/upload-files.spec.ts deleted file mode 100644 index 89206c99c..000000000 --- a/packages/blob/src/lib/upload-files.spec.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { assert, it, describe } from "vitest"; - -import { TEST_ACCESS_TOKEN, TEST_HUB_URL, TEST_USER } from "../test/consts"; -import type { RepoId } from "../types/public"; -import { insecureRandomString } from "../utils/insecureRandomString"; -import { createRepo } from "./create-repo"; -import { deleteRepo } from "./delete-repo"; -import { downloadFile } from "./download-file"; -import { uploadFiles } from "./upload-files"; - -describe("uploadFiles", () => { - it("should upload files", async () => { - const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; - const repo = { type: "model", name: repoName } satisfies RepoId; - - try { - const result = await createRepo({ - accessToken: TEST_ACCESS_TOKEN, - repo, - hubUrl: TEST_HUB_URL, - }); - - assert.deepStrictEqual(result, { - repoUrl: `${TEST_HUB_URL}/${repoName}`, - }); - - await uploadFiles({ - accessToken: TEST_ACCESS_TOKEN, - repo, - files: [ - { content: new Blob(["file1"]), path: "file1" }, - new URL("https://huggingface.co/gpt2/raw/main/config.json"), - ], - hubUrl: TEST_HUB_URL, - }); - - let content = await downloadFile({ - repo, - path: "file1", - hubUrl: TEST_HUB_URL, - }); - - assert.strictEqual(await content?.text(), "file1"); - - content = await downloadFile({ - repo, - path: "config.json", - hubUrl: TEST_HUB_URL, - }); - - assert.strictEqual( - (await content?.text())?.trim(), - ` -{ - "activation_function": "gelu_new", - "architectures": [ - "GPT2LMHeadModel" - ], - "attn_pdrop": 0.1, - "bos_token_id": 50256, - "embd_pdrop": 0.1, - "eos_token_id": 50256, - "initializer_range": 0.02, - "layer_norm_epsilon": 1e-05, - "model_type": "gpt2", - "n_ctx": 1024, - "n_embd": 768, - "n_head": 12, - "n_layer": 12, - "n_positions": 1024, - "resid_pdrop": 0.1, - "summary_activation": null, - "summary_first_dropout": 0.1, - "summary_proj_to_labels": true, - "summary_type": "cls_index", - "summary_use_proj": true, - "task_specific_params": { - "text-generation": { - "do_sample": true, - "max_length": 50 - } - }, - "vocab_size": 50257 -} - `.trim() - ); - } finally { - await deleteRepo({ - repo, - accessToken: TEST_ACCESS_TOKEN, - hubUrl: TEST_HUB_URL, - }); - } - }); -}, 10_000); diff --git a/packages/blob/src/lib/upload-files.ts b/packages/blob/src/lib/upload-files.ts deleted file mode 100644 index d205dba97..000000000 --- a/packages/blob/src/lib/upload-files.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { CredentialsParams } from "../types/public"; -import type { CommitOutput, CommitParams, ContentSource } from "./commit"; -import { commit } from "./commit"; - -export function uploadFiles( - params: { - repo: CommitParams["repo"]; - files: Array; - commitTitle?: CommitParams["title"]; - commitDescription?: CommitParams["description"]; - hubUrl?: CommitParams["hubUrl"]; - branch?: CommitParams["branch"]; - isPullRequest?: CommitParams["isPullRequest"]; - parentCommit?: CommitParams["parentCommit"]; - fetch?: CommitParams["fetch"]; - useWebWorkers?: CommitParams["useWebWorkers"]; - abortSignal?: CommitParams["abortSignal"]; - } & Partial -): Promise { - return commit({ - ...(params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }), - repo: params.repo, - operations: params.files.map((file) => ({ - operation: "addOrUpdate", - path: file instanceof URL ? file.pathname.split("/").at(-1) ?? "file" : "path" in file ? file.path : file.name, - content: "content" in file ? file.content : file, - })), - title: params.commitTitle ?? `Add ${params.files.length} files`, - description: params.commitDescription, - hubUrl: params.hubUrl, - branch: params.branch, - isPullRequest: params.isPullRequest, - parentCommit: params.parentCommit, - fetch: params.fetch, - useWebWorkers: params.useWebWorkers, - abortSignal: params.abortSignal, - }); -} diff --git a/packages/blob/src/lib/who-am-i.spec.ts b/packages/blob/src/lib/who-am-i.spec.ts deleted file mode 100644 index 387373923..000000000 --- a/packages/blob/src/lib/who-am-i.spec.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { assert, it, describe } from "vitest"; -import { TEST_ACCESS_TOKEN, TEST_HUB_URL } from "../test/consts"; -import { whoAmI } from "./who-am-i"; - -describe("whoAmI", () => { - it("should fetch identity info", async () => { - const info = await whoAmI({ accessToken: TEST_ACCESS_TOKEN, hubUrl: TEST_HUB_URL }); - - if (info.auth.accessToken?.createdAt instanceof Date) { - info.auth.accessToken.createdAt = new Date(0); - } - - assert.deepStrictEqual(info, { - type: "user", - id: "62f264b9f3c90f4b6514a269", - name: "hub.js", - fullname: "@huggingface/hub CI bot", - email: "eliott@huggingface.co", - emailVerified: true, - canPay: false, - isPro: false, - periodEnd: null, - avatarUrl: "/avatars/934b830e9fdaa879487852f79eef7165.svg", - orgs: [], - auth: { - type: "access_token", - accessToken: { - createdAt: new Date(0), - displayName: "ci-hub.js", - role: "write", - }, - }, - }); - }); -}); diff --git a/packages/blob/src/lib/who-am-i.ts b/packages/blob/src/lib/who-am-i.ts deleted file mode 100644 index 5f4c1845b..000000000 --- a/packages/blob/src/lib/who-am-i.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { HUB_URL } from "../consts"; -import { createApiError } from "../error"; -import type { ApiWhoAmIReponse } from "../types/api/api-who-am-i"; -import type { AccessTokenRole, AuthType, CredentialsParams } from "../types/public"; -import { checkCredentials } from "../utils/checkCredentials"; - -export interface WhoAmIUser { - /** Unique ID persistent across renames */ - id: string; - type: "user"; - email: string; - emailVerified: boolean; - isPro: boolean; - orgs: WhoAmIOrg[]; - name: string; - fullname: string; - canPay: boolean; - avatarUrl: string; - /** - * Unix timestamp in seconds - */ - periodEnd: number | null; -} - -export interface WhoAmIOrg { - /** Unique ID persistent across renames */ - id: string; - type: "org"; - name: string; - fullname: string; - email: string | null; - canPay: boolean; - avatarUrl: string; - /** - * Unix timestamp in seconds - */ - periodEnd: number | null; -} - -export interface WhoAmIApp { - id: string; - type: "app"; - name: string; - scope?: { - entities: string[]; - role: "admin" | "write" | "contributor" | "read"; - }; -} - -export type WhoAmI = WhoAmIApp | WhoAmIOrg | WhoAmIUser; -export interface AuthInfo { - type: AuthType; - accessToken?: { - displayName: string; - role: AccessTokenRole; - createdAt: Date; - }; - expiresAt?: Date; -} - -export async function whoAmI( - params: { - hubUrl?: string; - /** - * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. - */ - fetch?: typeof fetch; - } & CredentialsParams -): Promise { - const accessToken = checkCredentials(params); - - const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/whoami-v2`, { - headers: { - Authorization: `Bearer ${accessToken}`, - }, - }); - - if (!res.ok) { - throw await createApiError(res); - } - - const response: ApiWhoAmIReponse & { - auth: AuthInfo; - } = await res.json(); - - if (typeof response.auth.accessToken?.createdAt === "string") { - response.auth.accessToken.createdAt = new Date(response.auth.accessToken.createdAt); - } - - return response; -} diff --git a/packages/blob/src/test/consts.ts b/packages/blob/src/test/consts.ts deleted file mode 100644 index 297ea01d0..000000000 --- a/packages/blob/src/test/consts.ts +++ /dev/null @@ -1,3 +0,0 @@ -export const TEST_HUB_URL = "https://hub-ci.huggingface.co"; -export const TEST_USER = "hub.js"; -export const TEST_ACCESS_TOKEN = "hf_hub.js"; diff --git a/packages/blob/src/types/api/api-commit.ts b/packages/blob/src/types/api/api-commit.ts deleted file mode 100644 index b5fbfec4b..000000000 --- a/packages/blob/src/types/api/api-commit.ts +++ /dev/null @@ -1,191 +0,0 @@ -export interface ApiLfsBatchRequest { - /// github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md - operation: "download" | "upload"; - transfers?: string[]; - /** - * Optional object describing the server ref that the objects belong to. Note: Added in v2.4. - * - * We use this object for QOL and to fail early for users when they're trying to push to the wrong reference. - * But it does nothing for security. - */ - ref?: { - name: string; - } | null; - objects: { - oid: string; - /** - * Integer byte size of the LFS object. Must be at least zero. - */ - size: number; - }[]; - /** - * The hash algorithm used to name Git LFS objects. Optional; defaults to sha256 if not specified. - * */ - hash_algo?: string; -} - -export interface ApiLfsBatchResponse { - transfer?: ApiLfsResponseTransfer; - objects: ApiLfsResponseObject[]; -} - -export type ApiLfsResponseTransfer = "basic" | "multipart"; - -export interface ApiLfsCompleteMultipartRequest { - oid: string; - parts: { etag: string; partNumber: number }[]; -} - -export interface ApiLfsResponseObject { - /** - * Optional boolean specifying whether the request - * for this specific object is authenticated. - * If omitted or false, Git LFS will attempt to find credentials for this URL. - */ - authenticated?: boolean; - oid: string; - /** - * Integer byte size of the LFS object. Must be at least zero. - */ - size: number; - /** - * Applicable actions depend on which `operation` is specified in the request. - * How these properties are interpreted depends on which transfer adapter - * the client will be using. - */ - actions?: { - /** - * Download operations MUST specify a download action, - * or an object error if the object cannot be downloaded for some reason - */ - download?: ApiLfsAction; - /** - * Upload operations can specify an upload and a verify action. - * The upload action describes how to upload the object. - */ - upload?: ApiLfsAction; - /** - * The LFS client will hit this URL after a successful upload. - * Servers can use this for extra verification, if needed. - */ - verify?: ApiLfsAction; - }; - /** - * If there are problems accessing individual objects, servers should continue - * to return a 200 status code, and provide per-object errors - */ - error?: { - code: number; - message: string; - }; -} - -export interface ApiLfsAction { - href: string; - /** - * Optional hash of String HTTP header key/value pairs to apply to the request - */ - header?: { [key: string]: string } & { chunk_size?: string }; - /** - * Whole number of seconds after local client time when transfer will expire. - * Preferred over `expires_at` if both are provided. - * Maximum of 2147483647, minimum of -2147483647. - */ - expires_in?: number; - /** - * String uppercase RFC 3339-formatted timestamp with second precision - * for when the given action expires (usually due to a temporary token). - */ - expires_at?: string; -} - -export interface ApiPreuploadRequest { - /** - * Optional, otherwise takes the existing content of `.gitattributes` for the revision. - * - * Provide this parameter if you plan to modify `.gitattributes` yourself at the same - * time as uploading LFS files. - * - * Note that this is not needed if you solely rely on automatic LFS detection from HF: the commit endpoint - * will automatically edit the `.gitattributes` file to track the files passed to its `lfsFiles` param. - */ - gitAttributes?: string; - files: Array<{ - /** - * Path of the LFS file - */ - path: string; - /** - * Full size of the LFS file - */ - size: number; - /** - * Base64-encoded sample of the first 512 bytes of the file - */ - sample: string; - }>; -} - -export interface ApiPreuploadResponse { - files: Array<{ - path: string; - uploadMode: "lfs" | "regular"; - }>; -} - -export interface ApiCommitHeader { - summary: string; - description?: string; - /** - * Parent commit. Optional - * - * - When opening a PR: will use parentCommit as the parent commit - * - When committing on a branch: Will make sure that there were no intermediate commits - */ - parentCommit?: string; -} - -export interface ApiCommitDeletedEntry { - path: string; -} - -export interface ApiCommitLfsFile { - path: string; - oldPath?: string; - /** Required if {@link oldPath} is not set */ - algo?: "sha256"; - /** Required if {@link oldPath} is not set */ - oid?: string; - size?: number; -} - -export interface ApiCommitFile { - /** Required if {@link oldPath} is not set */ - content?: string; - path: string; - oldPath?: string; - encoding?: "utf-8" | "base64"; -} - -export type ApiCommitOperation = - | { - key: "file"; - value: ApiCommitFile; - } - | { - key: "lfsFile"; - value: ApiCommitLfsFile; - } - | { - key: "deletedFile"; - value: ApiCommitDeletedEntry; - }; - -export interface ApiCommitData { - id: string; - title: string; - message: string; - authors: Array<{ user: string; avatar: string }>; - date: string; - formatted?: string; -} diff --git a/packages/blob/src/types/api/api-create-repo.ts b/packages/blob/src/types/api/api-create-repo.ts deleted file mode 100644 index f701f2d52..000000000 --- a/packages/blob/src/types/api/api-create-repo.ts +++ /dev/null @@ -1,25 +0,0 @@ -import type { SetRequired } from "../../vendor/type-fest/set-required"; -import type { RepoType, SpaceHardwareFlavor, SpaceSdk } from "../public"; -import type { ApiCommitFile } from "./api-commit"; - -export type ApiCreateRepoPayload = { - name: string; - canonical?: boolean; - license?: string; - template?: string; - organization?: string; - /** @default false */ - private?: boolean; - lfsmultipartthresh?: number; - files?: SetRequired[]; -} & ( - | { - type: Exclude; - } - | { - type: "space"; - hardware?: SpaceHardwareFlavor; - sdk: SpaceSdk; - sdkVersion?: string; - } -); diff --git a/packages/blob/src/types/api/api-dataset.ts b/packages/blob/src/types/api/api-dataset.ts deleted file mode 100644 index 43b097853..000000000 --- a/packages/blob/src/types/api/api-dataset.ts +++ /dev/null @@ -1,89 +0,0 @@ -import type { License } from "../public"; - -export interface ApiDatasetInfo { - _id: string; - id: string; - arxivIds?: string[]; - author?: string; - cardExists?: true; - cardError?: unknown; - cardData?: ApiDatasetMetadata; - contributors?: Array<{ user: string; _id: string }>; - disabled: boolean; - discussionsDisabled: boolean; - gated: false | "auto" | "manual"; - gitalyUid: string; - lastAuthor: { email: string; user?: string }; - lastModified: string; // date - likes: number; - likesRecent: number; - private: boolean; - updatedAt: string; // date - createdAt: string; // date - tags: string[]; - paperswithcode_id?: string; - sha: string; - files?: string[]; - citation?: string; - description?: string; - downloads: number; - downloadsAllTime: number; - previewable?: boolean; - doi?: { id: string; commit: string }; -} - -export interface ApiDatasetMetadata { - licenses?: undefined; - license?: License | License[]; - license_name?: string; - license_link?: "LICENSE" | "LICENSE.md" | string; - license_details?: string; - languages?: undefined; - language?: string | string[]; - language_bcp47?: string[]; - language_details?: string; - tags?: string[]; - task_categories?: string[]; - task_ids?: string[]; - config_names?: string[]; - configs?: { - config_name: string; - data_files?: - | string - | string[] - | { - split: string; - path: string | string[]; - }[]; - data_dir?: string; - }[]; - benchmark?: string; - paperswithcode_id?: string | null; - pretty_name?: string; - viewer?: boolean; - viewer_display_urls?: boolean; - thumbnail?: string | null; - description?: string | null; - annotations_creators?: string[]; - language_creators?: string[]; - multilinguality?: string[]; - size_categories?: string[]; - source_datasets?: string[]; - extra_gated_prompt?: string; - extra_gated_fields?: { - /** - * "text" | "checkbox" | "date_picker" | "country" | "ip_location" | { type: "text" | "checkbox" | "date_picker" | "country" | "ip_location" } | { type: "select", options: Array } Property - */ - [x: string]: - | "text" - | "checkbox" - | "date_picker" - | "country" - | "ip_location" - | { type: "text" | "checkbox" | "date_picker" | "country" | "ip_location" } - | { type: "select"; options: Array }; - }; - extra_gated_heading?: string; - extra_gated_description?: string; - extra_gated_button_content?: string; -} diff --git a/packages/blob/src/types/api/api-index-tree.ts b/packages/blob/src/types/api/api-index-tree.ts deleted file mode 100644 index a467218cf..000000000 --- a/packages/blob/src/types/api/api-index-tree.ts +++ /dev/null @@ -1,46 +0,0 @@ -export interface ApiIndexTreeEntry { - type: "file" | "directory" | "unknown"; - size: number; - path: string; - oid: string; - lfs?: { - oid: string; - size: number; - /** Size of the raw pointer file, 100~200 bytes */ - pointerSize: number; - }; - lastCommit?: { - date: string; - id: string; - title: string; - }; - security?: ApiFileScanResult; -} - -export interface ApiFileScanResult { - /** namespaced by repo type (models/, datasets/, spaces/) */ - repositoryId: string; - blobId: string; - name: string; - safe: boolean; - avScan?: ApiAVScan; - pickleImportScan?: ApiPickleImportScan; -} - -interface ApiAVScan { - virusFound: boolean; - virusNames?: string[]; -} - -type ApiSafetyLevel = "innocuous" | "suspicious" | "dangerous"; - -interface ApiPickleImport { - module: string; - name: string; - safety: ApiSafetyLevel; -} - -interface ApiPickleImportScan { - highestSafetyLevel: ApiSafetyLevel; - imports: ApiPickleImport[]; -} diff --git a/packages/blob/src/types/api/api-model.ts b/packages/blob/src/types/api/api-model.ts deleted file mode 100644 index 589a51ab8..000000000 --- a/packages/blob/src/types/api/api-model.ts +++ /dev/null @@ -1,270 +0,0 @@ -import type { ModelLibraryKey, TransformersInfo } from "@huggingface/tasks"; -import type { License, PipelineType } from "../public"; - -export interface ApiModelInfo { - _id: string; - id: string; - arxivIds: string[]; - author?: string; - cardData?: ApiModelMetadata; - cardError: unknown; - cardExists?: true; - config: unknown; - contributors: Array<{ user: string; _id: string }>; - disabled: boolean; - discussionsDisabled: boolean; - doi?: { id: string; commit: string }; - downloads: number; - downloadsAllTime: number; - files: string[]; - gitalyUid: string; - lastAuthor: { email: string; user?: string }; - lastModified: string; // convert to date - library_name?: ModelLibraryKey; - likes: number; - likesRecent: number; - private: boolean; - gated: false | "auto" | "manual"; - sha: string; - spaces: string[]; - updatedAt: string; // convert to date - createdAt: string; // convert to date - pipeline_tag: PipelineType; - tags: string[]; - "model-index": unknown; - safetensors?: { - parameters: Record; - total: number; - }; - transformersInfo?: TransformersInfo; -} - -export interface ApiModelIndex { - name: string; - results: { - task: { - /** - * Example: automatic-speech-recognition -Use task id from https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/tasksData.ts - */ - type: string; - /** - * Example: Speech Recognition - */ - name?: string; - }; - /** - * This will switch to required at some point. -in any case, we need them to link to PWC - */ - dataset?: { - /** - * Example: common_voice. Use dataset id from https://hf.co/datasets - */ - type: string; - /** - * A pretty name for the dataset. Example: Common Voice zh-CN -Also encode config params into the name if relevant. - */ - name: string; - /** - * Optional. The name of the dataset configuration used in `load_dataset()` - */ - config?: string; - /** - * Optional. Example: test - */ - split?: string; - /** - * Optional. Example: 5503434ddd753f426f4b38109466949a1217c2bb - */ - revision?: string; - args?: - | string - | { - /** - * String Property - */ - [x: string]: string; - }; - }; - metrics: { - /** - * Example: wer. Use metric id from https://hf.co/metrics - */ - type: string; - /** - * Required. Example: 20.0 or "20.0 ± 1.2" - */ - value: unknown; - /** - * Example: Test WER - */ - name?: string; - /** - * Optional. The name of the metric configuration used in `load_metric()`. - */ - config?: string; - args?: - | string - | { - /** - * String Property - */ - [x: string]: string; - }; - /** - * [Automatically computed, do not set] Dynamically overriden by huggingface in API calls to indicate if it was verified by Hugging Face. - */ - verified?: boolean; - /** - * Generated by Hugging Face to prove the results are valid. - */ - verifyToken?: string; - }[]; - /** - * The source for this evaluation result. - */ - source?: { - /** - * Example: Open LLM Leaderboard - */ - name?: string; - /** - * Example: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard - */ - url: string; - }; - }[]; -} - -export interface ApiWidgetExampleFromModelcard { - example_title?: string; - group?: string; - text?: string; - src?: string; - table?: { - /** - * (string | number)[] Property - */ - [x: string]: (string | number)[]; - }; - structured_data?: { - /** - * (string | number)[] Property - */ - [x: string]: (string | number)[]; - }; - candidate_labels?: string; - messages?: { - role: "system" | "user" | "assistant"; - content: string; - }[]; - multi_class?: boolean; - source_sentence?: string; - sentences?: string[]; - parameters?: { - aggregation_strategy?: string; - top_k?: number; - top_p?: number; - temperature?: number; - max_new_tokens?: number; - do_sample?: boolean; - negative_prompt?: string; - guidance_scale?: number; - num_inference_steps?: number; - }; - output?: - | { - label: string; - score: number; - }[] - | { - answer: string; - score: number; - } - | { - text: string; - } - | { - url: string; - }; -} - -export interface ApiModelMetadata { - datasets?: string | string[]; - license?: License | License[]; - license_name?: string; - license_link?: "LICENSE" | "LICENSE.md" | string; - license_details?: string; - inference?: - | boolean - | { - parameters?: { - aggregation_strategy?: string; - top_k?: number; - top_p?: number; - temperature?: number; - max_new_tokens?: number; - do_sample?: boolean; - negative_prompt?: string; - guidance_scale?: number; - num_inference_steps?: number; - }; - }; - language?: string | string[]; - language_bcp47?: string[]; - language_details?: string; - tags?: string[]; - pipeline_tag?: string; - co2_eq_emissions?: - | number - | { - /** - * Emissions in grams of CO2 - */ - emissions: number; - /** - * source of the information, either directly from AutoTrain, code carbon or from a scientific article documenting the model - */ - source?: string; - /** - * pre-training or fine-tuning - */ - training_type?: string; - /** - * as granular as possible, for instance Quebec, Canada or Brooklyn, NY, USA - */ - geographical_location?: string; - /** - * how much compute and what kind, e.g. 8 v100 GPUs - */ - hardware_used?: string; - }; - library_name?: string; - thumbnail?: string | null; - description?: string | null; - mask_token?: string; - widget?: ApiWidgetExampleFromModelcard[]; - "model-index"?: ApiModelIndex[]; - finetuned_from?: string; - base_model?: string | string[]; - instance_prompt?: string | null; - extra_gated_prompt?: string; - extra_gated_fields?: { - /** - * "text" | "checkbox" | "date_picker" | "country" | "ip_location" | { type: "text" | "checkbox" | "date_picker" | "country" | "ip_location" } | { type: "select", options: Array } Property - */ - [x: string]: - | "text" - | "checkbox" - | "date_picker" - | "country" - | "ip_location" - | { type: "text" | "checkbox" | "date_picker" | "country" | "ip_location" } - | { type: "select"; options: Array }; - }; - extra_gated_heading?: string; - extra_gated_description?: string; - extra_gated_button_content?: string; -} diff --git a/packages/blob/src/types/api/api-space.ts b/packages/blob/src/types/api/api-space.ts deleted file mode 100644 index 151677f4b..000000000 --- a/packages/blob/src/types/api/api-space.ts +++ /dev/null @@ -1,93 +0,0 @@ -import type { License, SpaceRuntime, SpaceSdk } from "../public"; - -type Color = "red" | "yellow" | "green" | "blue" | "indigo" | "purple" | "pink" | "gray"; - -export interface ApiSpaceInfo { - _id: string; - id: string; - arxivIds?: string[]; - author: string; - cardExists?: true; - cardError?: unknown; - cardData?: unknown; - contributors?: Array<{ user: string; _id: string }>; - disabled: boolean; - discussionsDisabled: boolean; - duplicationDisabled: boolean; - gated: false | "auto" | "manual"; - gitalyUid: string; - lastAuthor: { email: string; user?: string }; - lastModified: string; // date - likes: number; - likesRecent: number; - private: boolean; - updatedAt: string; // date - createdAt: string; // date - tags: string[]; - sha: string; - subdomain: string; - title: string; - emoji: string; - colorFrom: Color; - colorTo: Color; - pinned: boolean; - siblings: Array<{ rfilename: string }>; - sdk?: SpaceSdk; - runtime?: SpaceRuntime; - models?: string[]; - datasets?: string[]; - originSpace?: { _id: string; authorId: string }; -} - -export interface ApiSpaceMetadata { - license?: License | License[]; - tags?: string[]; - title?: string; - colorFrom?: "red" | "yellow" | "green" | "blue" | "indigo" | "purple" | "pink" | "gray"; - colorTo?: "red" | "yellow" | "green" | "blue" | "indigo" | "purple" | "pink" | "gray"; - emoji?: string; - sdk?: "streamlit" | "gradio" | "docker" | "static"; - sdk_version?: string | string; - python_version?: string | string; - fullWidth?: boolean; - header?: "mini" | "default"; - app_file?: string; - app_port?: number; - base_path?: string; - models?: string[]; - datasets?: string[]; - pinned?: boolean; - metaTitle?: string; - description?: string; - thumbnail?: string; - /** - * If enabled, will associate an oauth app to the Space, adding variables and secrets to the Space's environment - */ - hf_oauth?: boolean; - /** - * The expiration of access tokens for your oauth app in minutes. max 30 days (43,200 minutes). Defaults to 8 hours (480 minutes) - */ - hf_oauth_expiration_minutes?: number; - /** - * OAuth scopes to request. By default you have access to the user's profile, you can request access to their repos or inference-api. - */ - hf_oauth_scopes?: ("email" | "read-repos" | "write-repos" | "manage-repos" | "inference-api")[]; - suggested_hardware?: - | "cpu-basic" - | "zero-a10g" - | "cpu-upgrade" - | "cpu-xl" - | "t4-small" - | "t4-medium" - | "a10g-small" - | "a10g-large" - | "a10g-largex2" - | "a10g-largex4" - | "a100-large"; - suggested_storage?: "small" | "medium" | "large"; - custom_headers?: { - "cross-origin-embedder-policy"?: "unsafe-none" | "require-corp" | "credentialless"; - "cross-origin-opener-policy"?: "same-origin" | "same-origin-allow-popups" | "unsafe-none"; - "cross-origin-resource-policy"?: "same-site" | "same-origin" | "cross-origin"; - }; -} diff --git a/packages/blob/src/types/api/api-who-am-i.ts b/packages/blob/src/types/api/api-who-am-i.ts deleted file mode 100644 index 1cb75c211..000000000 --- a/packages/blob/src/types/api/api-who-am-i.ts +++ /dev/null @@ -1,51 +0,0 @@ -import type { AccessTokenRole, AuthType } from "../public"; - -interface ApiWhoAmIBase { - /** Unique ID persistent across renames */ - id: string; - type: "user" | "org" | "app"; - name: string; -} - -interface ApiWhoAmIEntityBase extends ApiWhoAmIBase { - fullname: string; - email: string | null; - canPay: boolean; - avatarUrl: string; - /** - * Unix timestamp in seconds - */ - periodEnd: number | null; -} - -interface ApiWhoAmIOrg extends ApiWhoAmIEntityBase { - type: "org"; -} - -interface ApiWhoAmIUser extends ApiWhoAmIEntityBase { - type: "user"; - email: string; - emailVerified: boolean; - isPro: boolean; - orgs: ApiWhoAmIOrg[]; -} - -interface ApiWhoAmIApp extends ApiWhoAmIBase { - type: "app"; - name: string; - scope?: { - entities: string[]; - role: AccessTokenRole; - }; -} - -export type ApiWhoAmIReponse = ApiWhoAmIUser | ApiWhoAmIOrg | ApiWhoAmIApp; - -export interface ApiWhoAmIAuthInfo { - type: AuthType; - accessToken?: { - displayName: string; - expiration?: string; - role: AccessTokenRole; - }; -} diff --git a/packages/blob/src/types/public.ts b/packages/blob/src/types/public.ts deleted file mode 100644 index c348cdae4..000000000 --- a/packages/blob/src/types/public.ts +++ /dev/null @@ -1,181 +0,0 @@ -import type { PipelineType } from "@huggingface/tasks"; - -export type RepoType = "space" | "dataset" | "model"; - -export interface RepoId { - name: string; - type: RepoType; -} - -export type RepoFullName = string | `spaces/${string}` | `datasets/${string}`; - -export type RepoDesignation = RepoId | RepoFullName; - -/** Actually `hf_${string}`, but for convenience, using the string type */ -export type AccessToken = string; - -/** - * @deprecated Use `AccessToken` instead. Pass { accessToken: "hf_..." } instead of { credentials: { accessToken: "hf_..." } } - */ -export interface Credentials { - accessToken: AccessToken; -} - -export type CredentialsParams = - | { - accessToken?: undefined; - /** - * @deprecated Use `accessToken` instead - */ - credentials: Credentials; - } - | { - accessToken: AccessToken; - /** - * @deprecated Use `accessToken` instead - */ - credentials?: undefined; - }; - -export type SpaceHardwareFlavor = - | "cpu-basic" - | "cpu-upgrade" - | "t4-small" - | "t4-medium" - | "l4x1" - | "l4x4" - | "a10g-small" - | "a10g-large" - | "a10g-largex2" - | "a10g-largex4" - | "a100-large" - | "v5e-1x1" - | "v5e-2x2" - | "v5e-2x4"; - -export type SpaceSdk = "streamlit" | "gradio" | "docker" | "static"; - -export type SpaceStage = - | "NO_APP_FILE" - | "CONFIG_ERROR" - | "BUILDING" - | "BUILD_ERROR" - | "RUNNING" - | "RUNNING_BUILDING" - | "RUNTIME_ERROR" - | "DELETING" - | "PAUSED" - | "SLEEPING"; - -export type AccessTokenRole = "admin" | "write" | "contributor" | "read"; - -export type AuthType = "access_token" | "app_token" | "app_token_as_user"; - -export type { PipelineType }; - -export interface SpaceRuntime { - stage: SpaceStage; - sdk?: SpaceSdk; - sdkVersion?: string; - errorMessage?: string; - hardware?: { - current: SpaceHardwareFlavor | null; - currentPrettyName?: string; - requested: SpaceHardwareFlavor | null; - requestedPrettyName?: string; - }; - /** when calling /spaces, those props are only fetched if ?full=true */ - resources?: SpaceResourceConfig; - /** in seconds */ - gcTimeout?: number | null; -} - -export interface SpaceResourceRequirement { - cpu?: string; - memory?: string; - gpu?: string; - gpuModel?: string; - ephemeral?: string; -} - -export interface SpaceResourceConfig { - requests: SpaceResourceRequirement; - limits: SpaceResourceRequirement; - replicas?: number; - throttled?: boolean; - is_custom?: boolean; -} - -export type License = - | "apache-2.0" - | "mit" - | "openrail" - | "bigscience-openrail-m" - | "creativeml-openrail-m" - | "bigscience-bloom-rail-1.0" - | "bigcode-openrail-m" - | "afl-3.0" - | "artistic-2.0" - | "bsl-1.0" - | "bsd" - | "bsd-2-clause" - | "bsd-3-clause" - | "bsd-3-clause-clear" - | "c-uda" - | "cc" - | "cc0-1.0" - | "cc-by-2.0" - | "cc-by-2.5" - | "cc-by-3.0" - | "cc-by-4.0" - | "cc-by-sa-3.0" - | "cc-by-sa-4.0" - | "cc-by-nc-2.0" - | "cc-by-nc-3.0" - | "cc-by-nc-4.0" - | "cc-by-nd-4.0" - | "cc-by-nc-nd-3.0" - | "cc-by-nc-nd-4.0" - | "cc-by-nc-sa-2.0" - | "cc-by-nc-sa-3.0" - | "cc-by-nc-sa-4.0" - | "cdla-sharing-1.0" - | "cdla-permissive-1.0" - | "cdla-permissive-2.0" - | "wtfpl" - | "ecl-2.0" - | "epl-1.0" - | "epl-2.0" - | "etalab-2.0" - | "eupl-1.1" - | "agpl-3.0" - | "gfdl" - | "gpl" - | "gpl-2.0" - | "gpl-3.0" - | "lgpl" - | "lgpl-2.1" - | "lgpl-3.0" - | "isc" - | "lppl-1.3c" - | "ms-pl" - | "mpl-2.0" - | "odc-by" - | "odbl" - | "openrail++" - | "osl-3.0" - | "postgresql" - | "ofl-1.1" - | "ncsa" - | "unlicense" - | "zlib" - | "pddl" - | "lgpl-lr" - | "deepfloyd-if-license" - | "llama2" - | "llama3" - | "llama3.1" - | "llama3.2" - | "gemma" - | "unknown" - | "other"; diff --git a/packages/blob/src/utils/checkCredentials.ts b/packages/blob/src/utils/checkCredentials.ts deleted file mode 100644 index 0e1717054..000000000 --- a/packages/blob/src/utils/checkCredentials.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { CredentialsParams } from "../types/public"; - -export function checkAccessToken(accessToken: string): void { - if (!accessToken.startsWith("hf_")) { - throw new TypeError("Your access token must start with 'hf_'"); - } -} - -export function checkCredentials(params: Partial): string | undefined { - if (params.accessToken) { - checkAccessToken(params.accessToken); - return params.accessToken; - } - if (params.credentials?.accessToken) { - checkAccessToken(params.credentials.accessToken); - return params.credentials.accessToken; - } -} diff --git a/packages/blob/src/utils/omit.ts b/packages/blob/src/utils/omit.ts deleted file mode 100644 index 8743dba87..000000000 --- a/packages/blob/src/utils/omit.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { pick } from "./pick"; -import { typedInclude } from "./typedInclude"; - -/** - * Return copy of object, omitting blacklisted array of props - */ -export function omit, K extends keyof T>( - o: T, - props: K[] | K -): Pick> { - const propsArr = Array.isArray(props) ? props : [props]; - const letsKeep = (Object.keys(o) as (keyof T)[]).filter((prop) => !typedInclude(propsArr, prop)); - return pick(o, letsKeep); -} diff --git a/packages/blob/src/utils/parseLinkHeader.ts b/packages/blob/src/utils/parseLinkHeader.ts deleted file mode 100644 index 6939a89be..000000000 --- a/packages/blob/src/utils/parseLinkHeader.ts +++ /dev/null @@ -1,8 +0,0 @@ -/** - * Parse Link HTTP header, eg `; rel="next"` - */ -export function parseLinkHeader(header: string): Record { - const regex = /<(https?:[/][/][^>]+)>;\s+rel="([^"]+)"/g; - - return Object.fromEntries([...header.matchAll(regex)].map(([, url, rel]) => [rel, url])); -} diff --git a/packages/blob/src/utils/pick.ts b/packages/blob/src/utils/pick.ts deleted file mode 100644 index bd32e4532..000000000 --- a/packages/blob/src/utils/pick.ts +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Return copy of object, only keeping whitelisted properties. - */ -export function pick(o: T, props: K[] | ReadonlyArray): Pick { - return Object.assign( - {}, - ...props.map((prop) => { - if (o[prop] !== undefined) { - return { [prop]: o[prop] }; - } - }) - ); -} diff --git a/packages/blob/src/utils/sha256-node.ts b/packages/blob/src/utils/sha256-node.ts deleted file mode 100644 index b068d1a21..000000000 --- a/packages/blob/src/utils/sha256-node.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { Readable } from "node:stream"; -import type { ReadableStream } from "node:stream/web"; -import { createHash } from "node:crypto"; - -export async function* sha256Node( - buffer: ArrayBuffer | Blob, - opts?: { - abortSignal?: AbortSignal; - } -): AsyncGenerator { - const sha256Stream = createHash("sha256"); - const size = buffer instanceof Blob ? buffer.size : buffer.byteLength; - let done = 0; - const readable = - buffer instanceof Blob ? Readable.fromWeb(buffer.stream() as ReadableStream) : Readable.from(Buffer.from(buffer)); - - for await (const buffer of readable) { - sha256Stream.update(buffer); - done += buffer.length; - yield done / size; - - opts?.abortSignal?.throwIfAborted(); - } - - return sha256Stream.digest("hex"); -} diff --git a/packages/blob/src/utils/sha256.spec.ts b/packages/blob/src/utils/sha256.spec.ts deleted file mode 100644 index 8e62b936b..000000000 --- a/packages/blob/src/utils/sha256.spec.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { sha256 } from "./sha256"; - -const smallContent = "hello world"; -const smallContentSHA256 = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"; -const bigContent = "O123456789".repeat(100_000); -const bigContentSHA256 = "a3bbce7ee1df7233d85b5f4d60faa3755f93f537804f8b540c72b0739239ddf8"; -const biggerContent = "0123456789".repeat(1_000_000); -const biggerContentSHA256 = "d52fcc26b48dbd4d79b125eb0a29b803ade07613c67ac7c6f2751aefef008486"; - -describe("sha256", () => { - async function calcSHA256(content: string, useWebWorker: boolean) { - const iterator = sha256(new Blob([content]), { useWebWorker }); - let res: IteratorResult; - do { - res = await iterator.next(); - } while (!res.done); - return res.value; - } - - it("Calculate hash of a small file", async () => { - const sha = await calcSHA256(smallContent, false); - expect(sha).toBe(smallContentSHA256); - }); - - it("Calculate hash of a big file", async () => { - const sha = await calcSHA256(bigContent, false); - expect(sha).toBe(bigContentSHA256); - }); - - it("Calculate hash of a bigger file", async () => { - const sha = await calcSHA256(biggerContent, false); - expect(sha).toBe(biggerContentSHA256); - }); - - it("Calculate hash of a small file (+ web worker)", async () => { - const sha = await calcSHA256(smallContent, true); - expect(sha).toBe(smallContentSHA256); - }); - - it("Calculate hash of a big file (+ web worker)", async () => { - const sha = await calcSHA256(bigContent, true); - expect(sha).toBe(bigContentSHA256); - }); - - it("Calculate hash of a bigger file (+ web worker)", async () => { - const sha = await calcSHA256(biggerContent, true); - expect(sha).toBe(biggerContentSHA256); - }); -}); diff --git a/packages/blob/src/utils/sha256.ts b/packages/blob/src/utils/sha256.ts deleted file mode 100644 index d432909b3..000000000 --- a/packages/blob/src/utils/sha256.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { eventToGenerator } from "./eventToGenerator"; -import { hexFromBytes } from "./hexFromBytes"; -import { isFrontend } from "./isFrontend"; - -async function getWebWorkerCode() { - const sha256Module = await import("../vendor/hash-wasm/sha256-wrapper"); - return URL.createObjectURL(new Blob([sha256Module.createSHA256WorkerCode()])); -} - -const pendingWorkers: Worker[] = []; -const runningWorkers: Set = new Set(); - -let resolve: () => void; -let waitPromise: Promise = new Promise((r) => { - resolve = r; -}); - -async function getWorker(poolSize?: number): Promise { - { - const worker = pendingWorkers.pop(); - if (worker) { - runningWorkers.add(worker); - return worker; - } - } - if (!poolSize) { - const worker = new Worker(await getWebWorkerCode()); - runningWorkers.add(worker); - return worker; - } - - if (poolSize <= 0) { - throw new TypeError("Invalid webworker pool size: " + poolSize); - } - - while (runningWorkers.size >= poolSize) { - await waitPromise; - } - - const worker = new Worker(await getWebWorkerCode()); - runningWorkers.add(worker); - return worker; -} - -async function freeWorker(worker: Worker, poolSize: number | undefined): Promise { - if (!poolSize) { - return destroyWorker(worker); - } - runningWorkers.delete(worker); - pendingWorkers.push(worker); - const r = resolve; - waitPromise = new Promise((r) => { - resolve = r; - }); - r(); -} - -function destroyWorker(worker: Worker): void { - runningWorkers.delete(worker); - worker.terminate(); - const r = resolve; - waitPromise = new Promise((r) => { - resolve = r; - }); - r(); -} - -/** - * @returns hex-encoded sha - * @yields progress (0-1) - */ -export async function* sha256( - buffer: Blob, - opts?: { useWebWorker?: boolean | { minSize?: number; poolSize?: number }; abortSignal?: AbortSignal } -): AsyncGenerator { - yield 0; - - const maxCryptoSize = - typeof opts?.useWebWorker === "object" && opts?.useWebWorker.minSize !== undefined - ? opts.useWebWorker.minSize - : 10_000_000; - if (buffer.size < maxCryptoSize && globalThis.crypto?.subtle) { - const res = hexFromBytes( - new Uint8Array( - await globalThis.crypto.subtle.digest("SHA-256", buffer instanceof Blob ? await buffer.arrayBuffer() : buffer) - ) - ); - - yield 1; - - return res; - } - - if (isFrontend) { - if (opts?.useWebWorker) { - try { - const poolSize = typeof opts?.useWebWorker === "object" ? opts.useWebWorker.poolSize : undefined; - const worker = await getWorker(poolSize); - return yield* eventToGenerator((yieldCallback, returnCallback, rejectCallack) => { - worker.addEventListener("message", (event) => { - if (event.data.sha256) { - freeWorker(worker, poolSize); - returnCallback(event.data.sha256); - } else if (event.data.progress) { - yieldCallback(event.data.progress); - - try { - opts.abortSignal?.throwIfAborted(); - } catch (err) { - destroyWorker(worker); - rejectCallack(err); - } - } else { - destroyWorker(worker); - rejectCallack(event); - } - }); - worker.addEventListener("error", (event) => { - destroyWorker(worker); - rejectCallack(event.error); - }); - worker.postMessage({ file: buffer }); - }); - } catch (err) { - console.warn("Failed to use web worker for sha256", err); - } - } - if (!wasmModule) { - wasmModule = await import("../vendor/hash-wasm/sha256-wrapper"); - } - - const sha256 = await wasmModule.createSHA256(); - sha256.init(); - - const reader = buffer.stream().getReader(); - const total = buffer.size; - let bytesDone = 0; - - while (true) { - const { done, value } = await reader.read(); - - if (done) { - break; - } - - sha256.update(value); - bytesDone += value.length; - yield bytesDone / total; - - opts?.abortSignal?.throwIfAborted(); - } - - return sha256.digest("hex"); - } - - if (!cryptoModule) { - cryptoModule = await import("./sha256-node"); - } - - return yield* cryptoModule.sha256Node(buffer, { abortSignal: opts?.abortSignal }); -} - -// eslint-disable-next-line @typescript-eslint/consistent-type-imports -let cryptoModule: typeof import("./sha256-node"); -// eslint-disable-next-line @typescript-eslint/consistent-type-imports -let wasmModule: typeof import("../vendor/hash-wasm/sha256-wrapper"); diff --git a/packages/blob/src/utils/sum.ts b/packages/blob/src/utils/sum.ts deleted file mode 100644 index 9d3fe6f15..000000000 --- a/packages/blob/src/utils/sum.ts +++ /dev/null @@ -1,6 +0,0 @@ -/** - * Sum of elements in array - */ -export function sum(arr: number[]): number { - return arr.reduce((a, b) => a + b, 0); -} diff --git a/packages/blob/src/utils/toRepoId.ts b/packages/blob/src/utils/toRepoId.ts deleted file mode 100644 index 927326692..000000000 --- a/packages/blob/src/utils/toRepoId.ts +++ /dev/null @@ -1,54 +0,0 @@ -import type { RepoDesignation, RepoId } from "../types/public"; - -export function toRepoId(repo: RepoDesignation): RepoId { - if (typeof repo !== "string") { - return repo; - } - - if (repo.startsWith("model/") || repo.startsWith("models/")) { - throw new TypeError( - "A repo designation for a model should not start with 'models/', directly specify the model namespace / name" - ); - } - - if (repo.startsWith("space/")) { - throw new TypeError("Spaces should start with 'spaces/', plural, not 'space/'"); - } - - if (repo.startsWith("dataset/")) { - throw new TypeError("Datasets should start with 'dataset/', plural, not 'dataset/'"); - } - - const slashes = repo.split("/").length - 1; - - if (repo.startsWith("spaces/")) { - if (slashes !== 2) { - throw new TypeError("Space Id must include namespace and name of the space"); - } - - return { - type: "space", - name: repo.slice("spaces/".length), - }; - } - - if (repo.startsWith("datasets/")) { - if (slashes > 2) { - throw new TypeError("Too many slashes in repo designation: " + repo); - } - - return { - type: "dataset", - name: repo.slice("datasets/".length), - }; - } - - if (slashes > 1) { - throw new TypeError("Too many slashes in repo designation: " + repo); - } - - return { - type: "model", - name: repo, - }; -} diff --git a/packages/blob/src/utils/typedEntries.ts b/packages/blob/src/utils/typedEntries.ts deleted file mode 100644 index 031ba7daa..000000000 --- a/packages/blob/src/utils/typedEntries.ts +++ /dev/null @@ -1,5 +0,0 @@ -import type { Entries } from "../vendor/type-fest/entries"; - -export function typedEntries>(obj: T): Entries { - return Object.entries(obj) as Entries; -} diff --git a/packages/blob/src/utils/typedInclude.ts b/packages/blob/src/utils/typedInclude.ts deleted file mode 100644 index 71e2f7a7e..000000000 --- a/packages/blob/src/utils/typedInclude.ts +++ /dev/null @@ -1,3 +0,0 @@ -export function typedInclude(arr: readonly T[], v: V): v is T { - return arr.includes(v as T); -} diff --git a/packages/blob/src/vendor/hash-wasm/build.sh b/packages/blob/src/vendor/hash-wasm/build.sh deleted file mode 100755 index 18424dfbc..000000000 --- a/packages/blob/src/vendor/hash-wasm/build.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -CURRENT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -cd $CURRENT_PATH - -# Clean up -docker kill hash-wasm-builder -docker rm hash-wasm-builder - -# Start container -docker run -it -d --name hash-wasm-builder emscripten/emsdk:3.1.55 bash - -# Copy & compile -docker exec hash-wasm-builder bash -c "mkdir /source" -docker cp ./sha256.c hash-wasm-builder:/source -docker exec hash-wasm-builder bash -c "\ - cd /source && \ - emcc sha256.c -o sha256.js -msimd128 -sSINGLE_FILE -sMODULARIZE=1 -sENVIRONMENT=web,worker -sEXPORTED_FUNCTIONS=_Hash_Init,_Hash_Update,_Hash_Final,_GetBufferPtr -sFILESYSTEM=0 -fno-rtti -fno-exceptions -O1 -sMODULARIZE=1 -sEXPORT_ES6=1 \ - " -# Patch "_scriptDir" variable -docker exec hash-wasm-builder bash -c "\ - cd /source && \ - sed -i 's\var _scriptDir\var _unused\g' ./sha256.js && \ - sed -i 's\_scriptDir\false\g' ./sha256.js \ - " - -# Copy back compiled file -docker cp hash-wasm-builder:/source/sha256.js . - - -# Clean up -docker kill hash-wasm-builder -docker rm hash-wasm-builder diff --git a/packages/blob/src/vendor/hash-wasm/sha256-wrapper.ts b/packages/blob/src/vendor/hash-wasm/sha256-wrapper.ts deleted file mode 100644 index c6ad75b36..000000000 --- a/packages/blob/src/vendor/hash-wasm/sha256-wrapper.ts +++ /dev/null @@ -1,62 +0,0 @@ -import WasmModule from "./sha256"; - -export async function createSHA256(isInsideWorker = false): Promise<{ - init(): void; - update(data: Uint8Array): void; - digest(method: "hex"): string; -}> { - const BUFFER_MAX_SIZE = 8 * 1024 * 1024; - const wasm: Awaited> = isInsideWorker - ? // @ts-expect-error WasmModule will be populated inside self object - await self["SHA256WasmModule"]() - : await WasmModule(); - const heap = wasm.HEAPU8.subarray(wasm._GetBufferPtr()); - return { - init() { - wasm._Hash_Init(256); - }, - update(data: Uint8Array) { - let byteUsed = 0; - while (byteUsed < data.byteLength) { - const bytesLeft = data.byteLength - byteUsed; - const length = Math.min(bytesLeft, BUFFER_MAX_SIZE); - heap.set(data.subarray(byteUsed, byteUsed + length)); - wasm._Hash_Update(length); - byteUsed += length; - } - }, - digest(method: "hex") { - if (method !== "hex") { - throw new Error("Only digest hex is supported"); - } - wasm._Hash_Final(); - const result = Array.from(heap.slice(0, 32)); - return result.map((b) => b.toString(16).padStart(2, "0")).join(""); - }, - }; -} - -export function createSHA256WorkerCode(): string { - return ` - self.addEventListener('message', async (event) => { - const { file } = event.data; - const sha256 = await self.createSHA256(true); - sha256.init(); - const reader = file.stream().getReader(); - const total = file.size; - let bytesDone = 0; - while (true) { - const { done, value } = await reader.read(); - if (done) { - break; - } - sha256.update(value); - bytesDone += value.length; - postMessage({ progress: bytesDone / total }); - } - postMessage({ sha256: sha256.digest('hex') }); - }); - self.SHA256WasmModule = ${WasmModule.toString()}; - self.createSHA256 = ${createSHA256.toString()}; - `; -} diff --git a/packages/blob/src/vendor/hash-wasm/sha256.c b/packages/blob/src/vendor/hash-wasm/sha256.c deleted file mode 100644 index b8c0cb7ea..000000000 --- a/packages/blob/src/vendor/hash-wasm/sha256.c +++ /dev/null @@ -1,432 +0,0 @@ -/* sha256.c - an implementation of SHA-256/224 hash functions - * based on FIPS 180-3 (Federal Information Processing Standart). - * - * Copyright (c) 2010, Aleksey Kravchenko - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, - * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM - * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE - * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR - * PERFORMANCE OF THIS SOFTWARE. - - * Modified for hash-wasm by Dani Biró - */ - -#define WITH_BUFFER - - -////////////////////////////////////////////////////////////////////////// - -#include -#include - -#ifndef NULL -#define NULL 0 -#endif - -#ifdef _MSC_VER -#define WASM_EXPORT -#define __inline__ -#else -#define WASM_EXPORT __attribute__((visibility("default"))) -#endif - -#ifdef WITH_BUFFER - -#define MAIN_BUFFER_SIZE 8 * 1024 * 1024 -alignas(128) uint8_t main_buffer[MAIN_BUFFER_SIZE]; - -WASM_EXPORT -uint8_t *Hash_GetBuffer() { - return main_buffer; -} - -#endif - -// Sometimes LLVM emits these functions during the optimization step -// even with -nostdlib -fno-builtin flags -static __inline__ void* memcpy(void* dst, const void* src, uint32_t cnt) { - uint8_t *destination = dst; - const uint8_t *source = src; - while (cnt) { - *(destination++)= *(source++); - --cnt; - } - return dst; -} - -static __inline__ void* memset(void* dst, const uint8_t value, uint32_t cnt) { - uint8_t *p = dst; - while (cnt--) { - *p++ = value; - } - return dst; -} - -static __inline__ void* memcpy2(void* dst, const void* src, uint32_t cnt) { - uint64_t *destination64 = dst; - const uint64_t *source64 = src; - while (cnt >= 8) { - *(destination64++)= *(source64++); - cnt -= 8; - } - - uint8_t *destination = (uint8_t*)destination64; - const uint8_t *source = (uint8_t*)source64; - while (cnt) { - *(destination++)= *(source++); - --cnt; - } - return dst; -} - -static __inline__ void memcpy16(void* dst, const void* src) { - uint64_t* dst64 = (uint64_t*)dst; - uint64_t* src64 = (uint64_t*)src; - - dst64[0] = src64[0]; - dst64[1] = src64[1]; -} - -static __inline__ void memcpy32(void* dst, const void* src) { - uint64_t* dst64 = (uint64_t*)dst; - uint64_t* src64 = (uint64_t*)src; - - #pragma clang loop unroll(full) - for (int i = 0; i < 4; i++) { - dst64[i] = src64[i]; - } -} - -static __inline__ void memcpy64(void* dst, const void* src) { - uint64_t* dst64 = (uint64_t*)dst; - uint64_t* src64 = (uint64_t*)src; - - #pragma clang loop unroll(full) - for (int i = 0; i < 8; i++) { - dst64[i] = src64[i]; - } -} - -static __inline__ uint64_t widen8to64(const uint8_t value) { - return value | (value << 8) | (value << 16) | (value << 24); -} - -static __inline__ void memset16(void* dst, const uint8_t value) { - uint64_t val = widen8to64(value); - uint64_t* dst64 = (uint64_t*)dst; - - dst64[0] = val; - dst64[1] = val; -} - -static __inline__ void memset32(void* dst, const uint8_t value) { - uint64_t val = widen8to64(value); - uint64_t* dst64 = (uint64_t*)dst; - - #pragma clang loop unroll(full) - for (int i = 0; i < 4; i++) { - dst64[i] = val; - } -} - -static __inline__ void memset64(void* dst, const uint8_t value) { - uint64_t val = widen8to64(value); - uint64_t* dst64 = (uint64_t*)dst; - - #pragma clang loop unroll(full) - for (int i = 0; i < 8; i++) { - dst64[i] = val; - } -} - -static __inline__ void memset128(void* dst, const uint8_t value) { - uint64_t val = widen8to64(value); - uint64_t* dst64 = (uint64_t*)dst; - - #pragma clang loop unroll(full) - for (int i = 0; i < 16; i++) { - dst64[i] = val; - } -} - - -////////////////////////////////////////////////////////////////////////// - -#define sha256_block_size 64 -#define sha256_hash_size 32 -#define sha224_hash_size 28 -#define ROTR32(dword, n) ((dword) >> (n) ^ ((dword) << (32 - (n)))) -#define bswap_32(x) __builtin_bswap32(x) - -struct sha256_ctx { - uint32_t message[16]; /* 512-bit buffer for leftovers */ - uint64_t length; /* number of processed bytes */ - uint32_t hash[8]; /* 256-bit algorithm internal hashing state */ - uint32_t digest_length; /* length of the algorithm digest in bytes */ -}; - -struct sha256_ctx sctx; -struct sha256_ctx* ctx = &sctx; - -/* SHA-224 and SHA-256 constants for 64 rounds. These words represent - * the first 32 bits of the fractional parts of the cube - * roots of the first 64 prime numbers. */ -static const uint32_t rhash_k256[64] = { - 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, - 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, - 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, - 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, - 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, - 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, - 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, - 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, - 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, - 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, - 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 -}; - -/* The SHA256/224 functions defined by FIPS 180-3, 4.1.2 */ -/* Optimized version of Ch(x,y,z)=((x & y) | (~x & z)) */ -#define Ch(x, y, z) ((z) ^ ((x) & ((y) ^ (z)))) -/* Optimized version of Maj(x,y,z)=((x & y) ^ (x & z) ^ (y & z)) */ -#define Maj(x, y, z) (((x) & (y)) ^ ((z) & ((x) ^ (y)))) - -#define Sigma0(x) (ROTR32((x), 2) ^ ROTR32((x), 13) ^ ROTR32((x), 22)) -#define Sigma1(x) (ROTR32((x), 6) ^ ROTR32((x), 11) ^ ROTR32((x), 25)) -#define sigma0(x) (ROTR32((x), 7) ^ ROTR32((x), 18) ^ ((x) >> 3)) -#define sigma1(x) (ROTR32((x), 17) ^ ROTR32((x), 19) ^ ((x) >> 10)) - -/* Recalculate element n-th of circular buffer W using formula - * W[n] = sigma1(W[n - 2]) + W[n - 7] + sigma0(W[n - 15]) + W[n - 16]; */ -#define RECALCULATE_W(W, n) \ - (W[n] += \ - (sigma1(W[(n - 2) & 15]) + W[(n - 7) & 15] + sigma0(W[(n - 15) & 15]))) - -#define ROUND(a, b, c, d, e, f, g, h, k, data) \ - { \ - uint32_t T1 = h + Sigma1(e) + Ch(e, f, g) + k + (data); \ - d += T1, h = T1 + Sigma0(a) + Maj(a, b, c); \ - } -#define ROUND_1_16(a, b, c, d, e, f, g, h, n) \ - ROUND(a, b, c, d, e, f, g, h, rhash_k256[n], W[n] = bswap_32(block[n])) -#define ROUND_17_64(a, b, c, d, e, f, g, h, n) \ - ROUND(a, b, c, d, e, f, g, h, k[n], RECALCULATE_W(W, n)) - -/** - * Initialize context before calculaing hash. - * - */ -void sha256_init() { - /* Initial values. These words were obtained by taking the first 32 - * bits of the fractional parts of the square roots of the first - * eight prime numbers. */ - static const uint32_t SHA256_H0[8] = { - 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, - 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19 - }; - - ctx->length = 0; - ctx->digest_length = sha256_hash_size; - - /* initialize algorithm state */ - - #pragma clang loop vectorize(enable) - for (uint8_t i = 0; i < 8; i += 2) { - *(uint64_t*)&ctx->hash[i] = *(uint64_t*)&SHA256_H0[i]; - } -} - -/** - * Initialize context before calculaing hash. - * - */ -void sha224_init() { - /* Initial values from FIPS 180-3. These words were obtained by taking - * bits from 33th to 64th of the fractional parts of the square - * roots of ninth through sixteenth prime numbers. */ - static const uint32_t SHA224_H0[8] = { - 0xc1059ed8, 0x367cd507, 0x3070dd17, 0xf70e5939, - 0xffc00b31, 0x68581511, 0x64f98fa7, 0xbefa4fa4 - }; - - ctx->length = 0; - ctx->digest_length = sha224_hash_size; - - #pragma clang loop vectorize(enable) - for (uint8_t i = 0; i < 8; i += 2) { - *(uint64_t*)&ctx->hash[i] = *(uint64_t*)&SHA224_H0[i]; - } -} - -/** - * The core transformation. Process a 512-bit block. - * - * @param hash algorithm state - * @param block the message block to process - */ -static void sha256_process_block(uint32_t hash[8], uint32_t block[16]) { - uint32_t A, B, C, D, E, F, G, H; - uint32_t W[16]; - const uint32_t* k; - int i; - - A = hash[0], B = hash[1], C = hash[2], D = hash[3]; - E = hash[4], F = hash[5], G = hash[6], H = hash[7]; - - /* Compute SHA using alternate Method: FIPS 180-3 6.1.3 */ - ROUND_1_16(A, B, C, D, E, F, G, H, 0); - ROUND_1_16(H, A, B, C, D, E, F, G, 1); - ROUND_1_16(G, H, A, B, C, D, E, F, 2); - ROUND_1_16(F, G, H, A, B, C, D, E, 3); - ROUND_1_16(E, F, G, H, A, B, C, D, 4); - ROUND_1_16(D, E, F, G, H, A, B, C, 5); - ROUND_1_16(C, D, E, F, G, H, A, B, 6); - ROUND_1_16(B, C, D, E, F, G, H, A, 7); - ROUND_1_16(A, B, C, D, E, F, G, H, 8); - ROUND_1_16(H, A, B, C, D, E, F, G, 9); - ROUND_1_16(G, H, A, B, C, D, E, F, 10); - ROUND_1_16(F, G, H, A, B, C, D, E, 11); - ROUND_1_16(E, F, G, H, A, B, C, D, 12); - ROUND_1_16(D, E, F, G, H, A, B, C, 13); - ROUND_1_16(C, D, E, F, G, H, A, B, 14); - ROUND_1_16(B, C, D, E, F, G, H, A, 15); - - #pragma clang loop vectorize(enable) - for (i = 16, k = &rhash_k256[16]; i < 64; i += 16, k += 16) { - ROUND_17_64(A, B, C, D, E, F, G, H, 0); - ROUND_17_64(H, A, B, C, D, E, F, G, 1); - ROUND_17_64(G, H, A, B, C, D, E, F, 2); - ROUND_17_64(F, G, H, A, B, C, D, E, 3); - ROUND_17_64(E, F, G, H, A, B, C, D, 4); - ROUND_17_64(D, E, F, G, H, A, B, C, 5); - ROUND_17_64(C, D, E, F, G, H, A, B, 6); - ROUND_17_64(B, C, D, E, F, G, H, A, 7); - ROUND_17_64(A, B, C, D, E, F, G, H, 8); - ROUND_17_64(H, A, B, C, D, E, F, G, 9); - ROUND_17_64(G, H, A, B, C, D, E, F, 10); - ROUND_17_64(F, G, H, A, B, C, D, E, 11); - ROUND_17_64(E, F, G, H, A, B, C, D, 12); - ROUND_17_64(D, E, F, G, H, A, B, C, 13); - ROUND_17_64(C, D, E, F, G, H, A, B, 14); - ROUND_17_64(B, C, D, E, F, G, H, A, 15); - } - - hash[0] += A, hash[1] += B, hash[2] += C, hash[3] += D; - hash[4] += E, hash[5] += F, hash[6] += G, hash[7] += H; -} - -/** - * Calculate message hash. - * Can be called repeatedly with chunks of the message to be hashed. - * - * @param size length of the message chunk - */ -WASM_EXPORT -void Hash_Update(uint32_t size) { - const uint8_t* msg = main_buffer; - uint32_t index = (uint32_t)ctx->length & 63; - ctx->length += size; - - /* fill partial block */ - if (index) { - uint32_t left = sha256_block_size - index; - uint32_t end = size < left ? size : left; - uint8_t* message8 = (uint8_t*)ctx->message; - for (uint8_t i = 0; i < end; i++) { - *(message8 + index + i) = msg[i]; - } - if (size < left) return; - - /* process partial block */ - sha256_process_block(ctx->hash, (uint32_t*)ctx->message); - msg += left; - size -= left; - } - - while (size >= sha256_block_size) { - uint32_t* aligned_message_block = (uint32_t*)msg; - - sha256_process_block(ctx->hash, aligned_message_block); - msg += sha256_block_size; - size -= sha256_block_size; - } - - if (size) { - /* save leftovers */ - for (uint8_t i = 0; i < size; i++) { - *(((uint8_t*)ctx->message) + i) = msg[i]; - } - } -} - -/** - * Store calculated hash into the given array. - * - */ -WASM_EXPORT -void Hash_Final() { - uint32_t index = ((uint32_t)ctx->length & 63) >> 2; - uint32_t shift = ((uint32_t)ctx->length & 3) * 8; - - /* pad message and run for last block */ - - /* append the byte 0x80 to the message */ - ctx->message[index] &= ~(0xFFFFFFFFu << shift); - ctx->message[index++] ^= 0x80u << shift; - - /* if no room left in the message to store 64-bit message length */ - if (index > 14) { - /* then fill the rest with zeros and process it */ - while (index < 16) { - ctx->message[index++] = 0; - } - sha256_process_block(ctx->hash, ctx->message); - index = 0; - } - - while (index < 14) { - ctx->message[index++] = 0; - } - - ctx->message[14] = bswap_32((uint32_t)(ctx->length >> 29)); - ctx->message[15] = bswap_32((uint32_t)(ctx->length << 3)); - sha256_process_block(ctx->hash, ctx->message); - - #pragma clang loop vectorize(enable) - for (int32_t i = 7; i >= 0; i--) { - ctx->hash[i] = bswap_32(ctx->hash[i]); - } - - for (uint8_t i = 0; i < ctx->digest_length; i++) { - main_buffer[i] = *(((uint8_t*)ctx->hash) + i); - } -} - -WASM_EXPORT -uint32_t Hash_Init(uint32_t bits) { - if (bits == 224) { - sha224_init(); - } else { - sha256_init(); - } - return 0; -} - -WASM_EXPORT -const uint32_t STATE_SIZE = sizeof(*ctx); - -WASM_EXPORT -uint8_t* Hash_GetState() { - return (uint8_t*) ctx; -} - -WASM_EXPORT -uint32_t GetBufferPtr() { - return (uint32_t) main_buffer; -} diff --git a/packages/blob/src/vendor/hash-wasm/sha256.d.ts b/packages/blob/src/vendor/hash-wasm/sha256.d.ts deleted file mode 100644 index b6d0f5148..000000000 --- a/packages/blob/src/vendor/hash-wasm/sha256.d.ts +++ /dev/null @@ -1,8 +0,0 @@ -declare function Module(): Promise<{ - HEAPU8: Uint8Array; - _Hash_Init(type: number): void; - _Hash_Update(length: number): void; - _Hash_Final(): void; - _GetBufferPtr(): number; -}>; -export default Module; diff --git a/packages/blob/src/vendor/hash-wasm/sha256.js b/packages/blob/src/vendor/hash-wasm/sha256.js deleted file mode 100644 index 7ff85e582..000000000 --- a/packages/blob/src/vendor/hash-wasm/sha256.js +++ /dev/null @@ -1,685 +0,0 @@ - -var Module = (() => { - var _unused = import.meta.url; - - return ( -function(moduleArg = {}) { - -// include: shell.js -// The Module object: Our interface to the outside world. We import -// and export values on it. There are various ways Module can be used: -// 1. Not defined. We create it here -// 2. A function parameter, function(Module) { ..generated code.. } -// 3. pre-run appended it, var Module = {}; ..generated code.. -// 4. External script tag defines var Module. -// We need to check if Module already exists (e.g. case 3 above). -// Substitution will be replaced with actual code on later stage of the build, -// this way Closure Compiler will not mangle it (e.g. case 4. above). -// Note that if you want to run closure, and also to use Module -// after the generated code, you will need to define var Module = {}; -// before the code. Then that object will be used in the code, and you -// can continue to use Module afterwards as well. -var Module = moduleArg; - -// Set up the promise that indicates the Module is initialized -var readyPromiseResolve, readyPromiseReject; -Module['ready'] = new Promise((resolve, reject) => { - readyPromiseResolve = resolve; - readyPromiseReject = reject; -}); - -// --pre-jses are emitted after the Module integration code, so that they can -// refer to Module (if they choose; they can also define Module) - - -// Sometimes an existing Module object exists with properties -// meant to overwrite the default module functionality. Here -// we collect those properties and reapply _after_ we configure -// the current environment's defaults to avoid having to be so -// defensive during initialization. -var moduleOverrides = Object.assign({}, Module); - -var arguments_ = []; -var thisProgram = './this.program'; -var quit_ = (status, toThrow) => { - throw toThrow; -}; - -// Determine the runtime environment we are in. You can customize this by -// setting the ENVIRONMENT setting at compile time (see settings.js). - -// Attempt to auto-detect the environment -var ENVIRONMENT_IS_WEB = typeof window == 'object'; -var ENVIRONMENT_IS_WORKER = typeof importScripts == 'function'; -// N.b. Electron.js environment is simultaneously a NODE-environment, but -// also a web environment. -var ENVIRONMENT_IS_NODE = typeof process == 'object' && typeof process.versions == 'object' && typeof process.versions.node == 'string'; -var ENVIRONMENT_IS_SHELL = !ENVIRONMENT_IS_WEB && !ENVIRONMENT_IS_NODE && !ENVIRONMENT_IS_WORKER; - -// `/` should be present at the end if `scriptDirectory` is not empty -var scriptDirectory = ''; -function locateFile(path) { - if (Module['locateFile']) { - return Module['locateFile'](path, scriptDirectory); - } - return scriptDirectory + path; -} - -// Hooks that are implemented differently in different runtime environments. -var read_, - readAsync, - readBinary; - -// Note that this includes Node.js workers when relevant (pthreads is enabled). -// Node.js workers are detected as a combination of ENVIRONMENT_IS_WORKER and -// ENVIRONMENT_IS_NODE. -if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { - if (ENVIRONMENT_IS_WORKER) { // Check worker, not web, since window could be polyfilled - scriptDirectory = self.location.href; - } else if (typeof document != 'undefined' && document.currentScript) { // web - scriptDirectory = document.currentScript.src; - } - // When MODULARIZE, this JS may be executed later, after document.currentScript - // is gone, so we saved it, and we use it here instead of any other info. - if (false) { - scriptDirectory = false; - } - // blob urls look like blob:http://site.com/etc/etc and we cannot infer anything from them. - // otherwise, slice off the final part of the url to find the script directory. - // if scriptDirectory does not contain a slash, lastIndexOf will return -1, - // and scriptDirectory will correctly be replaced with an empty string. - // If scriptDirectory contains a query (starting with ?) or a fragment (starting with #), - // they are removed because they could contain a slash. - if (scriptDirectory.startsWith('blob:')) { - scriptDirectory = ''; - } else { - scriptDirectory = scriptDirectory.substr(0, scriptDirectory.replace(/[?#].*/, '').lastIndexOf('/')+1); - } - - // Differentiate the Web Worker from the Node Worker case, as reading must - // be done differently. - { -// include: web_or_worker_shell_read.js -read_ = (url) => { - var xhr = new XMLHttpRequest(); - xhr.open('GET', url, false); - xhr.send(null); - return xhr.responseText; - } - - if (ENVIRONMENT_IS_WORKER) { - readBinary = (url) => { - var xhr = new XMLHttpRequest(); - xhr.open('GET', url, false); - xhr.responseType = 'arraybuffer'; - xhr.send(null); - return new Uint8Array(/** @type{!ArrayBuffer} */(xhr.response)); - }; - } - - readAsync = (url, onload, onerror) => { - var xhr = new XMLHttpRequest(); - xhr.open('GET', url, true); - xhr.responseType = 'arraybuffer'; - xhr.onload = () => { - if (xhr.status == 200 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0 - onload(xhr.response); - return; - } - onerror(); - }; - xhr.onerror = onerror; - xhr.send(null); - } - -// end include: web_or_worker_shell_read.js - } -} else -{ -} - -var out = Module['print'] || console.log.bind(console); -var err = Module['printErr'] || console.error.bind(console); - -// Merge back in the overrides -Object.assign(Module, moduleOverrides); -// Free the object hierarchy contained in the overrides, this lets the GC -// reclaim data used. -moduleOverrides = null; - -// Emit code to handle expected values on the Module object. This applies Module.x -// to the proper local x. This has two benefits: first, we only emit it if it is -// expected to arrive, and second, by using a local everywhere else that can be -// minified. - -if (Module['arguments']) arguments_ = Module['arguments']; - -if (Module['thisProgram']) thisProgram = Module['thisProgram']; - -if (Module['quit']) quit_ = Module['quit']; - -// perform assertions in shell.js after we set up out() and err(), as otherwise if an assertion fails it cannot print the message -// end include: shell.js - -// include: preamble.js -// === Preamble library stuff === - -// Documentation for the public APIs defined in this file must be updated in: -// site/source/docs/api_reference/preamble.js.rst -// A prebuilt local version of the documentation is available at: -// site/build/text/docs/api_reference/preamble.js.txt -// You can also build docs locally as HTML or other formats in site/ -// An online HTML version (which may be of a different version of Emscripten) -// is up at http://kripken.github.io/emscripten-site/docs/api_reference/preamble.js.html - -var wasmBinary; -if (Module['wasmBinary']) wasmBinary = Module['wasmBinary']; - -if (typeof WebAssembly != 'object') { - abort('no native wasm support detected'); -} - -// include: base64Utils.js -// Converts a string of base64 into a byte array (Uint8Array). -function intArrayFromBase64(s) { - - var decoded = atob(s); - var bytes = new Uint8Array(decoded.length); - for (var i = 0 ; i < decoded.length ; ++i) { - bytes[i] = decoded.charCodeAt(i); - } - return bytes; -} - -// If filename is a base64 data URI, parses and returns data (Buffer on node, -// Uint8Array otherwise). If filename is not a base64 data URI, returns undefined. -function tryParseAsDataURI(filename) { - if (!isDataURI(filename)) { - return; - } - - return intArrayFromBase64(filename.slice(dataURIPrefix.length)); -} -// end include: base64Utils.js -// Wasm globals - -var wasmMemory; - -//======================================== -// Runtime essentials -//======================================== - -// whether we are quitting the application. no code should run after this. -// set in exit() and abort() -var ABORT = false; - -// set by exit() and abort(). Passed to 'onExit' handler. -// NOTE: This is also used as the process return code code in shell environments -// but only when noExitRuntime is false. -var EXITSTATUS; - -// In STRICT mode, we only define assert() when ASSERTIONS is set. i.e. we -// don't define it at all in release modes. This matches the behaviour of -// MINIMAL_RUNTIME. -// TODO(sbc): Make this the default even without STRICT enabled. -/** @type {function(*, string=)} */ -function assert(condition, text) { - if (!condition) { - // This build was created without ASSERTIONS defined. `assert()` should not - // ever be called in this configuration but in case there are callers in - // the wild leave this simple abort() implementation here for now. - abort(text); - } -} - -// Memory management - -var HEAP, -/** @type {!Int8Array} */ - HEAP8, -/** @type {!Uint8Array} */ - HEAPU8, -/** @type {!Int16Array} */ - HEAP16, -/** @type {!Uint16Array} */ - HEAPU16, -/** @type {!Int32Array} */ - HEAP32, -/** @type {!Uint32Array} */ - HEAPU32, -/** @type {!Float32Array} */ - HEAPF32, -/** @type {!Float64Array} */ - HEAPF64; - -// include: runtime_shared.js -function updateMemoryViews() { - var b = wasmMemory.buffer; - Module['HEAP8'] = HEAP8 = new Int8Array(b); - Module['HEAP16'] = HEAP16 = new Int16Array(b); - Module['HEAPU8'] = HEAPU8 = new Uint8Array(b); - Module['HEAPU16'] = HEAPU16 = new Uint16Array(b); - Module['HEAP32'] = HEAP32 = new Int32Array(b); - Module['HEAPU32'] = HEAPU32 = new Uint32Array(b); - Module['HEAPF32'] = HEAPF32 = new Float32Array(b); - Module['HEAPF64'] = HEAPF64 = new Float64Array(b); -} -// end include: runtime_shared.js -// include: runtime_stack_check.js -// end include: runtime_stack_check.js -// include: runtime_assertions.js -// end include: runtime_assertions.js -var __ATPRERUN__ = []; // functions called before the runtime is initialized -var __ATINIT__ = []; // functions called during startup -var __ATEXIT__ = []; // functions called during shutdown -var __ATPOSTRUN__ = []; // functions called after the main() is called - -var runtimeInitialized = false; - -function preRun() { - if (Module['preRun']) { - if (typeof Module['preRun'] == 'function') Module['preRun'] = [Module['preRun']]; - while (Module['preRun'].length) { - addOnPreRun(Module['preRun'].shift()); - } - } - callRuntimeCallbacks(__ATPRERUN__); -} - -function initRuntime() { - runtimeInitialized = true; - - - callRuntimeCallbacks(__ATINIT__); -} - -function postRun() { - - if (Module['postRun']) { - if (typeof Module['postRun'] == 'function') Module['postRun'] = [Module['postRun']]; - while (Module['postRun'].length) { - addOnPostRun(Module['postRun'].shift()); - } - } - - callRuntimeCallbacks(__ATPOSTRUN__); -} - -function addOnPreRun(cb) { - __ATPRERUN__.unshift(cb); -} - -function addOnInit(cb) { - __ATINIT__.unshift(cb); -} - -function addOnExit(cb) { -} - -function addOnPostRun(cb) { - __ATPOSTRUN__.unshift(cb); -} - -// include: runtime_math.js -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/imul - -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/fround - -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/clz32 - -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/trunc - -// end include: runtime_math.js -// A counter of dependencies for calling run(). If we need to -// do asynchronous work before running, increment this and -// decrement it. Incrementing must happen in a place like -// Module.preRun (used by emcc to add file preloading). -// Note that you can add dependencies in preRun, even though -// it happens right before run - run will be postponed until -// the dependencies are met. -var runDependencies = 0; -var runDependencyWatcher = null; -var dependenciesFulfilled = null; // overridden to take different actions when all run dependencies are fulfilled - -function getUniqueRunDependency(id) { - return id; -} - -function addRunDependency(id) { - runDependencies++; - - Module['monitorRunDependencies']?.(runDependencies); - -} - -function removeRunDependency(id) { - runDependencies--; - - Module['monitorRunDependencies']?.(runDependencies); - - if (runDependencies == 0) { - if (runDependencyWatcher !== null) { - clearInterval(runDependencyWatcher); - runDependencyWatcher = null; - } - if (dependenciesFulfilled) { - var callback = dependenciesFulfilled; - dependenciesFulfilled = null; - callback(); // can add another dependenciesFulfilled - } - } -} - -/** @param {string|number=} what */ -function abort(what) { - Module['onAbort']?.(what); - - what = 'Aborted(' + what + ')'; - // TODO(sbc): Should we remove printing and leave it up to whoever - // catches the exception? - err(what); - - ABORT = true; - EXITSTATUS = 1; - - what += '. Build with -sASSERTIONS for more info.'; - - // Use a wasm runtime error, because a JS error might be seen as a foreign - // exception, which means we'd run destructors on it. We need the error to - // simply make the program stop. - // FIXME This approach does not work in Wasm EH because it currently does not assume - // all RuntimeErrors are from traps; it decides whether a RuntimeError is from - // a trap or not based on a hidden field within the object. So at the moment - // we don't have a way of throwing a wasm trap from JS. TODO Make a JS API that - // allows this in the wasm spec. - - // Suppress closure compiler warning here. Closure compiler's builtin extern - // definition for WebAssembly.RuntimeError claims it takes no arguments even - // though it can. - // TODO(https://github.com/google/closure-compiler/pull/3913): Remove if/when upstream closure gets fixed. - /** @suppress {checkTypes} */ - var e = new WebAssembly.RuntimeError(what); - - readyPromiseReject(e); - // Throw the error whether or not MODULARIZE is set because abort is used - // in code paths apart from instantiation where an exception is expected - // to be thrown when abort is called. - throw e; -} - -// include: memoryprofiler.js -// end include: memoryprofiler.js -// include: URIUtils.js -// Prefix of data URIs emitted by SINGLE_FILE and related options. -var dataURIPrefix = 'data:application/octet-stream;base64,'; - -/** - * Indicates whether filename is a base64 data URI. - * @noinline - */ -var isDataURI = (filename) => filename.startsWith(dataURIPrefix); - -/** - * Indicates whether filename is delivered via file protocol (as opposed to http/https) - * @noinline - */ -var isFileURI = (filename) => filename.startsWith('file://'); -// end include: URIUtils.js -// include: runtime_exceptions.js -// end include: runtime_exceptions.js -var wasmBinaryFile; - wasmBinaryFile = 'data:application/octet-stream;base64,AGFzbQEAAAABHQZgAX8AYAABf2AAAGABfwF/YAJ/fwBgA39/fwF/Aw0MAgAEAgMBBQABAQADBAUBcAEBAQUGAQGAAoACBg4CfwFB8IuEBAt/AUEACweYAQoGbWVtb3J5AgARX193YXNtX2NhbGxfY3RvcnMAAAtIYXNoX1VwZGF0ZQABCkhhc2hfRmluYWwAAwlIYXNoX0luaXQABAxHZXRCdWZmZXJQdHIABRlfX2luZGlyZWN0X2Z1bmN0aW9uX3RhYmxlAQAJc3RhY2tTYXZlAAkMc3RhY2tSZXN0b3JlAAoKc3RhY2tBbGxvYwALCossDAIAC+4CAgV/AX5BACgCwAoiASABKQNAIgYgAK18NwNAAkACQAJAIAanQT9xIgINAEGACyEBIAAhAgwBC0HAACACayEDAkAgAEUNACADIAAgAyAASRshBCABIAJqIQVBACEBA0AgBSABIgFqQYALIAFqLQAAOgAAIAFBAWoiAiEBIAIgBEcNAAsLAkACQCAAIANJIgRFDQBBgAshASAAIQIMAQtBACgCwAoiAUHIAGogARACQYALIANqIQEgACADayECCyABIQEgAiECIAQNAQsgASEBAkACQCACIgJBwABPDQAgASEFIAIhAAwBCyACIQIgASEEA0BBACgCwApByABqIAQiBBACIAJBQGoiASECIARBwABqIgUhBCAFIQUgASEAIAFBP0sNAAsLIAUhBSAAIgBFDQBBACEBQQAhAgNAQQAoAsAKIAEiAWogBSABai0AADoAACACQQFqIgJB/wFxIgQhASACIQIgACAESw0ACwsLqCEBK38gACgCCCICIAAoAgQiAyAAKAIAIgRzcSADIARxcyAEQR53IARBE3dzIARBCndzaiAAKAIQIgVBGncgBUEVd3MgBUEHd3MgACgCHCIGaiAAKAIYIgcgACgCFCIIcyAFcSAHc2ogASgCACIJQRh0IAlBgP4DcUEIdHIgCUEIdkGA/gNxIAlBGHZyciIKakGY36iUBGoiC2oiCSAEcyADcSAJIARxcyAJQR53IAlBE3dzIAlBCndzaiAHIAEoAgQiDEEYdCAMQYD+A3FBCHRyIAxBCHZBgP4DcSAMQRh2cnIiDWogCyAAKAIMIg5qIg8gCCAFc3EgCHNqIA9BGncgD0EVd3MgD0EHd3NqQZGJ3YkHaiIQaiIMIAlzIARxIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAggASgCCCILQRh0IAtBgP4DcUEIdHIgC0EIdkGA/gNxIAtBGHZyciIRaiAQIAJqIhIgDyAFc3EgBXNqIBJBGncgEkEVd3MgEkEHd3NqQc/3g657aiITaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAUgASgCDCIQQRh0IBBBgP4DcUEIdHIgEEEIdkGA/gNxIBBBGHZyciIUaiATIANqIhMgEiAPc3EgD3NqIBNBGncgE0EVd3MgE0EHd3NqQaW3181+aiIVaiIQIAtzIAxxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIA8gASgCECIWQRh0IBZBgP4DcUEIdHIgFkEIdkGA/gNxIBZBGHZyciIXaiAVIARqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQduE28oDaiIYaiIPIBBzIAtxIA8gEHFzIA9BHncgD0ETd3MgD0EKd3NqIAEoAhQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGSASaiAYIAlqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQfGjxM8FaiIYaiIJIA9zIBBxIAkgD3FzIAlBHncgCUETd3MgCUEKd3NqIAEoAhgiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGiATaiAYIAxqIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQaSF/pF5aiIYaiIMIAlzIA9xIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAEoAhwiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGyAWaiAYIAtqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQdW98dh6aiIYaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAEoAiAiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHCASaiAYIBBqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQZjVnsB9aiIYaiIQIAtzIAxxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAiQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHSATaiAYIA9qIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQYG2jZQBaiIYaiIPIBBzIAtxIA8gEHFzIA9BHncgD0ETd3MgD0EKd3NqIAEoAigiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHiAWaiAYIAlqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQb6LxqECaiIYaiIJIA9zIBBxIAkgD3FzIAlBHncgCUETd3MgCUEKd3NqIAEoAiwiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHyASaiAYIAxqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQcP7sagFaiIYaiIMIAlzIA9xIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAEoAjAiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiICATaiAYIAtqIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQfS6+ZUHaiIYaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAEoAjQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiISAWaiAYIBBqIhAgEyASc3EgEnNqIBBBGncgEEEVd3MgEEEHd3NqQf7j+oZ4aiIYaiIWIAtzIAxxIBYgC3FzIBZBHncgFkETd3MgFkEKd3NqIAEoAjgiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiIiASaiAYIA9qIg8gECATc3EgE3NqIA9BGncgD0EVd3MgD0EHd3NqQaeN8N55aiIVaiISIBZzIAtxIBIgFnFzIBJBHncgEkETd3MgEkEKd3NqIAEoAjwiAUEYdCABQYD+A3FBCHRyIAFBCHZBgP4DcSABQRh2cnIiIyATaiAVIAlqIgEgDyAQc3EgEHNqIAFBGncgAUEVd3MgAUEHd3NqQfTi74x8aiIJaiEVIBIhGCAWISQgCyElIAkgDGohJiABIScgDyEoIBAhKSAjISMgIiEiICEhISAgISAgHyEfIB4hHiAdIR0gHCEcIBshGyAaIRogGSEZIBchFyAUIRQgESERIA0hECAKIQxBgAkhAUEQISoDQCAVIgkgGCIKcyAkIitxIAkgCnFzIAlBHncgCUETd3MgCUEKd3NqIBAiEEEZdyAQQQ53cyAQQQN2cyAMaiAdIh1qICIiFkEPdyAWQQ13cyAWQQp2c2oiDCApaiAmIhIgJyIPICgiE3NxIBNzaiASQRp3IBJBFXdzIBJBB3dzaiABIgEoAgBqIiRqIgsgCXMgCnEgCyAJcXMgC0EedyALQRN3cyALQQp3c2ogESIYQRl3IBhBDndzIBhBA3ZzIBBqIB4iHmogIyIVQQ93IBVBDXdzIBVBCnZzaiINIBNqIAEoAgRqICQgJWoiEyASIA9zcSAPc2ogE0EadyATQRV3cyATQQd3c2oiJWoiECALcyAJcSAQIAtxcyAQQR53IBBBE3dzIBBBCndzaiAUIiRBGXcgJEEOd3MgJEEDdnMgGGogHyIfaiAMQQ93IAxBDXdzIAxBCnZzaiIRIA9qIAEoAghqICUgK2oiGCATIBJzcSASc2ogGEEadyAYQRV3cyAYQQd3c2oiJWoiDyAQcyALcSAPIBBxcyAPQR53IA9BE3dzIA9BCndzaiAXIhdBGXcgF0EOd3MgF0EDdnMgJGogICIgaiANQQ93IA1BDXdzIA1BCnZzaiIUIBJqIAEoAgxqICUgCmoiCiAYIBNzcSATc2ogCkEadyAKQRV3cyAKQQd3c2oiJWoiEiAPcyAQcSASIA9xcyASQR53IBJBE3dzIBJBCndzaiATIBkiJEEZdyAkQQ53cyAkQQN2cyAXaiAhIiFqIBFBD3cgEUENd3MgEUEKdnNqIhdqIAEoAhBqICUgCWoiEyAKIBhzcSAYc2ogE0EadyATQRV3cyATQQd3c2oiJWoiCSAScyAPcSAJIBJxcyAJQR53IAlBE3dzIAlBCndzaiABKAIUIBoiGkEZdyAaQQ53cyAaQQN2cyAkaiAWaiAUQQ93IBRBDXdzIBRBCnZzaiIZaiAYaiAlIAtqIhggEyAKc3EgCnNqIBhBGncgGEEVd3MgGEEHd3NqIiVqIgsgCXMgEnEgCyAJcXMgC0EedyALQRN3cyALQQp3c2ogASgCGCAbIiRBGXcgJEEOd3MgJEEDdnMgGmogFWogF0EPdyAXQQ13cyAXQQp2c2oiGmogCmogJSAQaiIKIBggE3NxIBNzaiAKQRp3IApBFXdzIApBB3dzaiIlaiIQIAtzIAlxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAhwgHCIcQRl3IBxBDndzIBxBA3ZzICRqIAxqIBlBD3cgGUENd3MgGUEKdnNqIhtqIBNqICUgD2oiJCAKIBhzcSAYc2ogJEEadyAkQRV3cyAkQQd3c2oiE2oiDyAQcyALcSAPIBBxcyAPQR53IA9BE3dzIA9BCndzaiABKAIgIB1BGXcgHUEOd3MgHUEDdnMgHGogDWogGkEPdyAaQQ13cyAaQQp2c2oiHGogGGogEyASaiIYICQgCnNxIApzaiAYQRp3IBhBFXdzIBhBB3dzaiITaiISIA9zIBBxIBIgD3FzIBJBHncgEkETd3MgEkEKd3NqIAEoAiQgHkEZdyAeQQ53cyAeQQN2cyAdaiARaiAbQQ93IBtBDXdzIBtBCnZzaiIdaiAKaiATIAlqIgkgGCAkc3EgJHNqIAlBGncgCUEVd3MgCUEHd3NqIgpqIhMgEnMgD3EgEyAScXMgE0EedyATQRN3cyATQQp3c2ogASgCKCAfQRl3IB9BDndzIB9BA3ZzIB5qIBRqIBxBD3cgHEENd3MgHEEKdnNqIh5qICRqIAogC2oiCiAJIBhzcSAYc2ogCkEadyAKQRV3cyAKQQd3c2oiJGoiCyATcyAScSALIBNxcyALQR53IAtBE3dzIAtBCndzaiABKAIsICBBGXcgIEEOd3MgIEEDdnMgH2ogF2ogHUEPdyAdQQ13cyAdQQp2c2oiH2ogGGogJCAQaiIYIAogCXNxIAlzaiAYQRp3IBhBFXdzIBhBB3dzaiIkaiIQIAtzIBNxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAjAgIUEZdyAhQQ53cyAhQQN2cyAgaiAZaiAeQQ93IB5BDXdzIB5BCnZzaiIgaiAJaiAkIA9qIiQgGCAKc3EgCnNqICRBGncgJEEVd3MgJEEHd3NqIg9qIgkgEHMgC3EgCSAQcXMgCUEedyAJQRN3cyAJQQp3c2ogASgCNCAWQRl3IBZBDndzIBZBA3ZzICFqIBpqIB9BD3cgH0ENd3MgH0EKdnNqIiFqIApqIA8gEmoiDyAkIBhzcSAYc2ogD0EadyAPQRV3cyAPQQd3c2oiCmoiEiAJcyAQcSASIAlxcyASQR53IBJBE3dzIBJBCndzaiABKAI4IBVBGXcgFUEOd3MgFUEDdnMgFmogG2ogIEEPdyAgQQ13cyAgQQp2c2oiImogGGogCiATaiITIA8gJHNxICRzaiATQRp3IBNBFXdzIBNBB3dzaiIYaiIWIBJzIAlxIBYgEnFzIBZBHncgFkETd3MgFkEKd3NqIAEoAjwgDEEZdyAMQQ53cyAMQQN2cyAVaiAcaiAhQQ93ICFBDXdzICFBCnZzaiIKaiAkaiAYIAtqIgsgEyAPc3EgD3NqIAtBGncgC0EVd3MgC0EHd3NqIiZqIishFSAWIRggEiEkIAkhJSAmIBBqIiwhJiALIScgEyEoIA8hKSAKISMgIiEiICEhISAgISAgHyEfIB4hHiAdIR0gHCEcIBshGyAaIRogGSEZIBchFyAUIRQgESERIA0hECAMIQwgAUHAAGohASAqIgpBEGohKiAKQTBJDQALIAAgDyAGajYCHCAAIBMgB2o2AhggACALIAhqNgIUIAAgLCAFajYCECAAIAkgDmo2AgwgACASIAJqNgIIIAAgFiADajYCBCAAICsgBGo2AgAL1AMDBX8BfgF7QQAoAsAKIgAgACgCQCIBQQJ2QQ9xIgJBAnRqIgMgAygCAEF/IAFBA3QiAXRBf3NxQYABIAF0czYCAAJAAkAgAkEOTw0AIAJBAWohAAwBCwJAIAJBDkcNACAAQQA2AjwLIABByABqIAAQAkEAIQALAkAgACIAQQ1LDQBBACgCwAogAEECdCIAakEAQTggAGsQBhoLQQAoAsAKIgAgACkDQCIFpyICQRt0IAJBC3RBgID8B3FyIAJBBXZBgP4DcSACQQN0QRh2cnI2AjwgACAFQh2IpyICQRh0IAJBgP4DcUEIdHIgAkEIdkGA/gNxIAJBGHZycjYCOCAAQcgAaiAAEAJBACgCwApBPGohAUEAIQADQCABQQcgACIAa0ECdGoiAiAC/QACACAG/Q0MDQ4PCAkKCwQFBgcAAQIDIAb9DQMCAQAHBgUECwoJCA8ODQwgBv0NDA0ODwgJCgsEBQYHAAECA/0LAgAgAEEEaiICIQAgAkEIRw0ACwJAQQAoAsAKIgMoAmhFDQAgA0HIAGohBEEAIQBBACECA0BBgAsgACIAaiAEIABqLQAAOgAAIAJBAWoiAkH/AXEiASEAIAIhAiADKAJoIAFLDQALCwtxAQJ/QQAoAsAKIgFCADcDQCABQcgAaiECAkAgAEHgAUcNACABQRw2AmggAkEQakEA/QAEsAj9CwIAIAJBAP0ABKAI/QsCAEEADwsgAUEgNgJoIAJBEGpBAP0ABJAI/QsCACACQQD9AASACP0LAgBBAAsFAEGACwvyAgIDfwF+AkAgAkUNACAAIAE6AAAgACACaiIDQX9qIAE6AAAgAkEDSQ0AIAAgAToAAiAAIAE6AAEgA0F9aiABOgAAIANBfmogAToAACACQQdJDQAgACABOgADIANBfGogAToAACACQQlJDQAgAEEAIABrQQNxIgRqIgMgAUH/AXFBgYKECGwiATYCACADIAIgBGtBfHEiBGoiAkF8aiABNgIAIARBCUkNACADIAE2AgggAyABNgIEIAJBeGogATYCACACQXRqIAE2AgAgBEEZSQ0AIAMgATYCGCADIAE2AhQgAyABNgIQIAMgATYCDCACQXBqIAE2AgAgAkFsaiABNgIAIAJBaGogATYCACACQWRqIAE2AgAgBCADQQRxQRhyIgVrIgJBIEkNACABrUKBgICAEH4hBiADIAVqIQEDQCABIAY3AxggASAGNwMQIAEgBjcDCCABIAY3AwAgAUEgaiEBIAJBYGoiAkEfSw0ACwsgAAsGACAAJAELBAAjAQsEACMACwYAIAAkAAsSAQJ/IwAgAGtBcHEiASQAIAELC9ICAgBBgAgLwAJn5glqha5nu3Lzbjw69U+lf1IOUYxoBZur2YMfGc3gW9ieBcEH1Xw2F91wMDlZDvcxC8D/ERVYaKeP+WSkT/q+mC+KQpFEN3HP+8C1pdu16VvCVjnxEfFZpII/ktVeHKuYqgfYAVuDEr6FMSTDfQxVdF2+cv6x3oCnBtybdPGbwcFpm+SGR77vxp3BD8yhDCRvLOktqoR0StypsFzaiPl2UlE+mG3GMajIJwOwx39Zv/ML4MZHkafVUWPKBmcpKRSFCrcnOCEbLvxtLE0TDThTVHMKZbsKanYuycKBhSxykqHov6JLZhqocItLwqNRbMcZ6JLRJAaZ1oU1DvRwoGoQFsGkGQhsNx5Md0gntbywNLMMHDlKqthOT8qcW/NvLmjugo90b2OleBR4yIQIAseM+v++kOtsUKT3o/m+8nhxxgBBwAoLBIAFgAA='; - if (!isDataURI(wasmBinaryFile)) { - wasmBinaryFile = locateFile(wasmBinaryFile); - } - -function getBinarySync(file) { - if (file == wasmBinaryFile && wasmBinary) { - return new Uint8Array(wasmBinary); - } - var binary = tryParseAsDataURI(file); - if (binary) { - return binary; - } - if (readBinary) { - return readBinary(file); - } - throw 'both async and sync fetching of the wasm failed'; -} - -function getBinaryPromise(binaryFile) { - - // Otherwise, getBinarySync should be able to get it synchronously - return Promise.resolve().then(() => getBinarySync(binaryFile)); -} - -function instantiateArrayBuffer(binaryFile, imports, receiver) { - return getBinaryPromise(binaryFile).then((binary) => { - return WebAssembly.instantiate(binary, imports); - }).then(receiver, (reason) => { - err(`failed to asynchronously prepare wasm: ${reason}`); - - abort(reason); - }); -} - -function instantiateAsync(binary, binaryFile, imports, callback) { - return instantiateArrayBuffer(binaryFile, imports, callback); -} - -// Create the wasm instance. -// Receives the wasm imports, returns the exports. -function createWasm() { - // prepare imports - var info = { - 'env': wasmImports, - 'wasi_snapshot_preview1': wasmImports, - }; - // Load the wasm module and create an instance of using native support in the JS engine. - // handle a generated wasm instance, receiving its exports and - // performing other necessary setup - /** @param {WebAssembly.Module=} module*/ - function receiveInstance(instance, module) { - wasmExports = instance.exports; - - - - wasmMemory = wasmExports['memory']; - - updateMemoryViews(); - - addOnInit(wasmExports['__wasm_call_ctors']); - - removeRunDependency('wasm-instantiate'); - return wasmExports; - } - // wait for the pthread pool (if any) - addRunDependency('wasm-instantiate'); - - // Prefer streaming instantiation if available. - function receiveInstantiationResult(result) { - // 'result' is a ResultObject object which has both the module and instance. - // receiveInstance() will swap in the exports (to Module.asm) so they can be called - // TODO: Due to Closure regression https://github.com/google/closure-compiler/issues/3193, the above line no longer optimizes out down to the following line. - // When the regression is fixed, can restore the above PTHREADS-enabled path. - receiveInstance(result['instance']); - } - - // User shell pages can write their own Module.instantiateWasm = function(imports, successCallback) callback - // to manually instantiate the Wasm module themselves. This allows pages to - // run the instantiation parallel to any other async startup actions they are - // performing. - // Also pthreads and wasm workers initialize the wasm instance through this - // path. - if (Module['instantiateWasm']) { - - try { - return Module['instantiateWasm'](info, receiveInstance); - } catch(e) { - err(`Module.instantiateWasm callback failed with error: ${e}`); - // If instantiation fails, reject the module ready promise. - readyPromiseReject(e); - } - } - - // If instantiation fails, reject the module ready promise. - instantiateAsync(wasmBinary, wasmBinaryFile, info, receiveInstantiationResult).catch(readyPromiseReject); - return {}; // no exports yet; we'll fill them in later -} - -// Globals used by JS i64 conversions (see makeSetValue) -var tempDouble; -var tempI64; - -// include: runtime_debug.js -// end include: runtime_debug.js -// === Body === -// end include: preamble.js - - - /** @constructor */ - function ExitStatus(status) { - this.name = 'ExitStatus'; - this.message = `Program terminated with exit(${status})`; - this.status = status; - } - - var callRuntimeCallbacks = (callbacks) => { - while (callbacks.length > 0) { - // Pass the module as the first argument. - callbacks.shift()(Module); - } - }; - - - /** - * @param {number} ptr - * @param {string} type - */ - function getValue(ptr, type = 'i8') { - if (type.endsWith('*')) type = '*'; - switch (type) { - case 'i1': return HEAP8[ptr]; - case 'i8': return HEAP8[ptr]; - case 'i16': return HEAP16[((ptr)>>1)]; - case 'i32': return HEAP32[((ptr)>>2)]; - case 'i64': abort('to do getValue(i64) use WASM_BIGINT'); - case 'float': return HEAPF32[((ptr)>>2)]; - case 'double': return HEAPF64[((ptr)>>3)]; - case '*': return HEAPU32[((ptr)>>2)]; - default: abort(`invalid type for getValue: ${type}`); - } - } - - var noExitRuntime = Module['noExitRuntime'] || true; - - - /** - * @param {number} ptr - * @param {number} value - * @param {string} type - */ - function setValue(ptr, value, type = 'i8') { - if (type.endsWith('*')) type = '*'; - switch (type) { - case 'i1': HEAP8[ptr] = value; break; - case 'i8': HEAP8[ptr] = value; break; - case 'i16': HEAP16[((ptr)>>1)] = value; break; - case 'i32': HEAP32[((ptr)>>2)] = value; break; - case 'i64': abort('to do setValue(i64) use WASM_BIGINT'); - case 'float': HEAPF32[((ptr)>>2)] = value; break; - case 'double': HEAPF64[((ptr)>>3)] = value; break; - case '*': HEAPU32[((ptr)>>2)] = value; break; - default: abort(`invalid type for setValue: ${type}`); - } - } -var wasmImports = { - -}; -var wasmExports = createWasm(); -var ___wasm_call_ctors = () => (___wasm_call_ctors = wasmExports['__wasm_call_ctors'])(); -var _Hash_Update = Module['_Hash_Update'] = (a0) => (_Hash_Update = Module['_Hash_Update'] = wasmExports['Hash_Update'])(a0); -var _Hash_Final = Module['_Hash_Final'] = () => (_Hash_Final = Module['_Hash_Final'] = wasmExports['Hash_Final'])(); -var _Hash_Init = Module['_Hash_Init'] = (a0) => (_Hash_Init = Module['_Hash_Init'] = wasmExports['Hash_Init'])(a0); -var _GetBufferPtr = Module['_GetBufferPtr'] = () => (_GetBufferPtr = Module['_GetBufferPtr'] = wasmExports['GetBufferPtr'])(); -var stackSave = () => (stackSave = wasmExports['stackSave'])(); -var stackRestore = (a0) => (stackRestore = wasmExports['stackRestore'])(a0); -var stackAlloc = (a0) => (stackAlloc = wasmExports['stackAlloc'])(a0); - - -// include: postamble.js -// === Auto-generated postamble setup entry stuff === - - - - -var calledRun; - -dependenciesFulfilled = function runCaller() { - // If run has never been called, and we should call run (INVOKE_RUN is true, and Module.noInitialRun is not false) - if (!calledRun) run(); - if (!calledRun) dependenciesFulfilled = runCaller; // try this again later, after new deps are fulfilled -}; - -function run() { - - if (runDependencies > 0) { - return; - } - - preRun(); - - // a preRun added a dependency, run will be called later - if (runDependencies > 0) { - return; - } - - function doRun() { - // run may have just been called through dependencies being fulfilled just in this very frame, - // or while the async setStatus time below was happening - if (calledRun) return; - calledRun = true; - Module['calledRun'] = true; - - if (ABORT) return; - - initRuntime(); - - readyPromiseResolve(Module); - if (Module['onRuntimeInitialized']) Module['onRuntimeInitialized'](); - - postRun(); - } - - if (Module['setStatus']) { - Module['setStatus']('Running...'); - setTimeout(function() { - setTimeout(function() { - Module['setStatus'](''); - }, 1); - doRun(); - }, 1); - } else - { - doRun(); - } -} - -if (Module['preInit']) { - if (typeof Module['preInit'] == 'function') Module['preInit'] = [Module['preInit']]; - while (Module['preInit'].length > 0) { - Module['preInit'].pop()(); - } -} - -run(); - -// end include: postamble.js - - - - return moduleArg.ready -} -); -})(); -export default Module; \ No newline at end of file diff --git a/packages/blob/src/vendor/type-fest/basic.ts b/packages/blob/src/vendor/type-fest/basic.ts deleted file mode 100644 index 3fa40a039..000000000 --- a/packages/blob/src/vendor/type-fest/basic.ts +++ /dev/null @@ -1,31 +0,0 @@ -/** -Matches a JSON object. - -This type can be useful to enforce some input to be JSON-compatible or as a super-type to be extended from. Don't use this as a direct return type as the user would have to double-cast it: `jsonObject as unknown as CustomResponse`. Instead, you could extend your CustomResponse type from it to ensure your type only uses JSON-compatible types: `interface CustomResponse extends JsonObject { … }`. - -@category JSON -*/ -export type JsonObject = { [Key in string]: JsonValue } & { [Key in string]?: JsonValue | undefined }; - -/** -Matches a JSON array. - -@category JSON -*/ -export type JsonArray = JsonValue[] | readonly JsonValue[]; - -/** -Matches any valid JSON primitive value. - -@category JSON -*/ -export type JsonPrimitive = string | number | boolean | null; - -/** -Matches any valid JSON value. - -@see `Jsonify` if you need to transform a type to one that is assignable to `JsonValue`. - -@category JSON -*/ -export type JsonValue = JsonPrimitive | JsonObject | JsonArray; diff --git a/packages/blob/src/vendor/type-fest/entries.ts b/packages/blob/src/vendor/type-fest/entries.ts deleted file mode 100644 index 7716e4748..000000000 --- a/packages/blob/src/vendor/type-fest/entries.ts +++ /dev/null @@ -1,65 +0,0 @@ -import type { ArrayEntry, MapEntry, ObjectEntry, SetEntry } from "./entry"; - -type ArrayEntries = Array>; -type MapEntries = Array>; -type ObjectEntries = Array>; -type SetEntries> = Array>; - -/** -Many collections have an `entries` method which returns an array of a given object's own enumerable string-keyed property [key, value] pairs. The `Entries` type will return the type of that collection's entries. - -For example the {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/entries|`Object`}, {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map/entries|`Map`}, {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/entries|`Array`}, and {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set/entries|`Set`} collections all have this method. Note that `WeakMap` and `WeakSet` do not have this method since their entries are not enumerable. - -@see `Entry` if you want to just access the type of a single entry. - -@example -``` -import type {Entries} from 'type-fest'; - -interface Example { - someKey: number; -} - -const manipulatesEntries = (examples: Entries) => examples.map(example => [ - // Does some arbitrary processing on the key (with type information available) - example[0].toUpperCase(), - - // Does some arbitrary processing on the value (with type information available) - example[1].toFixed() -]); - -const example: Example = {someKey: 1}; -const entries = Object.entries(example) as Entries; -const output = manipulatesEntries(entries); - -// Objects -const objectExample = {a: 1}; -const objectEntries: Entries = [['a', 1]]; - -// Arrays -const arrayExample = ['a', 1]; -const arrayEntries: Entries = [[0, 'a'], [1, 1]]; - -// Maps -const mapExample = new Map([['a', 1]]); -const mapEntries: Entries = [['a', 1]]; - -// Sets -const setExample = new Set(['a', 1]); -const setEntries: Entries = [['a', 'a'], [1, 1]]; -``` - -@category Object -@category Map -@category Set -@category Array -*/ -export type Entries = BaseType extends Map - ? MapEntries - : BaseType extends Set - ? SetEntries - : BaseType extends readonly unknown[] - ? ArrayEntries - : BaseType extends object - ? ObjectEntries - : never; diff --git a/packages/blob/src/vendor/type-fest/entry.ts b/packages/blob/src/vendor/type-fest/entry.ts deleted file mode 100644 index ed3650c90..000000000 --- a/packages/blob/src/vendor/type-fest/entry.ts +++ /dev/null @@ -1,68 +0,0 @@ -type MapKey = BaseType extends Map ? KeyType : never; -type MapValue = BaseType extends Map ? ValueType : never; - -export type ArrayEntry = [number, BaseType[number]]; -export type MapEntry = [MapKey, MapValue]; -export type ObjectEntry = [keyof BaseType, BaseType[keyof BaseType]]; -export type SetEntry = BaseType extends Set ? [ItemType, ItemType] : never; - -/** -Many collections have an `entries` method which returns an array of a given object's own enumerable string-keyed property [key, value] pairs. The `Entry` type will return the type of that collection's entry. - -For example the {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/entries|`Object`}, {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map/entries|`Map`}, {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/entries|`Array`}, and {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set/entries|`Set`} collections all have this method. Note that `WeakMap` and `WeakSet` do not have this method since their entries are not enumerable. - -@see `Entries` if you want to just access the type of the array of entries (which is the return of the `.entries()` method). - -@example -``` -import type {Entry} from 'type-fest'; - -interface Example { - someKey: number; -} - -const manipulatesEntry = (example: Entry) => [ - // Does some arbitrary processing on the key (with type information available) - example[0].toUpperCase(), - - // Does some arbitrary processing on the value (with type information available) - example[1].toFixed(), -]; - -const example: Example = {someKey: 1}; -const entry = Object.entries(example)[0] as Entry; -const output = manipulatesEntry(entry); - -// Objects -const objectExample = {a: 1}; -const objectEntry: Entry = ['a', 1]; - -// Arrays -const arrayExample = ['a', 1]; -const arrayEntryString: Entry = [0, 'a']; -const arrayEntryNumber: Entry = [1, 1]; - -// Maps -const mapExample = new Map([['a', 1]]); -const mapEntry: Entry = ['a', 1]; - -// Sets -const setExample = new Set(['a', 1]); -const setEntryString: Entry = ['a', 'a']; -const setEntryNumber: Entry = [1, 1]; -``` - -@category Object -@category Map -@category Array -@category Set -*/ -export type Entry = BaseType extends Map - ? MapEntry - : BaseType extends Set - ? SetEntry - : BaseType extends readonly unknown[] - ? ArrayEntry - : BaseType extends object - ? ObjectEntry - : never; diff --git a/packages/blob/src/vendor/type-fest/except.ts b/packages/blob/src/vendor/type-fest/except.ts deleted file mode 100644 index b18f739d1..000000000 --- a/packages/blob/src/vendor/type-fest/except.ts +++ /dev/null @@ -1,71 +0,0 @@ -import type { IsEqual } from "./is-equal"; - -/** -Filter out keys from an object. - -Returns `never` if `Exclude` is strictly equal to `Key`. -Returns `never` if `Key` extends `Exclude`. -Returns `Key` otherwise. - -@example -``` -type Filtered = Filter<'foo', 'foo'>; -//=> never -``` - -@example -``` -type Filtered = Filter<'bar', string>; -//=> never -``` - -@example -``` -type Filtered = Filter<'bar', 'foo'>; -//=> 'bar' -``` - -@see {Except} -*/ -type Filter = IsEqual extends true - ? never - : KeyType extends ExcludeType - ? never - : KeyType; - -/** -Create a type from an object type without certain keys. - -We recommend setting the `requireExactProps` option to `true`. - -This type is a stricter version of [`Omit`](https://www.typescriptlang.org/docs/handbook/release-notes/typescript-3-5.html#the-omit-helper-type). The `Omit` type does not restrict the omitted keys to be keys present on the given type, while `Except` does. The benefits of a stricter type are avoiding typos and allowing the compiler to pick up on rename refactors automatically. - -This type was proposed to the TypeScript team, which declined it, saying they prefer that libraries implement stricter versions of the built-in types ([microsoft/TypeScript#30825](https://github.com/microsoft/TypeScript/issues/30825#issuecomment-523668235)). - -@example -``` -import type {Except} from 'type-fest'; - -type Foo = { - a: number; - b: string; -}; - -type FooWithoutA = Except; -//=> {b: string} - -const fooWithoutA: FooWithoutA = {a: 1, b: '2'}; -//=> errors: 'a' does not exist in type '{ b: string; }' - -type FooWithoutB = Except; -//=> {a: number} & Partial> - -const fooWithoutB: FooWithoutB = {a: 1, b: '2'}; -//=> errors at 'b': Type 'string' is not assignable to type 'undefined'. -``` - -@category Object -*/ -export type Except = { - [KeyType in keyof ObjectType as Filter]: ObjectType[KeyType]; -}; diff --git a/packages/blob/src/vendor/type-fest/is-equal.ts b/packages/blob/src/vendor/type-fest/is-equal.ts deleted file mode 100644 index d6ff2e53c..000000000 --- a/packages/blob/src/vendor/type-fest/is-equal.ts +++ /dev/null @@ -1,27 +0,0 @@ -/** -Returns a boolean for whether the two given types are equal. - -@link https://github.com/microsoft/TypeScript/issues/27024#issuecomment-421529650 -@link https://stackoverflow.com/questions/68961864/how-does-the-equals-work-in-typescript/68963796#68963796 - -Use-cases: -- If you want to make a conditional branch based on the result of a comparison of two types. - -@example -``` -import type {IsEqual} from 'type-fest'; - -// This type returns a boolean for whether the given array includes the given item. -// `IsEqual` is used to compare the given array at position 0 and the given item and then return true if they are equal. -type Includes = - Value extends readonly [Value[0], ...infer rest] - ? IsEqual extends true - ? true - : Includes - : false; -``` - -@category Type Guard -@category Utilities -*/ -export type IsEqual = (() => G extends A ? 1 : 2) extends () => G extends B ? 1 : 2 ? true : false; diff --git a/packages/blob/src/vendor/type-fest/set-required.ts b/packages/blob/src/vendor/type-fest/set-required.ts deleted file mode 100644 index 8e4c6417a..000000000 --- a/packages/blob/src/vendor/type-fest/set-required.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { Except } from "./except"; -import type { Simplify } from "./simplify"; - -/** -Create a type that makes the given keys required. The remaining keys are kept as is. The sister of the `SetOptional` type. - -Use-case: You want to define a single model where the only thing that changes is whether or not some of the keys are required. - -@example -``` -import type {SetRequired} from 'type-fest'; - -type Foo = { - a?: number; - b: string; - c?: boolean; -} - -type SomeRequired = SetRequired; -// type SomeRequired = { -// a?: number; -// b: string; // Was already required and still is. -// c: boolean; // Is now required. -// } -``` - -@category Object -*/ -export type SetRequired = Simplify< - // Pick just the keys that are optional from the base type. - Except & - // Pick the keys that should be required from the base type and make them required. - Required> ->; diff --git a/packages/blob/src/vendor/type-fest/simplify.ts b/packages/blob/src/vendor/type-fest/simplify.ts deleted file mode 100644 index f4564fe70..000000000 --- a/packages/blob/src/vendor/type-fest/simplify.ts +++ /dev/null @@ -1,59 +0,0 @@ -/** -Useful to flatten the type output to improve type hints shown in editors. And also to transform an interface into a type to aide with assignability. - -@example -``` -import type {Simplify} from 'type-fest'; - -type PositionProps = { - top: number; - left: number; -}; - -type SizeProps = { - width: number; - height: number; -}; - -// In your editor, hovering over `Props` will show a flattened object with all the properties. -type Props = Simplify; -``` - -Sometimes it is desired to pass a value as a function argument that has a different type. At first inspection it may seem assignable, and then you discover it is not because the `value`'s type definition was defined as an interface. In the following example, `fn` requires an argument of type `Record`. If the value is defined as a literal, then it is assignable. And if the `value` is defined as type using the `Simplify` utility the value is assignable. But if the `value` is defined as an interface, it is not assignable because the interface is not sealed and elsewhere a non-string property could be added to the interface. - -If the type definition must be an interface (perhaps it was defined in a third-party npm package), then the `value` can be defined as `const value: Simplify = ...`. Then `value` will be assignable to the `fn` argument. Or the `value` can be cast as `Simplify` if you can't re-declare the `value`. - -@example -``` -import type {Simplify} from 'type-fest'; - -interface SomeInterface { - foo: number; - bar?: string; - baz: number | undefined; -} - -type SomeType = { - foo: number; - bar?: string; - baz: number | undefined; -}; - -const literal = {foo: 123, bar: 'hello', baz: 456}; -const someType: SomeType = literal; -const someInterface: SomeInterface = literal; - -function fn(object: Record): void {} - -fn(literal); // Good: literal object type is sealed -fn(someType); // Good: type is sealed -fn(someInterface); // Error: Index signature for type 'string' is missing in type 'someInterface'. Because `interface` can be re-opened -fn(someInterface as Simplify); // Good: transform an `interface` into a `type` -``` - -@link https://github.com/microsoft/TypeScript/issues/15300 - -@category Object -*/ -// eslint-disable-next-line @typescript-eslint/ban-types -export type Simplify = { [KeyType in keyof T]: T[KeyType] } & {};