⚗️ Copy hub to blob

packages/blob/.eslintignore

@@ -0,0 +1,2 @@
+dist
+sha256.js

packages/blob/.prettierignore

@@ -0,0 +1,5 @@
+pnpm-lock.yaml
+# In order to avoid code samples to have tabs, they don't display well on npm
+README.md
+dist
+sha256.js

packages/blob/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Hugging Face
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/blob/README.md

@@ -0,0 +1,176 @@
+# 🤗 Hugging Face Hub API
+
+Official utilities to use the Hugging Face Hub API.
+
+## Install
+
+```console
+pnpm add @huggingface/hub
+
+npm add @huggingface/hub
+
+yarn add @huggingface/hub
+```
+
+### Deno
+
+```ts
+// esm.sh
+import { uploadFiles, listModels } from "https://esm.sh/@huggingface/hub"
+// or npm:
+import { uploadFiles, listModels } from "npm:@huggingface/hub"
+```
+
+Check out the [full documentation](https://huggingface.co/docs/huggingface.js/hub/README).
+
+## Usage
+
+For some of the calls, you need to create an account and generate an [access token](https://huggingface.co/settings/tokens).
+
+Learn how to find free models using the hub package in this [interactive tutorial](https://scrimba.com/scrim/c7BbVPcd?pl=pkVnrP7uP).
+
+```ts
+import * as hub from "@huggingface/hub";
+import type { RepoDesignation } from "@huggingface/hub";
+
+const repo: RepoDesignation = { type: "model", name: "myname/some-model" };
+
+const {name: username} = await hub.whoAmI({accessToken: "hf_..."});
+
+for await (const model of hub.listModels({search: {owner: username}, accessToken: "hf_..."})) {
+  console.log("My model:", model);
+}
+
+const specificModel = await hub.modelInfo({name: "openai-community/gpt2"});
+await hub.checkRepoAccess({repo, accessToken: "hf_..."});
+
+await hub.createRepo({ repo, accessToken: "hf_...", license: "mit" });
+
+await hub.uploadFiles({
+  repo,
+  accessToken: "hf_...",
+  files: [
+    // path + blob content
+    {
+      path: "file.txt",
+      content: new Blob(["Hello World"]),
+    },
+    // Local file URL
+    pathToFileURL("./pytorch-model.bin"),
+    // Web URL
+    new URL("https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json"),
+    // Path + Web URL
+    {
+      path: "myfile.bin",
+      content: new URL("https://huggingface.co/bert-base-uncased/resolve/main/pytorch_model.bin")
+    }
+    // Can also work with native File in browsers
+  ],
+});
+
+// or
+
+for await (const progressEvent of await hub.uploadFilesWithProgress({
+  repo,
+  accessToken: "hf_...",
+  files: [
+    ...
+  ],
+})) {
+  console.log(progressEvent);
+}
+
+await hub.deleteFile({repo, accessToken: "hf_...", path: "myfile.bin"});
+
+await (await hub.downloadFile({ repo, path: "README.md" })).text();
+
+for await (const fileInfo of hub.listFiles({repo})) {
+  console.log(fileInfo);
+}
+
+await hub.deleteRepo({ repo, accessToken: "hf_..." });
+```
+
+## OAuth Login
+
+It's possible to login using OAuth (["Sign in with HF"](https://huggingface.co/docs/hub/oauth)).
+
+This will allow you get an access token to use some of the API, depending on the scopes set inside the Space or the OAuth App.
+
+```ts
+import { oauthLoginUrl, oauthHandleRedirectIfPresent } from "@huggingface/hub";
+
+const oauthResult = await oauthHandleRedirectIfPresent();
+
+if (!oauthResult) {
+  // If the user is not logged in, redirect to the login page
+  window.location.href = await oauthLoginUrl();
+}
+
+// You can use oauthResult.accessToken, oauthResult.accessTokenExpiresAt and oauthResult.userInfo
+console.log(oauthResult);
+```
+
+Checkout the demo: https://huggingface.co/spaces/huggingfacejs/client-side-oauth
+
+## Hugging face cache
+
+The `@huggingface/hub` package provide basic capabilities to scan the cache directory. Learn more about [Manage huggingface_hub cache-system](https://huggingface.co/docs/huggingface_hub/en/guides/manage-cache).
+
+### `scanCacheDir`
+
+You can get the list of cached repositories using the `scanCacheDir` function.
+
+```ts
+import { scanCacheDir } from "@huggingface/hub";
+
+const result = await scanCacheDir();
+
+console.log(result);
+```
+Note: this does not work in the browser
+
+### `downloadFileToCacheDir`
+
+You can cache a file of a repository using the `downloadFileToCacheDir` function.
+
+```ts
+import { downloadFileToCacheDir } from "@huggingface/hub";
+
+const file = await downloadFileToCacheDir({
+  repo: 'foo/bar',
+  path: 'README.md'
+});
+
+console.log(file);
+```
+Note: this does not work in the browser
+
+### `snapshotDownload`
+
+You can download an entire repository at a given revision in the cache directory using the `snapshotDownload` function.
+
+```ts
+import { snapshotDownload } from "@huggingface/hub";
+
+const directory = await snapshotDownload({
+  repo: 'foo/bar',
+});
+
+console.log(directory);
+```
+The code use internally the `downloadFileToCacheDir` function.
+
+Note: this does not work in the browser
+
+## Performance considerations
+
+When uploading large files, you may want to run the `commit` calls inside a worker, to offload the sha256 computations.
+
+Remote resources and local files should be passed as `URL` whenever it's possible so they can be lazy loaded in chunks to reduce RAM usage. Passing a `File` inside the browser's context is fine, because it natively behaves as a `Blob`.
+
+Under the hood, `@huggingface/hub` uses a lazy blob implementation to load the file.
+
+## Dependencies
+
+- `@huggingface/tasks` : Typings only

packages/blob/index.ts

@@ -0,0 +1 @@
+export * from "./src";

packages/blob/package.json

@@ -0,0 +1,67 @@
+{
+	"name": "@huggingface/blob",
+	"packageManager": "pnpm@8.10.5",
+	"version": "0.21.0",
+	"description": "Utilities to interact with the Hugging Face hub",
+	"repository": "https://github.com/huggingface/huggingface.js.git",
+	"publishConfig": {
+		"access": "public"
+	},
+	"main": "./dist/index.js",
+	"module": "./dist/index.mjs",
+	"types": "./dist/index.d.ts",
+	"exports": {
+		".": {
+			"types": "./dist/index.d.ts",
+			"require": "./dist/index.js",
+			"import": "./dist/index.mjs"
+		}
+	},
+	"browser": {
+		"./src/utils/sha256-node.ts": false,
+		"./src/utils/FileBlob.ts": false,
+		"./src/lib/cache-management.ts": false,
+		"./src/lib/download-file-to-cache-dir.ts": false,
+		"./src/lib/snapshot-download.ts": false,
+		"./dist/index.js": "./dist/browser/index.js",
+		"./dist/index.mjs": "./dist/browser/index.mjs"
+	},
+	"engines": {
+		"node": ">=18"
+	},
+	"source": "index.ts",
+	"scripts": {
+		"lint": "eslint --quiet --fix --ext .cjs,.ts .",
+		"lint:check": "eslint --ext .cjs,.ts .",
+		"format": "prettier --write .",
+		"format:check": "prettier --check .",
+		"prepublishOnly": "pnpm run build",
+		"build": "tsup && tsc --emitDeclarationOnly --declaration",
+		"prepare": "pnpm run build",
+		"test": "vitest run",
+		"test:browser": "vitest run --browser.name=chrome --browser.headless --config vitest-browser.config.mts",
+		"check": "tsc"
+	},
+	"files": [
+		"src",
+		"dist",
+		"index.ts",
+		"tsconfig.json"
+	],
+	"keywords": [
+		"huggingface",
+		"hub",
+		"api",
+		"client",
+		"hugging",
+		"face"
+	],
+	"author": "Hugging Face",
+	"license": "MIT",
+	"devDependencies": {
+		"@types/node": "^20.11.28"
+	},
+	"dependencies": {
+		"@huggingface/tasks": "workspace:^"
+	}
+}

packages/blob/src/consts.ts

@@ -0,0 +1 @@
+export const HUB_URL = "https://huggingface.co";

packages/blob/src/error.ts

@@ -0,0 +1,46 @@
+import type { JsonObject } from "./vendor/type-fest/basic";
+
+export async function createApiError(
+	response: Response,
+	opts?: { requestId?: string; message?: string }
+): Promise<never> {
+	const error = new HubApiError(response.url, response.status, response.headers.get("X-Request-Id") ?? opts?.requestId);
+
+	error.message = `Api error with status ${error.statusCode}${opts?.message ? `. ${opts.message}` : ""}`;
+
+	const trailer = [`URL: ${error.url}`, error.requestId ? `Request ID: ${error.requestId}` : undefined]
+		.filter(Boolean)
+		.join(". ");
+
+	if (response.headers.get("Content-Type")?.startsWith("application/json")) {
+		const json = await response.json();
+		error.message = json.error || json.message || error.message;
+		error.data = json;
+	} else {
+		error.data = { message: await response.text() };
+	}
+
+	error.message += `. ${trailer}`;
+
+	throw error;
+}
+
+/**
+ * Error thrown when an API call to the Hugging Face Hub fails.
+ */
+export class HubApiError extends Error {
+	statusCode: number;
+	url: string;
+	requestId?: string;
+	data?: JsonObject;
+
+	constructor(url: string, statusCode: number, requestId?: string, message?: string) {
+		super(message);
+
+		this.statusCode = statusCode;
+		this.requestId = requestId;
+		this.url = url;
+	}
+}
+
+export class InvalidApiResponseFormatError extends Error {}

packages/blob/src/index.ts

@@ -0,0 +1,24 @@
+export * from "./lib";
+// Typescript 5 will add 'export type *'
+export type {
+	AccessToken,
+	AccessTokenRole,
+	AuthType,
+	Credentials,
+	PipelineType,
+	RepoDesignation,
+	RepoFullName,
+	RepoId,
+	RepoType,
+	SpaceHardwareFlavor,
+	SpaceResourceConfig,
+	SpaceResourceRequirement,
+	SpaceRuntime,
+	SpaceSdk,
+	SpaceStage,
+} from "./types/public";
+export { HubApiError, InvalidApiResponseFormatError } from "./error";
+/**
+ * Only exported for E2Es convenience
+ */
+export { sha256 as __internal_sha256 } from "./utils/sha256";