diff --git a/packages/auto-dag-data/README.md b/packages/auto-dag-data/README.md index 1d6b1e07..dd2cd02a 100644 --- a/packages/auto-dag-data/README.md +++ b/packages/auto-dag-data/README.md @@ -8,7 +8,7 @@ ## Overview -The **Autonomys Auto DAG Data SDK** (`@autonomys/auto-dag-data`) provides utilities for creating and managing IPLD DAGs (InterPlanetary Linked Data Directed Acyclic Graphs) for files and folders. It facilitates chunking large files, handling metadata, and creating folder structures suitable for distributed storage systems like IPFS. +The **Autonomys Auto Dag Data SDK** (`@autonomys/auto-dag-data`) provides utilities for creating and managing IPLD DAGs (InterPlanetary Linked Data Directed Acyclic Graphs) for files and folders. It facilitates chunking large files, handling metadata, and creating folder structures suitable for distributed storage systems like IPFS. ## Features @@ -20,7 +20,7 @@ The **Autonomys Auto DAG Data SDK** (`@autonomys/auto-dag-data`) provides utilit ## Installation -You can install Auto-DAG-Data using npm or yarn: +You can install Auto-Dag-Data using npm or yarn: ```bash npm install @autonomys/auto-dag-data @@ -36,23 +36,27 @@ yarn add @autonomys/auto-dag-data ### Creating an IPLD DAG from a File -To create an IPLD DAG from a file, you can use the `createFileIPLDDag` function: +To create an IPLD DAG from a file, you can use the `processFileToIPLDFormat` function: ```typescript -import { createFileIPLDDag } from '@autonomys/auto-dag-data' +import { processFileToIPLDFormat } from '@autonomys/auto-dag-data' +import { MemoryBlockstore } from 'blockstore-core/memory' import fs from 'fs' -const fileBuffer = fs.readFileSync('path/to/your/file.txt') +const fileStream = fs.createReadStream('path/to/your/file.txt') +const fileSize = fs.statSync('path/to/your/file.txt').size -const dag = createFileIPLDDag(fileBuffer, 'file.txt') +const blockstore = new MemoryBlockstore() +const fileCID = processFileToIPLDFormat(blockstore, fileStream, totalSize, 'file.txt') ``` ### Creating an IPLD DAG from a Folder -To create an IPLD DAG from a folder, you can use the `createFolderIPLDDag` function: +To generate an IPLD DAG from a folder, you can use the `processFolderToIPLDFormat` function: ```typescript -import { createFolderIPLDDag } from '@autonomys/auto-dag-data' +import { processFolderToIPLDFormat, decodeNode } from '@autonomys/auto-dag-data' +import { MemoryBlockstore } from 'blockstore-core/memory' import { CID } from 'multiformats' // Example child CIDs and folder information @@ -60,9 +64,12 @@ const childCIDs: CID[] = [ /* array of CIDs */ ] const folderName = 'my-folder' -const folderSize = 1024 // size in bytes +const folderSize = 1024 // size in bytes (the sum of their children size) -const folderDag = createFolderIPLDDag(childCIDs, folderName, folderSize) +const blockstore = new MemoryBlockstore() +const folderCID = processFolderToIPLDFormat(blockstore, childCIDs, folderName, folderSize) + +const node = decodeNode(blockstore.get(folderCID)) ``` ### Working with CIDs @@ -115,14 +122,16 @@ const metadataNode = createMetadataNode(metadata) ### Example: Creating a File DAG and Converting to CID ```typescript -import { createFileIPLDDag, cidOfNode, cidToString } from '@autonomys/auto-dag-data' +import { processFileToIPLDFormat } from '@autonomys/auto-dag-data' +import { MemoryBlockstore } from 'blockstore-core/memory' import fs from 'fs' -const fileBuffer = fs.readFileSync('path/to/your/file.txt') +const fileStream = fs.createReadStream('path/to/your/file.txt') +const fileSize = fs.statSync('path/to/your/file.txt').size -const dag = createFileIPLDDag(fileBuffer, 'file.txt') +const blockstore = new MemoryBlockstore() +const cid = processFileToIPLDFormat(blockstore, fileStream, totalSize, 'file.txt') -const cid = cidOfNode(dag.headCID) const cidString = cidToString(cid) console.log(`CID of the file DAG: ${cidString}`) @@ -137,13 +146,14 @@ import { cidToString, type OffchainMetadata, } from '@autonomys/auto-dag-data' +import { MemoryBlockstore } from 'blockstore-core/memory' import fs from 'fs' const metadata: OffchainMetadata = fs.readFileSync('path/to/your/metadata.json') -const dag = createMetadataIPLDDag(metadata) +const blockstore = new MemoryBlockstore() +const cid = processMetadataToIPLDFormat(blockstore, metadata) -const cid = cidOfNode(dag.headCID) const cidString = cidToString(cid) console.log(`CID of the metadata DAG: ${cidString}`) diff --git a/packages/auto-drive/README.md b/packages/auto-drive/README.md index d828e98d..b58d8c10 100644 --- a/packages/auto-drive/README.md +++ b/packages/auto-drive/README.md @@ -20,21 +20,21 @@ yarn add @autonomys/auto-drive ### How to use it? -### Example Usage +### How to upload a file from filepath? (Not available in browser) -Here is an example of how to use the `uploadFile` method to upload a file with optional encryption and compression: +Here is an example of how to use the `uploadFileFromFilepath` method to upload a file with optional encryption and compression: ```typescript -import { uploadFile } from '@autonomys/auto-drive' +import { uploadFileFromFilepath } from '@autonomys/auto-drive' -const api = new AutoDriveApi() // Initialize your API instance +const api = createAutoDriveApi({ apiKey: 'your-api-key' }) // Initialize your API instance with API key const filePath = 'path/to/your/file.txt' // Specify the path to your file const options = { password: 'your-encryption-password', // Optional: specify a password for encryption compression: true, } -uploadFile(api, filePath, options) +const uploadObservable = uploadFile(api, filePath, options) .then(() => { console.log('File uploaded successfully!') }) @@ -43,42 +43,185 @@ uploadFile(api, filePath, options) }) ``` -### Example Usage of Download - -Here is an example of how to use the `downloadFile` method to download a file from the server: +### How to upload [File](https://developer.mozilla.org/en-US/docs/Web/API/File) interface ```typescript -import { getRoots } from '@autonomys/auto-drive' +import { uploadFileFromFilepath } from '@autonomys/auto-drive' -const api = new AutoDriveApi() // Initialize your API instance +const api = createAutoDriveApi({ apiKey: 'your-api-key' }) // Initialize your API instance with API key +const filePath = 'path/to/your/file.txt' // Specify the path to your file +const options = { + password: 'your-encryption-password', // Optional: specify a password for encryption + compression: true, +} -getRoots(api) - .then((roots) => { - console.log('Root directories:', roots) +const uploadObservable = uploadFile(api, filePath, options) + .then(() => { + console.log('File uploaded successfully!') }) .catch((error) => { - console.error('Error retrieving root directories:', error) + console.error('Error uploading file:', error) }) ``` -### Example Usage of getRoots +### How to upload a file from a custom interface? -Here is an example of how to use the `getRoots` method to retrieve the root directories: +Some times you might have a custom interface that doesn't fit either File or filepath. For those cases exists the interface GenericFile: + +```typescript +export interface GenericFile { + read(): AsyncIterable // A buffer generator function that will output the bytes of the file + name: string + mimeType?: string + size: number + path: string // Could be ignored in file upload +} +``` + +For more info about asynn generator visit [this website](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/AsyncGenerator). + +You could upload any file that could be represented in that way. For example, uploading a file as a `Buffer` ```typescript -import { getRoots } from '@autonomys/auto-drive' +import { uploadFile } from '@autonomys/auto-drive' + +const api = createAutoDriveApi({ apiKey: 'your-api-key' }) // Initialize your API instance with API key +const buffer = Buffer.from(...); +const genericFile = { + read: async function *() { + yield buffer + }, + name: "autonomys-whitepaper.pdf", + mimeType: "application/pdf", + size: 1234556, + path: "autonomys-whitepaper.pdf" +} -const api = new AutoDriveApi() // Initialize your API instance +const options = { + password: 'your-encryption-password', // Optional: specify a password for encryption + compression: true, +} -getRoots(api) - .then((roots) => { - console.log('Root directories:', roots) +const uploadObservable = uploadFile(api, genericFile, options) + .then(() => { + console.log('File uploaded successfully!') }) .catch((error) => { - console.error('Error retrieving root directories:', error) + console.error('Error uploading file:', error) }) ``` +### How to upload a folder from folder? (Not available in browser) + +```ts +import { uploadFolderFromFolderPath } from '@autonomys/auto-drive' + +const api = createAutoDriveApi({ apiKey: 'your-api-key' }) // Initialize your API instance with API key +const folderPath = 'path/to/your/folder' // Specify the path to your folder + +const options = { + uploadChunkSize: 1024 * 1024, // Optional: specify the chunk size for uploads + password: 'your-encryption-password', // Optional: If folder is encrypted +} + +const uploadObservable = uploadFolderFromFolderPath(api, folderPath, options) +``` + +**Note: If a folder is tried to be encrypted a zip file would be generated and that file would be encrypted and uploaded.** + +### Handle observables + +Since uploads may take some time, specially in big-sized files. Uploads do implement `rxjs` observables so you could have feedback about the process or even show your users the progress of the upload. + +For that reason when file upload functions return `PromisedObservable`: + +```typescript +export type UploadFileStatus = { + type: 'file' + progress: number + cid?: CID +} +``` + +Being the cid only returned (and thus optional) when the upload is completed. + +Similarly, for folder uploads the functions return `PromisedObservable` + +```ts +export type UploadFolderStatus = { + type: 'folder' + progress: number + cid?: CID +} +``` + +**e.g Show upload progress in React** + +```typescript +const [progress, setProgress] = useState(0) + +useEffect(async () => { + const finalStatus = await uploadFileFromInput(api, genericFile, options).forEach((status) => { + setProgress(status.progress) + }) +}) +``` + +**e.g Ignore observables** + +Other users may want to not use the progress observability. For having a promise instead the field `promise` is a Promise that resolves into `UploadFileStatus` and `UploadFolderStatus` for files and folders respectively. + +e.g + +```ts +const status = await uploadFileFromInput(api, genericFile, options).promise +const cid = status.cid +``` + +### Example Usage of Download + +Here is an example of how to use the `downloadFile` method to download a file from the server: + +```typescript +import { downloadObject } from '@autonomys/auto-drive' + +const api = createAutoDriveApi({ apiKey: 'your-api-key' }) // Initialize your API instance with API key + +try { + const cid = ".." + const stream = await downloadObject(api, { cid }) + let file = Buffer.alloc(0); + for await (const chunk of stream) { + file = Buffer.concat([file, chunk]) + } + console.log('File downloaded successfully:', stream) +} catch (error) { + console.error('Error downloading file:', error) +} + +``` + +### Example Usage of getRoots + +Here is an example of how to use the `getRoots` method to retrieve the root directories: + +```typescript +import { createAutoDriveApi, downloadObject } from '@autonomys/auto-drive' +import fs from 'fs' + +const api = createAutoDriveApi({ apiKey: 'your-api-key' }) // Initialize your API instance with API key + +try { + const stream = fs.createWriteStream('/path/to/file') + const asyncBuffer = await downloadObject(api, { cid }) + for await (const buffer of asyncBuffer) { + stream.write(buffer) + } +} catch (error) { + console.error('Error downloading file:', error) +} +``` + ## License This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details. @@ -93,4 +236,4 @@ If you have any questions or need support, feel free to reach out: - **GitHub Issues**: [GitHub Issues Page](https://github.com/autonomys/auto-sdk/issues) -We appreciate your feedback and contributions! +We appreciate your feedback and contributions! \ No newline at end of file