diff --git a/packages/auto-drive/README.md b/packages/auto-drive/README.md index 64f6645..4d8e389 100644 --- a/packages/auto-drive/README.md +++ b/packages/auto-drive/README.md @@ -25,43 +25,40 @@ yarn add @autonomys/auto-drive Here is an example of how to use the `uploadFileFromFilepath` method to upload a file with optional encryption and compression: ```typescript -import { uploadFileFromFilepath } from '@autonomys/auto-drive' +import { uploadFileFromFilepath,createAutoDriveApi } from '@autonomys/auto-drive' const api = createAutoDriveApi({ apiKey: 'your-api-key' }) // Initialize your API instance with API key const filePath = 'path/to/your/file.txt' // Specify the path to your file const options = { password: 'your-encryption-password', // Optional: specify a password for encryption compression: true, + // an optional callback useful for large file uploads + onProgress?: (progress: number) => { + console.log(`The upload is completed is ${progress}% completed`) + } } -const uploadObservable = uploadFileFromFilepath(api, filePath, options) - .then(() => { - console.log('File uploaded successfully!') - }) - .catch((error) => { - console.error('Error uploading file:', error) - }) +const cid = await uploadFileFromFilepath(api, filePath, options) + +console.log(`The file is uploaded and its cid is ${cid}`) ``` ### How to upload [File](https://developer.mozilla.org/en-US/docs/Web/API/File) interface ```typescript -import { uploadFileFromFilepath } from '@autonomys/auto-drive' +import { uploadFileFromInput, createAutoDriveApi } from '@autonomys/auto-drive' const api = createAutoDriveApi({ apiKey: 'your-api-key' }) // Initialize your API instance with API key -const filePath = 'path/to/your/file.txt' // Specify the path to your file + +// e.g Get File from object from HTML event +const file: File = e.target.value // Substitute with your file const options = { password: 'your-encryption-password', // Optional: specify a password for encryption compression: true, } +const cid = await uploadFileFromInput(api, file, options) -const uploadObservable = uploadFile(api, filePath, options) - .then(() => { - console.log('File uploaded successfully!') - }) - .catch((error) => { - console.error('Error uploading file:', error) - }) +console.log(`The file is uploaded and its cid is ${cid}`) ``` ### How to upload a file from a custom interface? @@ -83,7 +80,7 @@ For more info about asynn generator visit [this website](https://developer.mozil You could upload any file that could be represented in that way. For example, uploading a file as a `Buffer` ```typescript -import { uploadFile } from '@autonomys/auto-drive' +import { createAutoDriveApi, uploadFile } from '@autonomys/auto-drive' const api = createAutoDriveApi({ apiKey: 'your-api-key' }) // Initialize your API instance with API key const buffer = Buffer.from(...); @@ -100,21 +97,21 @@ const genericFile = { const options = { password: 'your-encryption-password', // Optional: specify a password for encryption compression: true, + // an optional callback useful for large file uploads + onProgress?: (progress: number) => { + console.log(`The upload is completed is ${progress}% completed`) + } } -const uploadObservable = uploadFile(api, genericFile, options) - .then(() => { - console.log('File uploaded successfully!') - }) - .catch((error) => { - console.error('Error uploading file:', error) - }) +const cid = uploadFile(api, genericFile, options) + +console.log(`The file is uploaded and its cid is ${cid}`) ``` ### How to upload a folder from folder? (Not available in browser) ```ts -import { uploadFolderFromFolderPath } from '@autonomys/auto-drive' +import { createAutoDriveApi, uploadFolderFromFolderPath } from '@autonomys/auto-drive' const api = createAutoDriveApi({ apiKey: 'your-api-key' }) // Initialize your API instance with API key const folderPath = 'path/to/your/folder' // Specify the path to your folder @@ -122,74 +119,31 @@ const folderPath = 'path/to/your/folder' // Specify the path to your folder const options = { uploadChunkSize: 1024 * 1024, // Optional: specify the chunk size for uploads password: 'your-encryption-password', // Optional: If folder is encrypted + // an optional callback useful for large file uploads + onProgress: (progress: number) => { + console.log(`The upload is completed is ${progress}% completed`) + }, } -const uploadObservable = uploadFolderFromFolderPath(api, folderPath, options) -``` - -**Note: If a folder is tried to be encrypted a zip file would be generated and that file would be encrypted and uploaded.** - -### Handle observables - -Since uploads may take some time, specially in big-sized files. Uploads do implement `rxjs` observables so you could have feedback about the process or even show your users the progress of the upload. - -For that reason when file upload functions return `PromisedObservable`: - -```typescript -export type UploadFileStatus = { - type: 'file' - progress: number - cid?: CID -} -``` - -Being the cid only returned (and thus optional) when the upload is completed. - -Similarly, for folder uploads the functions return `PromisedObservable` - -```ts -export type UploadFolderStatus = { - type: 'folder' - progress: number - cid?: CID -} -``` - -**e.g Show upload progress in React** - -```typescript -const [progress, setProgress] = useState(0) +const folderCID = await uploadFolderFromFolderPath(api, folderPath, options) -useEffect(async () => { - const finalStatus = await uploadFileFromInput(api, genericFile, options).forEach((status) => { - setProgress(status.progress) - }) -}) +console.log(`The folder is uploaded and its cid is ${folderCID}`) ``` -**e.g Ignore observables** - -Other users may want to not use the progress observability. For having a promise instead the field `promise` is a Promise that resolves into `UploadFileStatus` and `UploadFolderStatus` for files and folders respectively. - -e.g - -```ts -const status = await uploadFileFromInput(api, genericFile, options).promise -const cid = status.cid -``` +**Note: If a folder is tried to be encrypted a zip file would be generated and that file would be encrypted and uploaded.** ### Example Usage of Download Here is an example of how to use the `downloadFile` method to download a file from the server: ```typescript -import { downloadObject } from '@autonomys/auto-drive' +import { createAutoDriveApi, downloadFile } from '@autonomys/auto-drive' const api = createAutoDriveApi({ apiKey: 'your-api-key' }) // Initialize your API instance with API key try { const cid = '..' - const stream = await downloadObject(api, { cid }) + const stream = await downloadFile(api, cid) let file = Buffer.alloc(0) for await (const chunk of stream) { file = Buffer.concat([file, chunk]) @@ -205,16 +159,20 @@ try { Here is an example of how to use the `getRoots` method to retrieve the root directories: ```typescript -import { createAutoDriveApi, downloadObject } from '@autonomys/auto-drive' -import fs from 'fs' +import { createAutoDriveApi, apiCalls, Scope } from '@autonomys/auto-drive' const api = createAutoDriveApi({ apiKey: 'your-api-key' }) // Initialize your API instance with API key try { - const stream = fs.createWriteStream('/path/to/file') - const asyncBuffer = await downloadObject(api, { cid }) - for await (const buffer of asyncBuffer) { - stream.write(buffer) + const myFiles = await apiCalls.getRoots(api, { + scope: Scope.User, + limit: 100, + offset: 0, + }) + + console.log(`Retrieved ${myFiles.rows.length} files of ${myFiles.totalCount} total`) + for (const file of myFiles.rows) { + console.log(`${file.name} - ${file.headCid}: ${file.size}`) } } catch (error) { console.error('Error downloading file:', error) diff --git a/packages/auto-drive/package.json b/packages/auto-drive/package.json index fe254c5..58e771e 100644 --- a/packages/auto-drive/package.json +++ b/packages/auto-drive/package.json @@ -46,7 +46,6 @@ "jszip": "^3.10.1", "mime-types": "^2.1.35", "process": "^0.11.10", - "rxjs": "^7.8.1", "stream": "^0.0.3", "zod": "^3.23.8" }, diff --git a/packages/auto-drive/src/api/wrappers.ts b/packages/auto-drive/src/api/wrappers.ts index 043f58a..9516eca 100644 --- a/packages/auto-drive/src/api/wrappers.ts +++ b/packages/auto-drive/src/api/wrappers.ts @@ -1,16 +1,15 @@ import mime from 'mime-types' import { asyncByChunk, asyncFromStream, fileToIterable } from '../utils/async' import { progressToPercentage } from '../utils/misc' -import { PromisedObservable } from '../utils/observable' import { apiCalls } from './calls/index' import { AutoDriveApi } from './connection' import { GenericFile, GenericFileWithinFolder } from './models/file' import { constructFromInput, constructZipBlobFromTreeAndPaths } from './models/folderTree' -import { UploadChunksStatus, UploadFileStatus, UploadFolderStatus } from './models/uploads' export type UploadFileOptions = { password?: string compression?: boolean + onProgress?: (progress: number) => void } const UPLOAD_FILE_CHUNK_SIZE = 1024 * 1024 @@ -20,17 +19,23 @@ const uploadFileChunks = ( fileUploadId: string, asyncIterable: AsyncIterable, uploadChunkSize: number = UPLOAD_FILE_CHUNK_SIZE, -): PromisedObservable => { - return new PromisedObservable(async (subscriber) => { - let index = 0 - let uploadBytes = 0 - for await (const chunk of asyncByChunk(asyncIterable, uploadChunkSize)) { - await apiCalls.uploadFileChunk(api, { uploadId: fileUploadId, chunk, index }) - uploadBytes += chunk.length - subscriber.next({ uploadBytes }) - index++ + onProgress?: (uploadedBytes: number) => void, +): Promise => { + return new Promise(async (resolve, reject) => { + try { + let index = 0 + let uploadBytes = 0 + for await (const chunk of asyncByChunk(asyncIterable, uploadChunkSize)) { + await apiCalls.uploadFileChunk(api, { uploadId: fileUploadId, chunk, index }) + uploadBytes += chunk.length + onProgress?.(uploadBytes) + index++ + } + + resolve() + } catch (e) { + reject(e) } - subscriber.complete() }) } @@ -56,11 +61,12 @@ export const uploadFileFromInput = ( file: File, options: UploadFileOptions = {}, uploadChunkSize?: number, -): PromisedObservable => { +): Promise => { const { password = undefined, compression = true } = options - return new PromisedObservable(async (subscriber) => { - const { stringToCid, compressFile, CompressionAlgorithm, encryptFile, EncryptionAlgorithm } = - await import('@autonomys/auto-dag-data') + return new Promise(async (resolve, reject) => { + const { compressFile, CompressionAlgorithm, encryptFile, EncryptionAlgorithm } = await import( + '@autonomys/auto-dag-data' + ) let asyncIterable: AsyncIterable = fileToIterable(file) if (compression) { @@ -95,14 +101,13 @@ export const uploadFileFromInput = ( uploadOptions, }) - await uploadFileChunks(api, fileUpload.id, asyncIterable, uploadChunkSize).forEach((e) => - subscriber.next({ type: 'file', progress: progressToPercentage(e.uploadBytes, file.size) }), - ) + await uploadFileChunks(api, fileUpload.id, asyncIterable, uploadChunkSize, (bytes) => { + options.onProgress?.(progressToPercentage(bytes, file.size)) + }) const result = await apiCalls.completeUpload(api, { uploadId: fileUpload.id }) - subscriber.next({ type: 'file', progress: 100, cid: result.cid }) - subscriber.complete() + resolve(result.cid) }) } @@ -123,60 +128,58 @@ export const uploadFileFromInput = ( * @returns {PromisedObservable} - An observable that emits the upload status. * @throws {Error} - Throws an error if the upload fails at any stage. */ -export const uploadFile = ( +export const uploadFile = async ( api: AutoDriveApi, file: GenericFile, options: UploadFileOptions = {}, uploadChunkSize?: number, -): PromisedObservable => { +): Promise => { const { password = undefined, compression = true } = options - return new PromisedObservable(async (subscriber) => { - const { stringToCid, compressFile, CompressionAlgorithm, encryptFile, EncryptionAlgorithm } = - await import('@autonomys/auto-dag-data') - let asyncIterable: AsyncIterable = file.read() - - if (compression) { - asyncIterable = compressFile(asyncIterable, { - level: 9, - algorithm: CompressionAlgorithm.ZLIB, - }) - } - - if (password) { - asyncIterable = encryptFile(asyncIterable, password, { - algorithm: EncryptionAlgorithm.AES_256_GCM, - }) - } + const { compressFile, CompressionAlgorithm, encryptFile, EncryptionAlgorithm } = await import( + '@autonomys/auto-dag-data' + ) + let asyncIterable: AsyncIterable = file.read() - const uploadOptions = { - compression: compression - ? { - level: 9, - algorithm: CompressionAlgorithm.ZLIB, - } - : undefined, - encryption: password - ? { - algorithm: EncryptionAlgorithm.AES_256_GCM, - } - : undefined, - } - const fileUpload = await apiCalls.createFileUpload(api, { - mimeType: mime.lookup(file.name) || undefined, - filename: file.name, - uploadOptions, + if (compression) { + asyncIterable = compressFile(asyncIterable, { + level: 9, + algorithm: CompressionAlgorithm.ZLIB, }) + } - await uploadFileChunks(api, fileUpload.id, asyncIterable, uploadChunkSize).forEach((e) => - subscriber.next({ type: 'file', progress: progressToPercentage(e.uploadBytes, file.size) }), - ) + if (password) { + asyncIterable = encryptFile(asyncIterable, password, { + algorithm: EncryptionAlgorithm.AES_256_GCM, + }) + } - const result = await apiCalls.completeUpload(api, { uploadId: fileUpload.id }) + const uploadOptions = { + compression: compression + ? { + level: 9, + algorithm: CompressionAlgorithm.ZLIB, + } + : undefined, + encryption: password + ? { + algorithm: EncryptionAlgorithm.AES_256_GCM, + } + : undefined, + } + const fileUpload = await apiCalls.createFileUpload(api, { + mimeType: mime.lookup(file.name) || undefined, + filename: file.name, + uploadOptions, + }) - subscriber.next({ type: 'file', progress: 100, cid: result.cid }) - subscriber.complete() + await uploadFileChunks(api, fileUpload.id, asyncIterable, uploadChunkSize, (bytes) => { + options.onProgress?.(progressToPercentage(bytes, file.size)) }) + + const result = await apiCalls.completeUpload(api, { uploadId: fileUpload.id }) + + return result.cid } /** @@ -199,8 +202,12 @@ export const uploadFile = ( export const uploadFolderFromInput = async ( api: AutoDriveApi, fileList: FileList | File[], - { uploadChunkSize, password }: { uploadChunkSize?: number; password?: string } = {}, -): Promise> => { + { + uploadChunkSize, + password, + onProgress, + }: { uploadChunkSize?: number; password?: string; onProgress?: (progress: number) => void } = {}, +): Promise => { const files = fileList instanceof FileList ? Array.from(fileList) : fileList const fileTree = constructFromInput(files) @@ -223,45 +230,42 @@ export const uploadFolderFromInput = async ( { password, compression: true, + onProgress, }, ) } - return new PromisedObservable(async (subscriber) => { - // Otherwise, we upload the files as a folder w/o compression or encryption - const folderUpload = await apiCalls.createFolderUpload(api, { - fileTree, - }) + // Otherwise, we upload the files as a folder w/o compression or encryption + const folderUpload = await apiCalls.createFolderUpload(api, { + fileTree, + }) - let currentBytesUploaded = 0 - const totalSize = files.reduce((acc, file) => acc + file.size, 0) - for (const file of files) { - await uploadFileWithinFolderUpload( - api, - folderUpload.id, - { - read: () => fileToIterable(file), - name: file.name, - mimeType: mime.lookup(file.name) || undefined, - size: file.size, - path: file.webkitRelativePath, + let currentBytesUploaded = 0 + const totalSize = files.reduce((acc, file) => acc + file.size, 0) + for (const file of files) { + await uploadFileWithinFolderUpload( + api, + folderUpload.id, + { + read: () => fileToIterable(file), + name: file.name, + mimeType: mime.lookup(file.name) || undefined, + size: file.size, + path: file.webkitRelativePath, + }, + uploadChunkSize, + { + onProgress: (progress) => { + onProgress?.(progressToPercentage(currentBytesUploaded + progress, totalSize)) }, - uploadChunkSize, - ).forEach((e) => { - subscriber.next({ - type: 'folder', - progress: progressToPercentage(currentBytesUploaded + e.uploadBytes, totalSize), - }) - }) - - currentBytesUploaded += file.size - } + }, + ) + currentBytesUploaded += file.size + } - const result = await apiCalls.completeUpload(api, { uploadId: folderUpload.id }) + const result = await apiCalls.completeUpload(api, { uploadId: folderUpload.id }) - subscriber.next({ type: 'folder', progress: 100, cid: result.cid }) - subscriber.complete() - }) + return result.cid } /** @@ -273,29 +277,26 @@ export const uploadFolderFromInput = async ( * * @returns {Promise} A promise that resolves when the file upload is complete. */ -export const uploadFileWithinFolderUpload = ( +export const uploadFileWithinFolderUpload = async ( api: AutoDriveApi, uploadId: string, file: GenericFileWithinFolder, uploadChunkSize?: number, -): PromisedObservable => { - return new PromisedObservable(async (subscriber) => { - const fileUpload = await apiCalls.createFileUploadWithinFolderUpload(api, { - uploadId, - name: file.name, - mimeType: file.mimeType, - relativeId: file.path, - uploadOptions: {}, - }) + options: Pick = {}, +): Promise => { + const fileUpload = await apiCalls.createFileUploadWithinFolderUpload(api, { + uploadId, + name: file.name, + mimeType: file.mimeType, + relativeId: file.path, + uploadOptions: {}, + }) - await uploadFileChunks(api, fileUpload.id, file.read(), uploadChunkSize).forEach((e) => - subscriber.next({ uploadBytes: e.uploadBytes }), - ) + await uploadFileChunks(api, fileUpload.id, file.read(), uploadChunkSize, options.onProgress) - await apiCalls.completeUpload(api, { uploadId: fileUpload.id }) + const result = await apiCalls.completeUpload(api, { uploadId: fileUpload.id }) - subscriber.complete() - }) + return result.cid } /** diff --git a/packages/auto-drive/src/fs/wrappers.ts b/packages/auto-drive/src/fs/wrappers.ts index 3cef329..ec0f05a 100644 --- a/packages/auto-drive/src/fs/wrappers.ts +++ b/packages/auto-drive/src/fs/wrappers.ts @@ -4,11 +4,10 @@ import { AutoDriveApi } from '../api/connection.js' import { apiCalls } from '../api/index.js' import { GenericFileWithinFolder } from '../api/models/file.js' import { constructFromFileSystemEntries } from '../api/models/folderTree.js' -import { UploadFileStatus, UploadFolderStatus } from '../api/models/uploads.js' +import { CompressionAlgorithm } from '../api/models/uploads.js' import { uploadFile, UploadFileOptions, uploadFileWithinFolderUpload } from '../api/wrappers.js' import { fileToIterable } from '../utils/index.js' import { progressToPercentage } from '../utils/misc.js' -import { PromisedObservable } from '../utils/observable.js' import { constructZipFromTreeAndFileSystemPaths, getFiles } from './utils.js' /** @@ -33,8 +32,8 @@ export const uploadFileFromFilepath = ( filePath: string, options: UploadFileOptions = {}, uploadChunkSize?: number, -): PromisedObservable => { - const { password = undefined, compression = true } = options +): Promise => { + const { password = undefined, compression = true, onProgress } = options const name = filePath.split('/').pop()! return uploadFile( @@ -48,6 +47,7 @@ export const uploadFileFromFilepath = ( { password, compression, + onProgress, }, uploadChunkSize, ) @@ -74,8 +74,16 @@ export const uploadFileFromFilepath = ( export const uploadFolderFromFolderPath = async ( api: AutoDriveApi, folderPath: string, - { uploadChunkSize, password }: { uploadChunkSize?: number; password?: string } = {}, -): Promise> => { + { + uploadChunkSize, + password, + onProgress, + }: { + uploadChunkSize?: number + password?: string + onProgress?: (progressInPercentage: number) => void + } = {}, +): Promise> => { const files = await getFiles(folderPath) const fileTree = constructFromFileSystemEntries(files) @@ -94,46 +102,43 @@ export const uploadFolderFromFolderPath = async ( { password, compression: true, + onProgress: (progressInPercentage) => { + onProgress?.(progressToPercentage(progressInPercentage, zipBlob.size)) + }, }, ) } - return new PromisedObservable(async (subscriber) => { - const { CompressionAlgorithm } = await import('@autonomys/auto-dag-data') - const folderUpload = await apiCalls.createFolderUpload(api, { - fileTree, - uploadOptions: { - compression: { - algorithm: CompressionAlgorithm.ZLIB, - level: 9, - }, + const folderUpload = await apiCalls.createFolderUpload(api, { + fileTree, + uploadOptions: { + compression: { + algorithm: CompressionAlgorithm.ZLIB, + level: 9, }, - }) - - const genericFiles: GenericFileWithinFolder[] = files.map((file) => ({ - read: () => fs.createReadStream(file), - name: file.split('/').pop()!, - mimeType: mime.lookup(file.split('/').pop()!) || undefined, - size: fs.statSync(file).size, - path: file, - })) + }, + }) - const totalSize = genericFiles.reduce((acc, file) => acc + file.size, 0) + const genericFiles: GenericFileWithinFolder[] = files.map((file) => ({ + read: () => fs.createReadStream(file), + name: file.split('/').pop()!, + mimeType: mime.lookup(file.split('/').pop()!) || undefined, + size: fs.statSync(file).size, + path: file, + })) - let progress = 0 - for (const file of genericFiles) { - await uploadFileWithinFolderUpload(api, folderUpload.id, file, uploadChunkSize).forEach((e) => - subscriber.next({ - type: 'folder', - progress: progressToPercentage(progress + e.uploadBytes, totalSize), - }), - ) - progress += file.size - } + const totalSize = genericFiles.reduce((acc, file) => acc + file.size, 0) + let progress = 0 + for (const file of genericFiles) { + await uploadFileWithinFolderUpload(api, folderUpload.id, file, uploadChunkSize, { + onProgress: (uploadedBytes) => { + onProgress?.(progressToPercentage(progress + uploadedBytes, totalSize)) + }, + }) + progress += file.size + } - const result = await apiCalls.completeUpload(api, { uploadId: folderUpload.id }) + const result = await apiCalls.completeUpload(api, { uploadId: folderUpload.id }) - subscriber.next({ type: 'folder', progress: 100, cid: result.cid }) - subscriber.complete() - }) + return result.cid } diff --git a/packages/auto-drive/src/utils/index.ts b/packages/auto-drive/src/utils/index.ts index 77a1171..eb87879 100644 --- a/packages/auto-drive/src/utils/index.ts +++ b/packages/auto-drive/src/utils/index.ts @@ -1,4 +1,3 @@ export * from './async' export * from './misc' -export * from './observable' export * from './types' diff --git a/packages/auto-drive/src/utils/observable.ts b/packages/auto-drive/src/utils/observable.ts deleted file mode 100644 index e687c5b..0000000 --- a/packages/auto-drive/src/utils/observable.ts +++ /dev/null @@ -1,19 +0,0 @@ -import * as rxjs from 'rxjs' - -const asyncCallback = (callback: (t: T) => O) => { - return (t: T) => { - callback(t) - } -} - -export class PromisedObservable extends rxjs.Observable { - constructor(subscribe?: (this: rxjs.Observable, subscriber: rxjs.Subscriber) => void) { - super(subscribe && asyncCallback(subscribe)) - } - - get promise(): Promise { - return lastValueFrom(this) - } -} - -export const { firstValueFrom, lastValueFrom } = rxjs diff --git a/packages/auto-drive/tsconfig.json b/packages/auto-drive/tsconfig.json index 49e05ce..b4430d1 100644 --- a/packages/auto-drive/tsconfig.json +++ b/packages/auto-drive/tsconfig.json @@ -2,7 +2,9 @@ "extends": "../../tsconfig.json", "compilerOptions": { "outDir": "./dist", - "rootDir": "./src" + "rootDir": "./src", + "module": "Node16", + "moduleResolution": "Node16" }, "include": ["src"] } diff --git a/yarn.lock b/yarn.lock index 21f0829..bc065f9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -82,7 +82,6 @@ __metadata: process: "npm:^0.11.10" rollup-plugin-jscc: "npm:^2.0.0" rollup-plugin-terser: "npm:^7.0.2" - rxjs: "npm:^7.8.1" stream: "npm:^0.0.3" tslib: "npm:^2.8.1" typescript: "npm:^5.6.3"