-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #144 from autonomys/update/separate-drive-pkg
Update pkg name and added encryption & compression logic
- Loading branch information
Showing
36 changed files
with
383 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| import { Unzlib, Zlib } from 'fflate' | ||
| import type { AwaitIterable } from 'interface-store' | ||
| import { CompressionAlgorithm } from '../metadata/index.js' | ||
| import { asyncByChunk } from '../utils/async.js' | ||
| import type { PickPartial } from '../utils/types.js' | ||
| import { CompressionOptions } from './types.js' | ||
|
|
||
| export const COMPRESSION_CHUNK_SIZE = 1024 * 1024 | ||
|
|
||
| export async function* compressFile( | ||
| file: AwaitIterable<Buffer>, | ||
| { | ||
| level = 9, | ||
| chunkSize = COMPRESSION_CHUNK_SIZE, | ||
| algorithm, | ||
| }: PickPartial<CompressionOptions, 'algorithm'>, | ||
| ): AsyncIterable<Buffer> { | ||
| if (algorithm !== CompressionAlgorithm.ZLIB) { | ||
| throw new Error('Unsupported compression algorithm') | ||
| } | ||
| if (level < 0 || level > 9) { | ||
| throw new Error('Invalid compression level') | ||
| } | ||
| if (chunkSize <= 0) { | ||
| throw new Error('Invalid chunk size') | ||
| } | ||
|
|
||
| const zlib = new Zlib({ level }) | ||
| const compressedChunks: Buffer[] = [] | ||
|
|
||
| zlib.ondata = (chunk) => { | ||
| compressedChunks.push(Buffer.from(chunk)) | ||
| } | ||
|
|
||
| for await (const chunk of asyncByChunk(file, chunkSize)) { | ||
| zlib.push(chunk, false) | ||
| while (compressedChunks.length > 0) { | ||
| yield compressedChunks.shift()! | ||
| } | ||
| } | ||
|
|
||
| zlib.push(new Uint8Array(), true) | ||
| while (compressedChunks.length > 0) { | ||
| yield compressedChunks.shift()! | ||
| } | ||
| } | ||
|
|
||
| export async function* decompressFile( | ||
| compressedFile: AwaitIterable<Buffer>, | ||
| { | ||
| chunkSize = COMPRESSION_CHUNK_SIZE, | ||
| algorithm = CompressionAlgorithm.ZLIB, | ||
| level = 9, | ||
| }: PickPartial<CompressionOptions, 'algorithm'>, | ||
| ): AsyncIterable<Buffer> { | ||
| if (algorithm !== CompressionAlgorithm.ZLIB) { | ||
| throw new Error('Unsupported compression algorithm') | ||
| } | ||
| if (chunkSize <= 0) { | ||
| throw new Error('Invalid chunk size') | ||
| } | ||
| if (level < 0 || level > 9) { | ||
| throw new Error('Invalid compression level') | ||
| } | ||
|
|
||
| const unzlib = new Unzlib() | ||
| const decompressedChunks: Buffer[] = [] | ||
|
|
||
| unzlib.ondata = (chunk) => { | ||
| decompressedChunks.push(Buffer.from(chunk)) | ||
| } | ||
|
|
||
| for await (const chunk of asyncByChunk(compressedFile, chunkSize)) { | ||
| unzlib.push(chunk, false) | ||
| while (decompressedChunks.length > 0) { | ||
| yield decompressedChunks.shift()! | ||
| } | ||
| } | ||
|
|
||
| unzlib.push(new Uint8Array(), true) | ||
| while (decompressedChunks.length > 0) { | ||
| yield decompressedChunks.shift()! | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| import { CompressionAlgorithm } from '../metadata/index.js' | ||
|
|
||
| export type CompressionLevel = 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | ||
|
|
||
| export type ZLibOptions = { | ||
| algorithm: CompressionAlgorithm.ZLIB | ||
| level: CompressionLevel | ||
| chunkSize: number | ||
| } | ||
|
|
||
| export type CompressionOptions = ZLibOptions |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| import { Crypto } from '@peculiar/webcrypto' | ||
| import { randomBytes } from 'crypto' | ||
| import { EncryptionAlgorithm, EncryptionOptions } from '../metadata/index.js' | ||
| import { asyncByChunk } from '../utils/async.js' | ||
| import type { PickPartial } from '../utils/types.js' | ||
| import { PasswordGenerationOptions } from './types.js' | ||
|
|
||
| const crypto = new Crypto() | ||
|
|
||
| export const ENCRYPTING_CHUNK_SIZE = 1024 * 1024 | ||
| const IV_SIZE = 16 | ||
| const TAG_SIZE = 16 | ||
| const ENCRYPTED_CHUNK_SIZE = ENCRYPTING_CHUNK_SIZE + IV_SIZE + TAG_SIZE | ||
| const SALT_SIZE = 32 | ||
|
|
||
| export const getKeyFromPassword = async ({ password, salt }: PasswordGenerationOptions) => { | ||
| const encoder = new TextEncoder() | ||
| const saltHash = | ||
| typeof salt === 'string' ? await crypto.subtle.digest('SHA-256', encoder.encode(salt)) : salt | ||
|
|
||
| const keyMaterial = await crypto.subtle.importKey( | ||
| 'raw', | ||
| encoder.encode(password), | ||
| 'PBKDF2', | ||
| false, | ||
| ['deriveBits', 'deriveKey'], | ||
| ) | ||
|
|
||
| return crypto.subtle.deriveKey( | ||
| { | ||
| name: 'PBKDF2', | ||
| salt: saltHash, | ||
| iterations: 100000, | ||
| hash: 'SHA-256', | ||
| }, | ||
| keyMaterial, | ||
| { name: 'AES-GCM', length: 256 }, | ||
| false, | ||
| ['encrypt', 'decrypt'], | ||
| ) | ||
| } | ||
|
|
||
| export const encryptFile = async function* ( | ||
| file: AsyncIterable<Buffer>, | ||
| password: string, | ||
| { chunkSize = ENCRYPTING_CHUNK_SIZE, algorithm }: PickPartial<EncryptionOptions, 'algorithm'>, | ||
| ): AsyncIterable<Buffer> { | ||
| if (algorithm !== EncryptionAlgorithm.AES_256_GCM) { | ||
| throw new Error('Unsupported encryption algorithm') | ||
| } | ||
|
|
||
| const salt = randomBytes(SALT_SIZE) | ||
| const key = await getKeyFromPassword({ password, salt }) | ||
|
|
||
| yield salt | ||
|
|
||
| for await (const chunk of asyncByChunk(file, chunkSize)) { | ||
| const iv = crypto.getRandomValues(new Uint8Array(IV_SIZE)) | ||
| const encrypted = await crypto.subtle.encrypt({ name: 'AES-GCM', iv }, key, chunk) | ||
| yield Buffer.concat([Buffer.from(iv), Buffer.from(encrypted)]) | ||
| } | ||
| } | ||
|
|
||
| export const decryptFile = async function* ( | ||
| file: AsyncIterable<Buffer>, | ||
| password: string, | ||
| { chunkSize = ENCRYPTED_CHUNK_SIZE, algorithm }: PickPartial<EncryptionOptions, 'algorithm'>, | ||
| ): AsyncIterable<Buffer> { | ||
| if (algorithm !== EncryptionAlgorithm.AES_256_GCM) { | ||
| throw new Error('Unsupported encryption algorithm') | ||
| } | ||
|
|
||
| let key: CryptoKey | undefined = undefined | ||
| let chunks = Buffer.alloc(0) | ||
| for await (const chunk of file) { | ||
| chunks = Buffer.concat([chunks, chunk]) | ||
|
|
||
| if (chunks.length >= SALT_SIZE && !key) { | ||
| const salt = chunks.subarray(0, 32) | ||
| key = await getKeyFromPassword({ password, salt }) | ||
| chunks = chunks.subarray(SALT_SIZE) | ||
| } | ||
|
|
||
| while (key && chunks.length >= chunkSize) { | ||
| const iv = chunks.subarray(0, IV_SIZE) | ||
| const encryptedChunk = chunk.subarray(IV_SIZE, chunkSize) | ||
| const decrypted = await crypto.subtle.decrypt({ name: 'AES-GCM', iv }, key, encryptedChunk) | ||
| chunks = chunks.subarray(chunkSize) | ||
| yield Buffer.from(decrypted) | ||
| } | ||
| } | ||
|
|
||
| if (key && chunks.length > 0) { | ||
| const iv = chunks.subarray(0, IV_SIZE) | ||
| const encryptedChunk = chunks.subarray(IV_SIZE, chunkSize) | ||
| const decrypted = await crypto.subtle.decrypt({ name: 'AES-GCM', iv }, key, encryptedChunk) | ||
| yield Buffer.from(decrypted) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| export type PasswordGenerationOptions = { | ||
| password: string | ||
| salt: string | Uint8Array | ||
| } |
2 changes: 2 additions & 0 deletions
2
packages/auto-drive/src/index.ts → packages/auto-dag-data/src/index.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,5 @@ | ||
| export * from './cid/index.js' | ||
| export * from './compression/index.js' | ||
| export * from './encryption/index.js' | ||
| export * from './ipld/index.js' | ||
| export * from './metadata/index.js' |
2 changes: 1 addition & 1 deletion
2
packages/auto-drive/src/ipld/builders.ts → packages/auto-dag-data/src/ipld/builders.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
File renamed without changes.
2 changes: 1 addition & 1 deletion
2
packages/auto-drive/src/ipld/nodes.ts → packages/auto-dag-data/src/ipld/nodes.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Oops, something went wrong.