From f5c963fdeeb504b3dac7b3c8e9fcac1d679285d8 Mon Sep 17 00:00:00 2001 From: Carlos Lostao Date: Fri, 25 Oct 2024 21:15:54 +0200 Subject: [PATCH] chore: remove custom blockstore --- .../auto-drive/src/ipld/blockstore/base.ts | 9 ----- .../auto-drive/src/ipld/blockstore/index.ts | 2 -- .../auto-drive/src/ipld/blockstore/memory.ts | 33 ------------------- packages/auto-drive/src/ipld/chunker.ts | 24 +++++++------- packages/auto-drive/src/ipld/index.ts | 1 - 5 files changed, 11 insertions(+), 58 deletions(-) delete mode 100644 packages/auto-drive/src/ipld/blockstore/base.ts delete mode 100644 packages/auto-drive/src/ipld/blockstore/index.ts delete mode 100644 packages/auto-drive/src/ipld/blockstore/memory.ts diff --git a/packages/auto-drive/src/ipld/blockstore/base.ts b/packages/auto-drive/src/ipld/blockstore/base.ts deleted file mode 100644 index 5f73c922..00000000 --- a/packages/auto-drive/src/ipld/blockstore/base.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { BaseBlockstore } from 'blockstore-core' -import type { Pair } from 'interface-blockstore' -import { AbortOptions, AwaitIterable } from 'interface-store' -import { MetadataType } from '../../metadata/index.js' - -export interface IPLDBlockstore extends BaseBlockstore { - getFilteredMany(nodeType: MetadataType, options?: AbortOptions): AwaitIterable - getSize(cid: Pair['cid']): Promise -} diff --git a/packages/auto-drive/src/ipld/blockstore/index.ts b/packages/auto-drive/src/ipld/blockstore/index.ts deleted file mode 100644 index 4d7dc570..00000000 --- a/packages/auto-drive/src/ipld/blockstore/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './base.js' -export * from './memory.js' diff --git a/packages/auto-drive/src/ipld/blockstore/memory.ts b/packages/auto-drive/src/ipld/blockstore/memory.ts deleted file mode 100644 index 1dc1521f..00000000 --- a/packages/auto-drive/src/ipld/blockstore/memory.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { MemoryBlockstore } from 'blockstore-core' -import { Pair } from 'interface-blockstore' -import { AbortOptions, AwaitIterable } from 'interface-store' -import { CID, Version } from 'multiformats' -import { decodeIPLDNodeData, IPLDNodeData, MetadataType } from '../../metadata/index.js' -import { IPLDBlockstore } from './base.js' - -export class MemoryIPLDBlockstore extends MemoryBlockstore implements IPLDBlockstore { - private readonly nodeByType = new Map() - async *getFilteredMany( - nodeType: MetadataType, - options?: AbortOptions, - ): AwaitIterable { - for (const cid of this.nodeByType.get(nodeType) ?? []) { - yield cid - } - } - - async put( - key: CID, - val: Uint8Array, - ): Promise> { - const data = decodeIPLDNodeData(val) - this.nodeByType.set(data.type, [...(this.nodeByType.get(data.type) ?? []), key]) - return super.put(key, val) - } - - async getSize(cid: CID): Promise { - const bytes = await this.get(cid) - const data = decodeIPLDNodeData(bytes) - return data.size ?? 0 - } -} diff --git a/packages/auto-drive/src/ipld/chunker.ts b/packages/auto-drive/src/ipld/chunker.ts index 997b2b7a..6a964172 100644 --- a/packages/auto-drive/src/ipld/chunker.ts +++ b/packages/auto-drive/src/ipld/chunker.ts @@ -4,7 +4,6 @@ import { AwaitIterable } from 'interface-store' import { CID } from 'multiformats' import { cidOfNode, cidToString } from '../cid/index.js' import { decodeIPLDNodeData, IPLDNodeData, OffchainMetadata } from '../metadata/index.js' -import { IPLDBlockstore } from './blockstore/base.js' import { Builders, fileBuilders, metadataBuilders } from './builders.js' import { createFolderInlinkIpldNode, createFolderIpldNode } from './nodes.js' import { chunkBuffer, encodeNode } from './utils.js' @@ -13,22 +12,23 @@ export const DEFAULT_MAX_CHUNK_SIZE = 1024 * 64 export const DEFAULT_MAX_LINK_PER_NODE = DEFAULT_MAX_CHUNK_SIZE / 64 export const processFileToIPLDFormat = ( - blockstore: IPLDBlockstore, + blockstore: BaseBlockstore, file: AwaitIterable, + totalSize: number, filename?: string, { maxChunkSize, maxLinkPerNode }: { maxChunkSize: number; maxLinkPerNode: number } = { maxChunkSize: DEFAULT_MAX_CHUNK_SIZE, maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE, }, ): Promise => { - return processBufferToIPLDFormat(blockstore, file, filename, fileBuilders, { + return processBufferToIPLDFormat(blockstore, file, filename, totalSize, fileBuilders, { maxChunkSize, maxLinkPerNode, }) } export const processMetadataToIPLDFormat = async ( - blockstore: IPLDBlockstore, + blockstore: BaseBlockstore, metadata: OffchainMetadata, limits: { maxChunkSize: number; maxLinkPerNode: number } = { maxChunkSize: DEFAULT_MAX_CHUNK_SIZE, @@ -43,15 +43,17 @@ export const processMetadataToIPLDFormat = async ( yield buffer })(), name, + buffer.byteLength, metadataBuilders, limits, ) } const processBufferToIPLDFormat = async ( - blockstore: IPLDBlockstore, + blockstore: BaseBlockstore, buffer: AwaitIterable, filename: string | undefined, + totalSize: number, builders: Builders, { maxChunkSize, maxLinkPerNode }: { maxChunkSize: number; maxLinkPerNode: number } = { maxChunkSize: DEFAULT_MAX_CHUNK_SIZE, @@ -59,39 +61,35 @@ const processBufferToIPLDFormat = async ( }, ): Promise => { const bufferChunks = chunkBuffer(buffer, { maxChunkSize }) - let totalSize = 0 let CIDs: CID[] = [] for await (const chunk of bufferChunks) { const node = builders.chunk(chunk) const cid = cidOfNode(node) await blockstore.put(cid, encodeNode(node)) - totalSize += chunk.byteLength CIDs.push(cid) } - return processBufferToIPLDFormatFromChunks(blockstore, CIDs, filename, builders, { + return processBufferToIPLDFormatFromChunks(blockstore, CIDs, filename, totalSize, builders, { maxLinkPerNode, maxChunkSize, }) } export const processBufferToIPLDFormatFromChunks = async ( - blockstore: IPLDBlockstore, + blockstore: BaseBlockstore, chunks: AwaitIterable, filename: string | undefined, + totalSize: number, builders: Builders, { maxLinkPerNode, maxChunkSize }: { maxLinkPerNode: number; maxChunkSize: number } = { maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE, maxChunkSize: DEFAULT_MAX_CHUNK_SIZE, }, ): Promise => { - let totalSize = 0 let chunkCount = 0 let CIDs: CID[] = [] for await (const chunk of chunks) { - totalSize += await blockstore.getSize(chunk) - CIDs.push(chunk) chunkCount++ } @@ -162,7 +160,7 @@ export const processFolderToIPLDFormat = async ( * @returns the last chunk if it's not full, otherwise an empty buffer */ export const processChunksToIPLDFormat = async ( - blockstore: IPLDBlockstore, + blockstore: BaseBlockstore, chunks: AwaitIterable, builders: Builders, { maxChunkSize = DEFAULT_MAX_CHUNK_SIZE }: { maxChunkSize?: number }, diff --git a/packages/auto-drive/src/ipld/index.ts b/packages/auto-drive/src/ipld/index.ts index c775d8d3..e9994d86 100644 --- a/packages/auto-drive/src/ipld/index.ts +++ b/packages/auto-drive/src/ipld/index.ts @@ -1,4 +1,3 @@ -export * from './blockstore/index.js' export * from './builders.js' export * from './chunker.js' export * from './nodes.js'