diff --git a/packages/codecs-data-structures/src/__tests__/array-test.ts b/packages/codecs-data-structures/src/__tests__/array-test.ts index 6167958dc01d..da3183b7690b 100644 --- a/packages/codecs-data-structures/src/__tests__/array-test.ts +++ b/packages/codecs-data-structures/src/__tests__/array-test.ts @@ -79,15 +79,15 @@ describe('getArrayCodec', () => { expect(array(string({ size: 1 }), remainder).encode(['a', 'b'])).toStrictEqual(b('6162')); expect(array(string({ size: 1 }), remainder).read(b('6162'), 0)).toStrictEqual([['a', 'b'], 2]); + // Variable sized items. + expect(array(string({ size: u8() }), remainder).encode(['a', 'bc'])).toStrictEqual(b('0161026263')); + expect(array(string({ size: u8() }), remainder).read(b('0161026263'), 0)).toStrictEqual([['a', 'bc'], 5]); + // Different From and To types. const arrayU64 = array(u64(), remainder); expect(arrayU64.encode([2])).toStrictEqual(b('0200000000000000')); expect(arrayU64.encode([2n])).toStrictEqual(b('0200000000000000')); expect(arrayU64.read(b('0200000000000000'), 0)).toStrictEqual([[2n], 8]); - - // It fails with variable size items. - // @ts-expect-error Remainder size cannot be used with fixed-size items. - expect(() => array(string(), remainder)).toThrow('Codecs of "remainder" size must have fixed-size items'); }); it('has the right sizes', () => { diff --git a/packages/codecs-data-structures/src/__tests__/map-test.ts b/packages/codecs-data-structures/src/__tests__/map-test.ts index e906c3d96d0f..cf636dfd1552 100644 --- a/packages/codecs-data-structures/src/__tests__/map-test.ts +++ b/packages/codecs-data-structures/src/__tests__/map-test.ts @@ -95,15 +95,20 @@ describe('getMapCodec', () => { expect(letters.encode(lettersMap)).toStrictEqual(b('61016202')); expect(letters.read(b('61016202'), 0)).toStrictEqual([lettersMap, 4]); + // Variable sized items. + const prefixedLetters = map(string({ size: u8() }), u8(), remainder); + const prefixedLettersMap = new Map([ + ['a', 6], + ['bc', 7], + ]); + expect(prefixedLetters.encode(prefixedLettersMap)).toStrictEqual(b('01610602626307')); + expect(prefixedLetters.read(b('01610602626307'), 0)).toStrictEqual([prefixedLettersMap, 7]); + // Different From and To types. const mapU64 = map(u8(), u64(), remainder); expect(mapU64.encode(new Map([[1, 2]]))).toStrictEqual(b('010200000000000000')); expect(mapU64.encode(new Map([[1, 2n]]))).toStrictEqual(b('010200000000000000')); expect(mapU64.read(b('010200000000000000'), 0)).toStrictEqual([new Map([[1, 2n]]), 9]); - - // It fails with variable size items. - // @ts-expect-error Remainder size needs a fixed-size item. - expect(() => map(u8(), string(), remainder)).toThrow('Codecs of "remainder" size must have fixed-size items.'); }); it('has the right sizes', () => { diff --git a/packages/codecs-data-structures/src/__tests__/set-test.ts b/packages/codecs-data-structures/src/__tests__/set-test.ts index 236a51f0408e..62e85cd5f2a1 100644 --- a/packages/codecs-data-structures/src/__tests__/set-test.ts +++ b/packages/codecs-data-structures/src/__tests__/set-test.ts @@ -79,15 +79,18 @@ describe('getSetCodec', () => { expect(set(string({ size: 1 }), remainder).encode(new Set(['a', 'b']))).toStrictEqual(b('6162')); expect(set(string({ size: 1 }), remainder).read(b('6162'), 0)).toStrictEqual([new Set(['a', 'b']), 2]); + // Variable sized items. + expect(set(string({ size: u8() }), remainder).encode(new Set(['a', 'bc']))).toStrictEqual(b('0161026263')); + expect(set(string({ size: u8() }), remainder).read(b('0161026263'), 0)).toStrictEqual([ + new Set(['a', 'bc']), + 5, + ]); + // Different From and To types. const setU64 = set(u64(), remainder); expect(setU64.encode(new Set([2]))).toStrictEqual(b('0200000000000000')); expect(setU64.encode(new Set([2n]))).toStrictEqual(b('0200000000000000')); expect(setU64.read(b('0200000000000000'), 0)).toStrictEqual([new Set([2n]), 8]); - - // It fails with variable size items. - // @ts-expect-error Remainder size needs a fixed-size item. - expect(() => set(string(), remainder)).toThrow('Codecs of "remainder" size must have fixed-size items'); }); it('has the right sizes', () => { diff --git a/packages/codecs-data-structures/src/__typetests__/array-typetest.ts b/packages/codecs-data-structures/src/__typetests__/array-typetest.ts index 08fab7176884..7355d9ef00a4 100644 --- a/packages/codecs-data-structures/src/__typetests__/array-typetest.ts +++ b/packages/codecs-data-structures/src/__typetests__/array-typetest.ts @@ -18,9 +18,7 @@ import { getArrayCodec, getArrayDecoder, getArrayEncoder } from '../array'; getArrayEncoder({} as FixedSizeEncoder, { size: 42 }) satisfies FixedSizeEncoder; getArrayEncoder({} as Encoder, { size: 0 }) satisfies FixedSizeEncoder; getArrayEncoder({} as FixedSizeEncoder, { size: 'remainder' }) satisfies VariableSizeEncoder; - - // @ts-expect-error Remainder size cannot be used with fixed-size items. - getArrayEncoder({} as VariableSizeEncoder, { size: 'remainder' }); + getArrayEncoder({} as VariableSizeEncoder, { size: 'remainder' }) satisfies VariableSizeEncoder; } { @@ -29,9 +27,7 @@ import { getArrayCodec, getArrayDecoder, getArrayEncoder } from '../array'; getArrayDecoder({} as FixedSizeDecoder, { size: 42 }) satisfies FixedSizeDecoder; getArrayDecoder({} as Decoder, { size: 0 }) satisfies FixedSizeDecoder; getArrayDecoder({} as FixedSizeDecoder, { size: 'remainder' }) satisfies VariableSizeDecoder; - - // @ts-expect-error Remainder size cannot be used with fixed-size items. - getArrayDecoder({} as VariableSizeDecoder, { size: 'remainder' }); + getArrayDecoder({} as VariableSizeDecoder, { size: 'remainder' }) satisfies VariableSizeDecoder; } { @@ -40,7 +36,5 @@ import { getArrayCodec, getArrayDecoder, getArrayEncoder } from '../array'; getArrayCodec({} as FixedSizeCodec, { size: 42 }) satisfies FixedSizeCodec; getArrayCodec({} as Codec, { size: 0 }) satisfies FixedSizeCodec; getArrayCodec({} as FixedSizeCodec, { size: 'remainder' }) satisfies VariableSizeCodec; - - // @ts-expect-error Remainder size cannot be used with fixed-size items. - getArrayCodec({} as VariableSizeCodec, { size: 'remainder' }); + getArrayCodec({} as VariableSizeCodec, { size: 'remainder' }) satisfies VariableSizeCodec; } diff --git a/packages/codecs-data-structures/src/__typetests__/map-typetest.ts b/packages/codecs-data-structures/src/__typetests__/map-typetest.ts index 4d54645546e1..10b1c7265cba 100644 --- a/packages/codecs-data-structures/src/__typetests__/map-typetest.ts +++ b/packages/codecs-data-structures/src/__typetests__/map-typetest.ts @@ -21,12 +21,7 @@ import { getMapCodec, getMapDecoder, getMapEncoder } from '../map'; getMapEncoder(...fixedKeyValue, { size: 42 }) satisfies FixedSizeEncoder>; getMapEncoder(...anyKeyValue, { size: 0 }) satisfies FixedSizeEncoder, 0>; getMapEncoder(...fixedKeyValue, { size: 'remainder' }) satisfies VariableSizeEncoder>; - - // @ts-expect-error Remainder size cannot be used with fixed-size keys. - getMapEncoder({} as VariableSizeEncoder, {} as FixedSizeEncoder, { size: 'remainder' }); - - // @ts-expect-error Remainder size cannot be used with fixed-size values. - getMapEncoder({} as FixedSizeEncoder, {} as VariableSizeEncoder, { size: 'remainder' }); + getMapEncoder(...anyKeyValue, { size: 'remainder' }) satisfies VariableSizeEncoder>; } { @@ -38,12 +33,7 @@ import { getMapCodec, getMapDecoder, getMapEncoder } from '../map'; getMapDecoder(...fixedKeyValue, { size: 42 }) satisfies FixedSizeDecoder>; getMapDecoder(...anyKeyValue, { size: 0 }) satisfies FixedSizeDecoder, 0>; getMapDecoder(...fixedKeyValue, { size: 'remainder' }) satisfies VariableSizeDecoder>; - - // @ts-expect-error Remainder size cannot be used with fixed-size keys. - getMapDecoder({} as VariableSizeDecoder, {} as FixedSizeDecoder, { size: 'remainder' }); - - // @ts-expect-error Remainder size cannot be used with fixed-size values. - getMapDecoder({} as FixedSizeDecoder, {} as VariableSizeDecoder, { size: 'remainder' }); + getMapDecoder(...anyKeyValue, { size: 'remainder' }) satisfies VariableSizeDecoder>; } { @@ -55,10 +45,5 @@ import { getMapCodec, getMapDecoder, getMapEncoder } from '../map'; getMapCodec(...fixedKeyValue, { size: 42 }) satisfies FixedSizeCodec>; getMapCodec(...anyKeyValue, { size: 0 }) satisfies FixedSizeCodec, Map, 0>; getMapCodec(...fixedKeyValue, { size: 'remainder' }) satisfies VariableSizeCodec>; - - // @ts-expect-error Remainder size cannot be used with fixed-size keys. - getMapCodec({} as VariableSizeCodec, {} as FixedSizeCodec, { size: 'remainder' }); - - // @ts-expect-error Remainder size cannot be used with fixed-size values. - getMapCodec({} as FixedSizeCodec, {} as VariableSizeCodec, { size: 'remainder' }); + getMapCodec(...anyKeyValue, { size: 'remainder' }) satisfies VariableSizeCodec>; } diff --git a/packages/codecs-data-structures/src/__typetests__/set-typetest.ts b/packages/codecs-data-structures/src/__typetests__/set-typetest.ts index 45c8c9c8e0a8..4455b1484b65 100644 --- a/packages/codecs-data-structures/src/__typetests__/set-typetest.ts +++ b/packages/codecs-data-structures/src/__typetests__/set-typetest.ts @@ -18,9 +18,7 @@ import { getSetCodec, getSetDecoder, getSetEncoder } from '../set'; getSetEncoder({} as FixedSizeEncoder, { size: 42 }) satisfies FixedSizeEncoder>; getSetEncoder({} as Encoder, { size: 0 }) satisfies FixedSizeEncoder, 0>; getSetEncoder({} as FixedSizeEncoder, { size: 'remainder' }) satisfies VariableSizeEncoder>; - - // @ts-expect-error Remainder size cannot be used with fixed-size items. - getSetEncoder({} as VariableSizeEncoder, { size: 'remainder' }); + getSetEncoder({} as VariableSizeEncoder, { size: 'remainder' }) satisfies VariableSizeEncoder>; } { @@ -29,9 +27,7 @@ import { getSetCodec, getSetDecoder, getSetEncoder } from '../set'; getSetDecoder({} as FixedSizeDecoder, { size: 42 }) satisfies FixedSizeDecoder>; getSetDecoder({} as Decoder, { size: 0 }) satisfies FixedSizeDecoder, 0>; getSetDecoder({} as FixedSizeDecoder, { size: 'remainder' }) satisfies VariableSizeDecoder>; - - // @ts-expect-error Remainder size cannot be used with fixed-size items. - getSetDecoder({} as VariableSizeDecoder, { size: 'remainder' }); + getSetDecoder({} as VariableSizeDecoder, { size: 'remainder' }) satisfies VariableSizeDecoder>; } { @@ -40,7 +36,5 @@ import { getSetCodec, getSetDecoder, getSetEncoder } from '../set'; getSetCodec({} as FixedSizeCodec, { size: 42 }) satisfies FixedSizeCodec>; getSetCodec({} as Codec, { size: 0 }) satisfies FixedSizeCodec, Set, 0>; getSetCodec({} as FixedSizeCodec, { size: 'remainder' }) satisfies VariableSizeCodec>; - - // @ts-expect-error Remainder size cannot be used with fixed-size items. - getSetCodec({} as VariableSizeCodec, { size: 'remainder' }); + getSetCodec({} as VariableSizeCodec, { size: 'remainder' }) satisfies VariableSizeCodec>; } diff --git a/packages/codecs-data-structures/src/array.ts b/packages/codecs-data-structures/src/array.ts index 44bb3dedfd7e..9b616fa2aa36 100644 --- a/packages/codecs-data-structures/src/array.ts +++ b/packages/codecs-data-structures/src/array.ts @@ -1,5 +1,4 @@ import { - assertIsFixedSize, Codec, combineCodec, createDecoder, @@ -10,7 +9,6 @@ import { FixedSizeDecoder, FixedSizeEncoder, getEncodedSize, - Offset, VariableSizeCodec, VariableSizeDecoder, VariableSizeEncoder, @@ -59,23 +57,15 @@ export function getArrayEncoder( item: FixedSizeEncoder, config: ArrayCodecConfig & { size: number }, ): FixedSizeEncoder; -export function getArrayEncoder( - item: FixedSizeEncoder, - config: ArrayCodecConfig & { size: 'remainder' }, -): VariableSizeEncoder; export function getArrayEncoder( item: Encoder, - config?: ArrayCodecConfig & { size?: number | NumberEncoder }, + config?: ArrayCodecConfig, ): VariableSizeEncoder; export function getArrayEncoder( item: Encoder, config: ArrayCodecConfig = {}, ): Encoder { const size = config.size ?? getU32Encoder(); - if (size === 'remainder') { - assertIsFixedSize(item, 'Codecs of "remainder" size must have fixed-size items.'); - } - const fixedSize = computeArrayLikeCodecSize(size, getFixedSize(item)); const maxSize = computeArrayLikeCodecSize(size, getMaxSize(item)) ?? undefined; @@ -118,20 +108,12 @@ export function getArrayDecoder( item: FixedSizeDecoder, config: ArrayCodecConfig & { size: number }, ): FixedSizeDecoder; -export function getArrayDecoder( - item: FixedSizeDecoder, - config: ArrayCodecConfig & { size: 'remainder' }, -): VariableSizeDecoder; export function getArrayDecoder( item: Decoder, - config?: ArrayCodecConfig & { size?: number | NumberDecoder }, + config?: ArrayCodecConfig, ): VariableSizeDecoder; export function getArrayDecoder(item: Decoder, config: ArrayCodecConfig = {}): Decoder { const size = config.size ?? getU32Decoder(); - if (size === 'remainder') { - assertIsFixedSize(item, 'Codecs of "remainder" size must have fixed-size items.'); - } - const itemSize = getFixedSize(item); const fixedSize = computeArrayLikeCodecSize(size, itemSize); const maxSize = computeArrayLikeCodecSize(size, getMaxSize(item)) ?? undefined; @@ -143,7 +125,17 @@ export function getArrayDecoder(item: Decoder, config: ArrayCodecConfi if (typeof size === 'object' && bytes.slice(offset).length === 0) { return [array, offset]; } - const [resolvedSize, newOffset] = readArrayLikeCodecSize(size, itemSize, bytes, offset); + + if (size === 'remainder') { + while (offset < bytes.length) { + const [value, newOffset] = item.read(bytes, offset); + offset = newOffset; + array.push(value); + } + return [array, offset]; + } + + const [resolvedSize, newOffset] = typeof size === 'number' ? [size, offset] : size.read(bytes, offset); offset = newOffset; for (let i = 0; i < resolvedSize; i += 1) { const [value, newOffset] = item.read(bytes, offset); @@ -169,13 +161,9 @@ export function getArrayCodec( item: FixedSizeCodec, config: ArrayCodecConfig & { size: number }, ): FixedSizeCodec; -export function getArrayCodec( - item: FixedSizeCodec, - config: ArrayCodecConfig & { size: 'remainder' }, -): VariableSizeCodec; export function getArrayCodec( item: Codec, - config?: ArrayCodecConfig & { size?: number | NumberCodec }, + config?: ArrayCodecConfig, ): VariableSizeCodec; export function getArrayCodec( item: Codec, @@ -184,41 +172,6 @@ export function getArrayCodec( return combineCodec(getArrayEncoder(item, config as object), getArrayDecoder(item, config as object)); } -function readArrayLikeCodecSize( - size: ArrayLikeCodecSize, - itemSize: number | null, - bytes: Uint8Array, - offset: Offset, -): [number | bigint, Offset] { - if (typeof size === 'number') { - return [size, offset]; - } - - if (typeof size === 'object') { - return size.read(bytes, offset); - } - - if (size === 'remainder') { - if (itemSize === null) { - // TODO: Coded error. - throw new Error('Codecs of "remainder" size must have fixed-size items.'); - } - const remainder = Math.max(0, bytes.length - offset); - if (remainder % itemSize !== 0) { - // TODO: Coded error. - throw new Error( - `The remainder of the byte array (${remainder} bytes) cannot be split into chunks of ${itemSize} bytes. ` + - `Codecs of "remainder" size must have a remainder that is a multiple of its item size. ` + - `In other words, ${remainder} modulo ${itemSize} should be equal to zero.`, - ); - } - return [remainder / itemSize, offset]; - } - - // TODO: Coded error. - throw new Error(`Unrecognized array-like codec size: ${JSON.stringify(size)}`); -} - function computeArrayLikeCodecSize(size: object | number | 'remainder', itemSize: number | null): number | null { if (typeof size !== 'number') return null; if (size === 0) return 0; diff --git a/packages/codecs-data-structures/src/map.ts b/packages/codecs-data-structures/src/map.ts index a81b8642d19d..78e2e75ef1a4 100644 --- a/packages/codecs-data-structures/src/map.ts +++ b/packages/codecs-data-structures/src/map.ts @@ -43,15 +43,10 @@ export function getMapEncoder( value: FixedSizeEncoder, config: MapCodecConfig & { size: number }, ): FixedSizeEncoder>; -export function getMapEncoder( - key: FixedSizeEncoder, - value: FixedSizeEncoder, - config: MapCodecConfig & { size: 'remainder' }, -): VariableSizeEncoder>; export function getMapEncoder( key: Encoder, value: Encoder, - config?: MapCodecConfig & { size?: number | NumberEncoder }, + config?: MapCodecConfig, ): VariableSizeEncoder>; export function getMapEncoder( key: Encoder, @@ -81,15 +76,10 @@ export function getMapDecoder( value: FixedSizeDecoder, config: MapCodecConfig & { size: number }, ): FixedSizeDecoder>; -export function getMapDecoder( - key: FixedSizeDecoder, - value: FixedSizeDecoder, - config: MapCodecConfig & { size: 'remainder' }, -): VariableSizeDecoder>; export function getMapDecoder( key: Decoder, value: Decoder, - config?: MapCodecConfig & { size?: number | NumberDecoder }, + config?: MapCodecConfig, ): VariableSizeDecoder>; export function getMapDecoder( key: Decoder, @@ -129,16 +119,6 @@ export function getMapCodec< value: FixedSizeCodec, config: MapCodecConfig & { size: number }, ): FixedSizeCodec, Map>; -export function getMapCodec< - TFromKey, - TFromValue, - TToKey extends TFromKey = TFromKey, - TToValue extends TFromValue = TFromValue, ->( - key: FixedSizeCodec, - value: FixedSizeCodec, - config: MapCodecConfig & { size: 'remainder' }, -): VariableSizeCodec, Map>; export function getMapCodec< TFromKey, TFromValue, @@ -147,7 +127,7 @@ export function getMapCodec< >( key: Codec, value: Codec, - config?: MapCodecConfig & { size?: number | NumberCodec }, + config?: MapCodecConfig, ): VariableSizeCodec, Map>; export function getMapCodec< TFromKey, diff --git a/packages/codecs-data-structures/src/set.ts b/packages/codecs-data-structures/src/set.ts index 6f3c8d38420f..d8fe4fa3ef53 100644 --- a/packages/codecs-data-structures/src/set.ts +++ b/packages/codecs-data-structures/src/set.ts @@ -39,13 +39,9 @@ export function getSetEncoder( item: FixedSizeEncoder, config: SetCodecConfig & { size: number }, ): FixedSizeEncoder>; -export function getSetEncoder( - item: FixedSizeEncoder, - config: SetCodecConfig & { size: 'remainder' }, -): VariableSizeEncoder>; export function getSetEncoder( item: Encoder, - config?: SetCodecConfig & { size?: number | NumberEncoder }, + config?: SetCodecConfig, ): VariableSizeEncoder>; export function getSetEncoder( item: Encoder, @@ -68,13 +64,9 @@ export function getSetDecoder( item: FixedSizeDecoder, config: SetCodecConfig & { size: number }, ): FixedSizeDecoder>; -export function getSetDecoder( - item: FixedSizeDecoder, - config: SetCodecConfig & { size: 'remainder' }, -): VariableSizeDecoder>; export function getSetDecoder( item: Decoder, - config?: SetCodecConfig & { size?: number | NumberDecoder }, + config?: SetCodecConfig, ): VariableSizeDecoder>; export function getSetDecoder(item: Decoder, config: SetCodecConfig = {}): Decoder> { return mapDecoder(getArrayDecoder(item, config as object), (entries: TTo[]): Set => new Set(entries)); @@ -94,13 +86,9 @@ export function getSetCodec( item: FixedSizeCodec, config: SetCodecConfig & { size: number }, ): FixedSizeCodec, Set>; -export function getSetCodec( - item: FixedSizeCodec, - config: SetCodecConfig & { size: 'remainder' }, -): VariableSizeCodec, Set>; export function getSetCodec( item: Codec, - config?: SetCodecConfig & { size?: number | NumberCodec }, + config?: SetCodecConfig, ): VariableSizeCodec, Set>; export function getSetCodec( item: Codec,