solana-labs · lorisleiva · Jan 10, 2024 · Jan 9, 2024
diff --git a/packages/codecs-data-structures/src/__tests__/array-test.ts b/packages/codecs-data-structures/src/__tests__/array-test.ts
@@ -79,15 +79,15 @@ describe('getArrayCodec', () => {
         expect(array(string({ size: 1 }), remainder).encode(['a', 'b'])).toStrictEqual(b('6162'));
         expect(array(string({ size: 1 }), remainder).read(b('6162'), 0)).toStrictEqual([['a', 'b'], 2]);
 
+        // Variable sized items.
+        expect(array(string({ size: u8() }), remainder).encode(['a', 'bc'])).toStrictEqual(b('0161026263'));
+        expect(array(string({ size: u8() }), remainder).read(b('0161026263'), 0)).toStrictEqual([['a', 'bc'], 5]);
+
         // Different From and To types.
         const arrayU64 = array<number | bigint, bigint>(u64(), remainder);
         expect(arrayU64.encode([2])).toStrictEqual(b('0200000000000000'));
         expect(arrayU64.encode([2n])).toStrictEqual(b('0200000000000000'));
         expect(arrayU64.read(b('0200000000000000'), 0)).toStrictEqual([[2n], 8]);
-
-        // It fails with variable size items.
-        // @ts-expect-error Remainder size cannot be used with fixed-size items.
-        expect(() => array(string(), remainder)).toThrow('Codecs of "remainder" size must have fixed-size items');
     });
 
     it('has the right sizes', () => {

diff --git a/packages/codecs-data-structures/src/__tests__/map-test.ts b/packages/codecs-data-structures/src/__tests__/map-test.ts
@@ -95,15 +95,20 @@ describe('getMapCodec', () => {
         expect(letters.encode(lettersMap)).toStrictEqual(b('61016202'));
         expect(letters.read(b('61016202'), 0)).toStrictEqual([lettersMap, 4]);
 
+        // Variable sized items.
+        const prefixedLetters = map(string({ size: u8() }), u8(), remainder);
+        const prefixedLettersMap = new Map([
+            ['a', 6],
+            ['bc', 7],
+        ]);
+        expect(prefixedLetters.encode(prefixedLettersMap)).toStrictEqual(b('01610602626307'));
+        expect(prefixedLetters.read(b('01610602626307'), 0)).toStrictEqual([prefixedLettersMap, 7]);
+
         // Different From and To types.
         const mapU64 = map<number, number | bigint, number, bigint>(u8(), u64(), remainder);
         expect(mapU64.encode(new Map([[1, 2]]))).toStrictEqual(b('010200000000000000'));
         expect(mapU64.encode(new Map([[1, 2n]]))).toStrictEqual(b('010200000000000000'));
         expect(mapU64.read(b('010200000000000000'), 0)).toStrictEqual([new Map([[1, 2n]]), 9]);
-
-        // It fails with variable size items.
-        // @ts-expect-error Remainder size needs a fixed-size item.
-        expect(() => map(u8(), string(), remainder)).toThrow('Codecs of "remainder" size must have fixed-size items.');
     });
 
     it('has the right sizes', () => {

diff --git a/packages/codecs-data-structures/src/__tests__/set-test.ts b/packages/codecs-data-structures/src/__tests__/set-test.ts
@@ -79,15 +79,18 @@ describe('getSetCodec', () => {
         expect(set(string({ size: 1 }), remainder).encode(new Set(['a', 'b']))).toStrictEqual(b('6162'));
         expect(set(string({ size: 1 }), remainder).read(b('6162'), 0)).toStrictEqual([new Set(['a', 'b']), 2]);
 
+        // Variable sized items.
+        expect(set(string({ size: u8() }), remainder).encode(new Set(['a', 'bc']))).toStrictEqual(b('0161026263'));
+        expect(set(string({ size: u8() }), remainder).read(b('0161026263'), 0)).toStrictEqual([
+            new Set(['a', 'bc']),
+            5,
+        ]);
+
         // Different From and To types.
         const setU64 = set<number | bigint, bigint>(u64(), remainder);
         expect(setU64.encode(new Set([2]))).toStrictEqual(b('0200000000000000'));
         expect(setU64.encode(new Set([2n]))).toStrictEqual(b('0200000000000000'));
         expect(setU64.read(b('0200000000000000'), 0)).toStrictEqual([new Set([2n]), 8]);
-
-        // It fails with variable size items.
-        // @ts-expect-error Remainder size needs a fixed-size item.
-        expect(() => set(string(), remainder)).toThrow('Codecs of "remainder" size must have fixed-size items');
     });
 
     it('has the right sizes', () => {

diff --git a/packages/codecs-data-structures/src/__typetests__/array-typetest.ts b/packages/codecs-data-structures/src/__typetests__/array-typetest.ts
@@ -18,9 +18,7 @@ import { getArrayCodec, getArrayDecoder, getArrayEncoder } from '../array';
     getArrayEncoder({} as FixedSizeEncoder<string>, { size: 42 }) satisfies FixedSizeEncoder<string[]>;
     getArrayEncoder({} as Encoder<string>, { size: 0 }) satisfies FixedSizeEncoder<string[], 0>;
     getArrayEncoder({} as FixedSizeEncoder<string>, { size: 'remainder' }) satisfies VariableSizeEncoder<string[]>;
-
-    // @ts-expect-error Remainder size cannot be used with fixed-size items.
-    getArrayEncoder({} as VariableSizeEncoder<string>, { size: 'remainder' });
+    getArrayEncoder({} as VariableSizeEncoder<string>, { size: 'remainder' }) satisfies VariableSizeEncoder<string[]>;
 }
 
 {
@@ -29,9 +27,7 @@ import { getArrayCodec, getArrayDecoder, getArrayEncoder } from '../array';
     getArrayDecoder({} as FixedSizeDecoder<string>, { size: 42 }) satisfies FixedSizeDecoder<string[]>;
     getArrayDecoder({} as Decoder<string>, { size: 0 }) satisfies FixedSizeDecoder<string[], 0>;
     getArrayDecoder({} as FixedSizeDecoder<string>, { size: 'remainder' }) satisfies VariableSizeDecoder<string[]>;
-
-    // @ts-expect-error Remainder size cannot be used with fixed-size items.
-    getArrayDecoder({} as VariableSizeDecoder<string>, { size: 'remainder' });
+    getArrayDecoder({} as VariableSizeDecoder<string>, { size: 'remainder' }) satisfies VariableSizeDecoder<string[]>;
 }
 
 {
@@ -40,7 +36,5 @@ import { getArrayCodec, getArrayDecoder, getArrayEncoder } from '../array';
     getArrayCodec({} as FixedSizeCodec<string>, { size: 42 }) satisfies FixedSizeCodec<string[]>;
     getArrayCodec({} as Codec<string>, { size: 0 }) satisfies FixedSizeCodec<string[], string[], 0>;
     getArrayCodec({} as FixedSizeCodec<string>, { size: 'remainder' }) satisfies VariableSizeCodec<string[]>;
-
-    // @ts-expect-error Remainder size cannot be used with fixed-size items.
-    getArrayCodec({} as VariableSizeCodec<string>, { size: 'remainder' });
+    getArrayCodec({} as VariableSizeCodec<string>, { size: 'remainder' }) satisfies VariableSizeCodec<string[]>;
 }
diff --git a/packages/codecs-data-structures/src/__typetests__/map-typetest.ts b/packages/codecs-data-structures/src/__typetests__/map-typetest.ts
@@ -21,12 +21,7 @@ import { getMapCodec, getMapDecoder, getMapEncoder } from '../map';
     getMapEncoder(...fixedKeyValue, { size: 42 }) satisfies FixedSizeEncoder<Map<string, number>>;
     getMapEncoder(...anyKeyValue, { size: 0 }) satisfies FixedSizeEncoder<Map<string, number>, 0>;
     getMapEncoder(...fixedKeyValue, { size: 'remainder' }) satisfies VariableSizeEncoder<Map<string, number>>;
-
-    // @ts-expect-error Remainder size cannot be used with fixed-size keys.
-    getMapEncoder({} as VariableSizeEncoder<string>, {} as FixedSizeEncoder<number>, { size: 'remainder' });
-
-    // @ts-expect-error Remainder size cannot be used with fixed-size values.
-    getMapEncoder({} as FixedSizeEncoder<string>, {} as VariableSizeEncoder<number>, { size: 'remainder' });
+    getMapEncoder(...anyKeyValue, { size: 'remainder' }) satisfies VariableSizeEncoder<Map<string, number>>;
 }
 
 {
@@ -38,12 +33,7 @@ import { getMapCodec, getMapDecoder, getMapEncoder } from '../map';
     getMapDecoder(...fixedKeyValue, { size: 42 }) satisfies FixedSizeDecoder<Map<string, number>>;
     getMapDecoder(...anyKeyValue, { size: 0 }) satisfies FixedSizeDecoder<Map<string, number>, 0>;
     getMapDecoder(...fixedKeyValue, { size: 'remainder' }) satisfies VariableSizeDecoder<Map<string, number>>;
-
-    // @ts-expect-error Remainder size cannot be used with fixed-size keys.
-    getMapDecoder({} as VariableSizeDecoder<string>, {} as FixedSizeDecoder<number>, { size: 'remainder' });
-
-    // @ts-expect-error Remainder size cannot be used with fixed-size values.
-    getMapDecoder({} as FixedSizeDecoder<string>, {} as VariableSizeDecoder<number>, { size: 'remainder' });
+    getMapDecoder(...anyKeyValue, { size: 'remainder' }) satisfies VariableSizeDecoder<Map<string, number>>;
 }
 
 {
@@ -55,10 +45,5 @@ import { getMapCodec, getMapDecoder, getMapEncoder } from '../map';
     getMapCodec(...fixedKeyValue, { size: 42 }) satisfies FixedSizeCodec<Map<string, number>>;
     getMapCodec(...anyKeyValue, { size: 0 }) satisfies FixedSizeCodec<Map<string, number>, Map<string, number>, 0>;
     getMapCodec(...fixedKeyValue, { size: 'remainder' }) satisfies VariableSizeCodec<Map<string, number>>;
-
-    // @ts-expect-error Remainder size cannot be used with fixed-size keys.
-    getMapCodec({} as VariableSizeCodec<string>, {} as FixedSizeCodec<number>, { size: 'remainder' });
-
-    // @ts-expect-error Remainder size cannot be used with fixed-size values.
-    getMapCodec({} as FixedSizeCodec<string>, {} as VariableSizeCodec<number>, { size: 'remainder' });
+    getMapCodec(...anyKeyValue, { size: 'remainder' }) satisfies VariableSizeCodec<Map<string, number>>;
 }
diff --git a/packages/codecs-data-structures/src/__typetests__/set-typetest.ts b/packages/codecs-data-structures/src/__typetests__/set-typetest.ts
@@ -18,9 +18,7 @@ import { getSetCodec, getSetDecoder, getSetEncoder } from '../set';
     getSetEncoder({} as FixedSizeEncoder<string>, { size: 42 }) satisfies FixedSizeEncoder<Set<string>>;
     getSetEncoder({} as Encoder<string>, { size: 0 }) satisfies FixedSizeEncoder<Set<string>, 0>;
     getSetEncoder({} as FixedSizeEncoder<string>, { size: 'remainder' }) satisfies VariableSizeEncoder<Set<string>>;
-
-    // @ts-expect-error Remainder size cannot be used with fixed-size items.
-    getSetEncoder({} as VariableSizeEncoder<string>, { size: 'remainder' });
+    getSetEncoder({} as VariableSizeEncoder<string>, { size: 'remainder' }) satisfies VariableSizeEncoder<Set<string>>;
 }
 
 {
@@ -29,9 +27,7 @@ import { getSetCodec, getSetDecoder, getSetEncoder } from '../set';
     getSetDecoder({} as FixedSizeDecoder<string>, { size: 42 }) satisfies FixedSizeDecoder<Set<string>>;
     getSetDecoder({} as Decoder<string>, { size: 0 }) satisfies FixedSizeDecoder<Set<string>, 0>;
     getSetDecoder({} as FixedSizeDecoder<string>, { size: 'remainder' }) satisfies VariableSizeDecoder<Set<string>>;
-
-    // @ts-expect-error Remainder size cannot be used with fixed-size items.
-    getSetDecoder({} as VariableSizeDecoder<string>, { size: 'remainder' });
+    getSetDecoder({} as VariableSizeDecoder<string>, { size: 'remainder' }) satisfies VariableSizeDecoder<Set<string>>;
 }
 
 {
@@ -40,7 +36,5 @@ import { getSetCodec, getSetDecoder, getSetEncoder } from '../set';
     getSetCodec({} as FixedSizeCodec<string>, { size: 42 }) satisfies FixedSizeCodec<Set<string>>;
     getSetCodec({} as Codec<string>, { size: 0 }) satisfies FixedSizeCodec<Set<string>, Set<string>, 0>;
     getSetCodec({} as FixedSizeCodec<string>, { size: 'remainder' }) satisfies VariableSizeCodec<Set<string>>;
-
-    // @ts-expect-error Remainder size cannot be used with fixed-size items.
-    getSetCodec({} as VariableSizeCodec<string>, { size: 'remainder' });
+    getSetCodec({} as VariableSizeCodec<string>, { size: 'remainder' }) satisfies VariableSizeCodec<Set<string>>;
 }
diff --git a/packages/codecs-data-structures/src/array.ts b/packages/codecs-data-structures/src/array.ts
@@ -1,5 +1,4 @@
 import {
-    assertIsFixedSize,
     Codec,
     combineCodec,
     createDecoder,
@@ -10,7 +9,6 @@ import {
     FixedSizeDecoder,
     FixedSizeEncoder,
     getEncodedSize,
-    Offset,
     VariableSizeCodec,
     VariableSizeDecoder,
     VariableSizeEncoder,
@@ -59,23 +57,15 @@ export function getArrayEncoder<TFrom>(
     item: FixedSizeEncoder<TFrom>,
     config: ArrayCodecConfig<NumberEncoder> & { size: number },
 ): FixedSizeEncoder<TFrom[]>;
-export function getArrayEncoder<TFrom>(
-    item: FixedSizeEncoder<TFrom>,
-    config: ArrayCodecConfig<NumberEncoder> & { size: 'remainder' },
-): VariableSizeEncoder<TFrom[]>;
 export function getArrayEncoder<TFrom>(
     item: Encoder<TFrom>,
-    config?: ArrayCodecConfig<NumberEncoder> & { size?: number | NumberEncoder },
+    config?: ArrayCodecConfig<NumberEncoder>,
 ): VariableSizeEncoder<TFrom[]>;
 export function getArrayEncoder<TFrom>(
     item: Encoder<TFrom>,
     config: ArrayCodecConfig<NumberEncoder> = {},
 ): Encoder<TFrom[]> {
     const size = config.size ?? getU32Encoder();
-    if (size === 'remainder') {
-        assertIsFixedSize(item, 'Codecs of "remainder" size must have fixed-size items.');
-    }
-
     const fixedSize = computeArrayLikeCodecSize(size, getFixedSize(item));
     const maxSize = computeArrayLikeCodecSize(size, getMaxSize(item)) ?? undefined;
 
@@ -118,20 +108,12 @@ export function getArrayDecoder<TTo>(
     item: FixedSizeDecoder<TTo>,
     config: ArrayCodecConfig<NumberDecoder> & { size: number },
 ): FixedSizeDecoder<TTo[]>;
-export function getArrayDecoder<TTo>(
-    item: FixedSizeDecoder<TTo>,
-    config: ArrayCodecConfig<NumberDecoder> & { size: 'remainder' },
-): VariableSizeDecoder<TTo[]>;
 export function getArrayDecoder<TTo>(
     item: Decoder<TTo>,
-    config?: ArrayCodecConfig<NumberDecoder> & { size?: number | NumberDecoder },
+    config?: ArrayCodecConfig<NumberDecoder>,
 ): VariableSizeDecoder<TTo[]>;
 export function getArrayDecoder<TTo>(item: Decoder<TTo>, config: ArrayCodecConfig<NumberDecoder> = {}): Decoder<TTo[]> {
     const size = config.size ?? getU32Decoder();
-    if (size === 'remainder') {
-        assertIsFixedSize(item, 'Codecs of "remainder" size must have fixed-size items.');
-    }
-
     const itemSize = getFixedSize(item);
     const fixedSize = computeArrayLikeCodecSize(size, itemSize);
     const maxSize = computeArrayLikeCodecSize(size, getMaxSize(item)) ?? undefined;
@@ -143,7 +125,17 @@ export function getArrayDecoder<TTo>(item: Decoder<TTo>, config: ArrayCodecConfi
             if (typeof size === 'object' && bytes.slice(offset).length === 0) {
                 return [array, offset];
             }
-            const [resolvedSize, newOffset] = readArrayLikeCodecSize(size, itemSize, bytes, offset);
+
+            if (size === 'remainder') {
+                while (offset < bytes.length) {
+                    const [value, newOffset] = item.read(bytes, offset);
+                    offset = newOffset;
+                    array.push(value);
+                }
+                return [array, offset];
+            }
+
+            const [resolvedSize, newOffset] = typeof size === 'number' ? [size, offset] : size.read(bytes, offset);
             offset = newOffset;
             for (let i = 0; i < resolvedSize; i += 1) {
                 const [value, newOffset] = item.read(bytes, offset);
@@ -169,13 +161,9 @@ export function getArrayCodec<TFrom, TTo extends TFrom = TFrom>(
     item: FixedSizeCodec<TFrom, TTo>,
     config: ArrayCodecConfig<NumberCodec> & { size: number },
 ): FixedSizeCodec<TFrom[], TTo[]>;
-export function getArrayCodec<TFrom, TTo extends TFrom = TFrom>(
-    item: FixedSizeCodec<TFrom, TTo>,
-    config: ArrayCodecConfig<NumberCodec> & { size: 'remainder' },
-): VariableSizeCodec<TFrom[], TTo[]>;
 export function getArrayCodec<TFrom, TTo extends TFrom = TFrom>(
     item: Codec<TFrom, TTo>,
-    config?: ArrayCodecConfig<NumberCodec> & { size?: number | NumberCodec },
+    config?: ArrayCodecConfig<NumberCodec>,
 ): VariableSizeCodec<TFrom[], TTo[]>;
 export function getArrayCodec<TFrom, TTo extends TFrom = TFrom>(
     item: Codec<TFrom, TTo>,
@@ -184,41 +172,6 @@ export function getArrayCodec<TFrom, TTo extends TFrom = TFrom>(
     return combineCodec(getArrayEncoder(item, config as object), getArrayDecoder(item, config as object));
 }
 
-function readArrayLikeCodecSize(
-    size: ArrayLikeCodecSize<NumberDecoder>,
-    itemSize: number | null,
-    bytes: Uint8Array,
-    offset: Offset,
-): [number | bigint, Offset] {
-    if (typeof size === 'number') {
-        return [size, offset];
-    }
-
-    if (typeof size === 'object') {
-        return size.read(bytes, offset);
-    }
-
-    if (size === 'remainder') {
-        if (itemSize === null) {
-            // TODO: Coded error.
-            throw new Error('Codecs of "remainder" size must have fixed-size items.');
-        }
-        const remainder = Math.max(0, bytes.length - offset);
-        if (remainder % itemSize !== 0) {
-            // TODO: Coded error.
-            throw new Error(
-                `The remainder of the byte array (${remainder} bytes) cannot be split into chunks of ${itemSize} bytes. ` +
-                    `Codecs of "remainder" size must have a remainder that is a multiple of its item size. ` +
-                    `In other words, ${remainder} modulo ${itemSize} should be equal to zero.`,
-            );
-        }
-        return [remainder / itemSize, offset];
-    }
-
-    // TODO: Coded error.
-    throw new Error(`Unrecognized array-like codec size: ${JSON.stringify(size)}`);
-}
-
 function computeArrayLikeCodecSize(size: object | number | 'remainder', itemSize: number | null): number | null {
     if (typeof size !== 'number') return null;
     if (size === 0) return 0;

diff --git a/packages/codecs-data-structures/src/map.ts b/packages/codecs-data-structures/src/map.ts
@@ -43,15 +43,10 @@ export function getMapEncoder<TFromKey, TFromValue>(
     value: FixedSizeEncoder<TFromValue>,
     config: MapCodecConfig<NumberEncoder> & { size: number },
 ): FixedSizeEncoder<Map<TFromKey, TFromValue>>;
-export function getMapEncoder<TFromKey, TFromValue>(
-    key: FixedSizeEncoder<TFromKey>,
-    value: FixedSizeEncoder<TFromValue>,
-    config: MapCodecConfig<NumberEncoder> & { size: 'remainder' },
-): VariableSizeEncoder<Map<TFromKey, TFromValue>>;
 export function getMapEncoder<TFromKey, TFromValue>(
     key: Encoder<TFromKey>,
     value: Encoder<TFromValue>,
-    config?: MapCodecConfig<NumberEncoder> & { size?: number | NumberEncoder },
+    config?: MapCodecConfig<NumberEncoder>,
 ): VariableSizeEncoder<Map<TFromKey, TFromValue>>;
 export function getMapEncoder<TFromKey, TFromValue>(
     key: Encoder<TFromKey>,
@@ -81,15 +76,10 @@ export function getMapDecoder<TToKey, TToValue>(
     value: FixedSizeDecoder<TToValue>,
     config: MapCodecConfig<NumberDecoder> & { size: number },
 ): FixedSizeDecoder<Map<TToKey, TToValue>>;
-export function getMapDecoder<TToKey, TToValue>(
-    key: FixedSizeDecoder<TToKey>,
-    value: FixedSizeDecoder<TToValue>,
-    config: MapCodecConfig<NumberDecoder> & { size: 'remainder' },
-): VariableSizeDecoder<Map<TToKey, TToValue>>;
 export function getMapDecoder<TToKey, TToValue>(
     key: Decoder<TToKey>,
     value: Decoder<TToValue>,
-    config?: MapCodecConfig<NumberDecoder> & { size?: number | NumberDecoder },
+    config?: MapCodecConfig<NumberDecoder>,
 ): VariableSizeDecoder<Map<TToKey, TToValue>>;
 export function getMapDecoder<TToKey, TToValue>(
     key: Decoder<TToKey>,
@@ -129,16 +119,6 @@ export function getMapCodec<
     value: FixedSizeCodec<TFromValue, TToValue>,
     config: MapCodecConfig<NumberCodec> & { size: number },
 ): FixedSizeCodec<Map<TFromKey, TFromValue>, Map<TToKey, TToValue>>;
-export function getMapCodec<
-    TFromKey,
-    TFromValue,
-    TToKey extends TFromKey = TFromKey,
-    TToValue extends TFromValue = TFromValue,
->(
-    key: FixedSizeCodec<TFromKey, TToKey>,
-    value: FixedSizeCodec<TFromValue, TToValue>,
-    config: MapCodecConfig<NumberCodec> & { size: 'remainder' },
-): VariableSizeCodec<Map<TFromKey, TFromValue>, Map<TToKey, TToValue>>;
 export function getMapCodec<
     TFromKey,
     TFromValue,
@@ -147,7 +127,7 @@ export function getMapCodec<
 >(
     key: Codec<TFromKey, TToKey>,
     value: Codec<TFromValue, TToValue>,
-    config?: MapCodecConfig<NumberCodec> & { size?: number | NumberCodec },
+    config?: MapCodecConfig<NumberCodec>,
 ): VariableSizeCodec<Map<TFromKey, TFromValue>, Map<TToKey, TToValue>>;
 export function getMapCodec<
     TFromKey,