Add support for variable-sized items in remainder-sized serializers

.changeset/cold-cats-nail.md

@@ -0,0 +1,5 @@
+---
+'@metaplex-foundation/umi-serializers': patch
+---
+
+Add support for variable-sized items in remainder-sized serializers

packages/umi-serializers/src/array.ts

@@ -1,6 +1,5 @@
 import {
   BaseSerializerOptions,
-  ExpectedFixedSizeSerializerError,
   Serializer,
   mergeBytes,
 } from '@metaplex-foundation/umi-serializers-core';
@@ -38,11 +37,6 @@ export function array<T, U extends T = T>(
   options: ArraySerializerOptions = {}
 ): Serializer<T[], U[]> {
   const size = options.size ?? u32();
-  if (size === 'remainder' && item.fixedSize === null) {
-    throw new ExpectedFixedSizeSerializerError(
-      'Serializers of "remainder" size must have fixed-size items.'
-    );
-  }
   return {
     description:
       options.description ??
@@ -59,17 +53,20 @@ export function array<T, U extends T = T>(
       ]);
     },
     deserialize: (bytes: Uint8Array, offset = 0) => {
+      const values: U[] = [];
       if (typeof size === 'object' && bytes.slice(offset).length === 0) {
-        return [[], offset];
+        return [values, offset];
+      }
+      if (size === 'remainder') {
+        while (offset < bytes.length) {
+          const [value, newOffset] = item.deserialize(bytes, offset);
+          values.push(value);
+          offset = newOffset;
+        }
+        return [values, offset];
       }
-      const [resolvedSize, newOffset] = getResolvedSize(
-        size,
-        [item.fixedSize],
-        bytes,
-        offset
-      );
+      const [resolvedSize, newOffset] = getResolvedSize(size, bytes, offset);
       offset = newOffset;
-      const values: U[] = [];
       for (let i = 0; i < resolvedSize; i += 1) {
         const [value, newOffset] = item.deserialize(bytes, offset);
         values.push(value);

packages/umi-serializers/src/map.ts

@@ -1,18 +1,17 @@
 import {
   BaseSerializerOptions,
-  ExpectedFixedSizeSerializerError,
   mergeBytes,
   Serializer,
 } from '@metaplex-foundation/umi-serializers-core';
 import { u32 } from '@metaplex-foundation/umi-serializers-numbers';
 import { ArrayLikeSerializerSize } from './arrayLikeSerializerSize';
+import { InvalidNumberOfItemsError } from './errors';
 import {
   getResolvedSize,
   getSizeDescription,
   getSizeFromChildren,
   getSizePrefix,
 } from './utils';
-import { InvalidNumberOfItemsError } from './errors';
 
 /**
  * Defines the options for `Map` serializers.
@@ -40,14 +39,6 @@ export function map<TK, TV, UK extends TK = TK, UV extends TV = TV>(
   options: MapSerializerOptions = {}
 ): Serializer<Map<TK, TV>, Map<UK, UV>> {
   const size = options.size ?? u32();
-  if (
-    size === 'remainder' &&
-    (key.fixedSize === null || value.fixedSize === null)
-  ) {
-    throw new ExpectedFixedSizeSerializerError(
-      'Serializers of "remainder" size must have fixed-size items.'
-    );
-  }
   return {
     description:
       options.description ??
@@ -70,12 +61,17 @@ export function map<TK, TV, UK extends TK = TK, UV extends TV = TV>(
       if (typeof size === 'object' && bytes.slice(offset).length === 0) {
         return [map, offset];
       }
-      const [resolvedSize, newOffset] = getResolvedSize(
-        size,
-        [key.fixedSize, value.fixedSize],
-        bytes,
-        offset
-      );
+      if (size === 'remainder') {
+        while (offset < bytes.length) {
+          const [deserializedKey, kOffset] = key.deserialize(bytes, offset);
+          offset = kOffset;
+          const [deserializedValue, vOffset] = value.deserialize(bytes, offset);
+          offset = vOffset;
+          map.set(deserializedKey, deserializedValue);
+        }
+        return [map, offset];
+      }
+      const [resolvedSize, newOffset] = getResolvedSize(size, bytes, offset);
       offset = newOffset;
       for (let i = 0; i < resolvedSize; i += 1) {
         const [deserializedKey, kOffset] = key.deserialize(bytes, offset);

packages/umi-serializers/src/set.ts

@@ -1,18 +1,17 @@
 import {
   BaseSerializerOptions,
-  ExpectedFixedSizeSerializerError,
   mergeBytes,
   Serializer,
 } from '@metaplex-foundation/umi-serializers-core';
 import { u32 } from '@metaplex-foundation/umi-serializers-numbers';
 import { ArrayLikeSerializerSize } from './arrayLikeSerializerSize';
+import { InvalidNumberOfItemsError } from './errors';
 import {
   getResolvedSize,
   getSizeDescription,
   getSizeFromChildren,
   getSizePrefix,
 } from './utils';
-import { InvalidNumberOfItemsError } from './errors';
 
 /**
  * Defines the options for `Set` serializers.
@@ -38,11 +37,6 @@ export function set<T, U extends T = T>(
   options: SetSerializerOptions = {}
 ): Serializer<Set<T>, Set<U>> {
   const size = options.size ?? u32();
-  if (size === 'remainder' && item.fixedSize === null) {
-    throw new ExpectedFixedSizeSerializerError(
-      'Serializers of "remainder" size must have fixed-size items.'
-    );
-  }
   return {
     description:
       options.description ??
@@ -61,17 +55,20 @@ export function set<T, U extends T = T>(
       if (typeof size === 'object' && bytes.slice(offset).length === 0) {
         return [set, offset];
       }
-      const [resolvedSize, newOffset] = getResolvedSize(
-        size,
-        [item.fixedSize],
-        bytes,
-        offset
-      );
+      if (size === 'remainder') {
+        while (offset < bytes.length) {
+          const [value, newOffset] = item.deserialize(bytes, offset);
+          set.add(value);
+          offset = newOffset;
+        }
+        return [set, offset];
+      }
+      const [resolvedSize, newOffset] = getResolvedSize(size, bytes, offset);
       offset = newOffset;
       for (let i = 0; i < resolvedSize; i += 1) {
         const [value, newOffset] = item.deserialize(bytes, offset);
-        offset = newOffset;
         set.add(value);
+        offset = newOffset;
       }
       return [set, offset];
     },

packages/umi-serializers/src/utils.ts

@@ -1,14 +1,10 @@
-import { ExpectedFixedSizeSerializerError } from '@metaplex-foundation/umi-serializers-core';
+import type { NumberSerializer } from '@metaplex-foundation/umi-serializers-numbers';
 import { ArrayLikeSerializerSize } from './arrayLikeSerializerSize';
-import {
-  InvalidArrayLikeRemainderSizeError,
-  UnrecognizedArrayLikeSerializerSizeError,
-} from './errors';
+import { UnrecognizedArrayLikeSerializerSizeError } from './errors';
 import { sumSerializerSizes } from './sumSerializerSizes';
 
 export function getResolvedSize(
-  size: ArrayLikeSerializerSize,
-  childrenSizes: (number | null)[],
+  size: number | NumberSerializer,
   bytes: Uint8Array,
   offset: number
 ): [number | bigint, number] {
@@ -20,20 +16,6 @@ export function getResolvedSize(
     return size.deserialize(bytes, offset);
   }
 
-  if (size === 'remainder') {
-    const childrenSize = sumSerializerSizes(childrenSizes);
-    if (childrenSize === null) {
-      throw new ExpectedFixedSizeSerializerError(
-        'Serializers of "remainder" size must have fixed-size items.'
-      );
-    }
-    const remainder = bytes.slice(offset).length;
-    if (remainder % childrenSize !== 0) {
-      throw new InvalidArrayLikeRemainderSizeError(remainder, childrenSize);
-    }
-    return [remainder / childrenSize, offset];
-  }
-
   throw new UnrecognizedArrayLikeSerializerSizeError(size);
 }
 

packages/umi-serializers/test/array.test.ts

@@ -70,16 +70,14 @@ test('remainder (de)serialization', (t) => {
   s(t, array(string({ size: 1 }), remainder), ['a', 'b'], '6162');
   d(t, array(string({ size: 1 }), remainder), '6162', ['a', 'b'], 2);
 
+  // Variable sized items.
+  s(t, array(string({ size: u8() }), remainder), ['a', 'bc'], '0161026263');
+  d(t, array(string({ size: u8() }), remainder), '0161026263', ['a', 'bc'], 5);
+
   // Different From and To types.
   const arrayU64 = array<number | bigint, bigint>(u64(), remainder);
   s(t, arrayU64, [2], '0200000000000000');
   d(t, arrayU64, '0200000000000000', [2n], 8);
-
-  // It fails with variable size items.
-  t.throws(() => array(string(), remainder), {
-    message: (m) =>
-      m.includes('Serializers of "remainder" size must have fixed-size items'),
-  });
 });
 
 test('description', (t) => {

packages/umi-serializers/test/map.test.ts

@@ -88,6 +88,15 @@ test('remainder (de)serialization', (t) => {
   s(t, letters, lettersMap, '61016202');
   d(t, letters, '61016202', lettersMap, 4);
 
+  // Variable sized items.
+  const prefixedLetters = map(string({ size: u8() }), u8(), remainder);
+  const prefixedLettersMap = new Map([
+    ['a', 6],
+    ['bc', 7],
+  ]);
+  s(t, prefixedLetters, prefixedLettersMap, '01610602626307');
+  d(t, prefixedLetters, '01610602626307', prefixedLettersMap, 7);
+
   // Different From and To types.
   const mapU64 = map<number, number | bigint, number, bigint>(
     u8(),
@@ -96,12 +105,6 @@ test('remainder (de)serialization', (t) => {
   );
   s(t, mapU64, new Map([[1, 2]]), '010200000000000000');
   d(t, mapU64, '010200000000000000', new Map([[1, 2n]]), 9);
-
-  // It fails with variable size items.
-  t.throws(() => map(u8(), string(), remainder), {
-    message: (m) =>
-      m.includes('Serializers of "remainder" size must have fixed-size items'),
-  });
 });
 
 test('description', (t) => {

packages/umi-serializers/test/set.test.ts

@@ -81,16 +81,15 @@ test('remainder (de)serialization', (t) => {
   s(t, set(string({ size: 1 }), remainder), new Set(['a', 'b']), '6162');
   d(t, set(string({ size: 1 }), remainder), '6162', new Set(['a', 'b']), 2);
 
+  // Variable sized items.
+  const prefixedSet = set(string({ size: u8() }), remainder);
+  s(t, prefixedSet, new Set(['a', 'bc']), '0161026263');
+  d(t, prefixedSet, '0161026263', new Set(['a', 'bc']), 5);
+
   // Different From and To types.
   const setU64 = set<number | bigint, bigint>(u64(), remainder);
   s(t, setU64, new Set([2]), '0200000000000000');
   d(t, setU64, '0200000000000000', new Set([2n]), 8);
-
-  // It fails with variable size items.
-  t.throws(() => set(string(), remainder), {
-    message: (m) =>
-      m.includes('Serializers of "remainder" size must have fixed-size items'),
-  });
 });
 
 test('description', (t) => {