zcbor.py: Add support for unordered maps in generated code

Add support for the --unordered-maps option to zcbor code
which makes use of the zcbor_unordered_maps_*() API in generated code.

Signed-off-by: Øyvind Rønningstad <oyvind.ronningstad@nordicsemi.no>

MIGRATION_GUIDE.md

@@ -8,7 +8,10 @@
   `zcbor_bool_*()`, `zcbor_nil_*()`, and `zcbor_undefined_*()`.
   If a removed variant is strictly needed, add your own forward declaration in your code.
 
-* Code generation naming:
+* Code generation:
+
+  * `zcbor_entry_function()` gained a new argument.
+    Regenerate or manually edit your generated files to add a 0 parameter to all calls to `zcbor_entry_function()`.
 
   * More C keywords are now capitalized to avoid naming collision.
     You might have to capitalize some instances if your code was generated to have those names.

README.md

@@ -424,7 +424,7 @@ usage: zcbor code [-h] -c CDDL [--no-prelude] [-v]
                   ENTRY_TYPES [ENTRY_TYPES ...] [-d] [-e] [--time-header]
                   [--git-sha-header] [-b {32,64}]
                   [--include-prefix INCLUDE_PREFIX] [-s]
-                  [--file-header FILE_HEADER]
+                  [--file-header FILE_HEADER] [--unordered-maps]
 
 Parse a CDDL file and produce C code that validates and xcodes CBOR.
 The output from this script is a C file and a header file. The header file
@@ -525,6 +525,13 @@ options:
                         generated files, e.g. copyright. Can be a string or a
                         path to a file. If interpreted as a path to an
                         existing file, the file's contents will be used.
+  --unordered-maps      Add support in the generated code for parsing maps
+                        with unknown element order. When enabled, the
+                        generated code will use the zcbor_unordered_map_*()
+                        API to decode data whenever inside a map. zcbor
+                        detects when ZCBOR_MAP_SMART_SEARCH is needed and
+                        enable This places restrictions on the level of
+                        ambiguity allowed between map keys in a map.
 
 ```
 

include/zcbor_common.h

@@ -342,7 +342,7 @@ void zcbor_new_state(zcbor_state_t *state_array, size_t n_states,
  */
 int zcbor_entry_function(const uint8_t *payload, size_t payload_len,
 	void *result, size_t *payload_len_out, zcbor_state_t *state, zcbor_decoder_t func,
-	size_t n_states, size_t elem_count);
+	size_t n_states, size_t elem_count, size_t n_flags);
 
 #ifdef ZCBOR_STOP_ON_ERROR
 /** Check stored error and fail if present, but only if stop_on_error is true.
@@ -530,7 +530,7 @@ static inline size_t zcbor_flags_to_states(size_t num_flags)
 #define ZCBOR_FLAG_STATES(n_flags) zcbor_flags_to_states(n_flags)
 
 #else
-#define ZCBOR_FLAG_STATES(n_flags) 0
+#define ZCBOR_FLAG_STATES(n_flags) (n_flags * 0)
 #endif
 
 size_t strnlen(const char *, size_t);

samples/pet/pet.cmake

@@ -21,3 +21,4 @@ target_include_directories(pet PUBLIC
     ${CMAKE_CURRENT_LIST_DIR}/../../include
     ${CMAKE_CURRENT_LIST_DIR}/include
     )
+target_compile_definitions(pet PUBLIC ZCBOR_MAP_SMART_SEARCH)

samples/pet/src/pet_decode.c

@@ -20,6 +20,7 @@
 #error "The type file was generated with a different default_max_qty than this file"
 #endif
 
+
 #define log_result(state, result, func) do { \
 	if (!result) { \
 		zcbor_trace_file(state); \
@@ -62,8 +63,8 @@ int cbor_decode_Pet(
 		struct Pet *result,
 		size_t *payload_len_out)
 {
-	zcbor_state_t states[4];
+	zcbor_state_t states[4 + 0];
 
 	return zcbor_entry_function(payload, payload_len, (void *)result, payload_len_out, states,
-		(zcbor_decoder_t *)decode_Pet, sizeof(states) / sizeof(zcbor_state_t), 1);
+		(zcbor_decoder_t *)decode_Pet, sizeof(states) / sizeof(zcbor_state_t), 1, 0);
 }

samples/pet/src/pet_encode.c

@@ -20,6 +20,7 @@
 #error "The type file was generated with a different default_max_qty than this file"
 #endif
 
+
 #define log_result(state, result, func) do { \
 	if (!result) { \
 		zcbor_trace_file(state); \
@@ -56,8 +57,8 @@ int cbor_encode_Pet(
 		const struct Pet *input,
 		size_t *payload_len_out)
 {
-	zcbor_state_t states[4];
+	zcbor_state_t states[4 + 0];
 
 	return zcbor_entry_function(payload, payload_len, (void *)input, payload_len_out, states,
-		(zcbor_decoder_t *)encode_Pet, sizeof(states) / sizeof(zcbor_state_t), 1);
+		(zcbor_decoder_t *)encode_Pet, sizeof(states) / sizeof(zcbor_state_t), 1, 0);
 }

src/zcbor_common.c

@@ -301,9 +301,13 @@ size_t zcbor_remaining_str_len(zcbor_state_t *state)
 
 int zcbor_entry_function(const uint8_t *payload, size_t payload_len,
 	void *result, size_t *payload_len_out, zcbor_state_t *state, zcbor_decoder_t func,
-	size_t n_states, size_t elem_count)
+	size_t n_states, size_t elem_count, size_t n_flags)
 {
-	zcbor_new_state(state, n_states, payload, payload_len, elem_count, NULL, 0);
+	zcbor_new_state(state, n_states, payload, payload_len, elem_count,
+		(uint8_t *)&state[n_states - 1 - ZCBOR_FLAG_STATES(n_flags)],
+		ZCBOR_FLAG_STATES(n_flags) * sizeof(zcbor_state_t));
+
+	state->constant_state->manually_process_elem = true;
 
 	bool ret = func(state, result);
 

src/zcbor_decode.c

@@ -959,6 +959,10 @@ bool zcbor_unordered_map_search(zcbor_decoder_t key_decoder, zcbor_state_t *stat
 			(void)old_flags;
 		}
 
+		if (!should_try_key(state)) {
+			zcbor_log("Skipping element at index %zu.\n", get_current_index(state, 0));
+		}
+
 		if (should_try_key(state) && try_key(state, key_result, key_decoder)) {
 			if (!ZCBOR_MANUALLY_PROCESS_ELEM(state)) {
 				ZCBOR_FAIL_IF(!zcbor_elem_processed(state));
@@ -1576,8 +1580,8 @@ bool zcbor_multi_decode(size_t min_decode,
 			*num_decode = i;
 			state->payload = payload_bak;
 			state->elem_count = elem_count_bak;
-			ZCBOR_ERR_IF(i < min_decode, ZCBOR_ERR_ITERATIONS);
 			zcbor_log("Found %zu elements.\r\n", i);
+			ZCBOR_ERR_IF(i < min_decode, ZCBOR_ERR_ITERATIONS);
 			return true;
 		}
 	}

tests/cases/serial_recovery.cddl

@@ -4,13 +4,11 @@
 ; SPDX-License-Identifier: Apache-2.0
 ;
 
-Member = ("image" => int) /
-	("data" => bstr) /
-	("len" => int) /
-	("off" => int) /
-	("sha" => bstr) /
-	(tstr => any)
-
 Upload = {
-	3*8members: Member
+	? "image" => uint,
+	? "len" => uint,
+	  "off" => uint,
+	? "sha" => bstr,
+	? "data" => bstr,
+	? "upgrade" => bool,
 }

tests/cases/unordered_map.cddl

@@ -0,0 +1,46 @@
+;
+; Copyright (c) 2023 Nordic Semiconductor ASA
+;
+; SPDX-License-Identifier: Apache-2.0
+;
+
+UnorderedMap1 = {
+	1 => "one",
+	2 => "two",
+	int => bstr,
+	"foo" => "bar",
+	bazboz,
+	*bstr => intlist,
+}
+
+bazboz = ("baz" => "boz")
+intlist = [*int]
+
+UnorderedMap2 = {
+	+typeUnion,
+	"map" => {
+		?-1 => bstr,
+		?-2 => bstr,
+		?bool => int,
+	}
+}
+
+typeUnion = type1 / type2 / typeDefault
+type1 = 1 => tstr
+type2 = 2 => tstr
+typeDefault = int => nil
+
+UnorderedMap3 = {
+	1*2000 uint => uint,
+	1*2000 nint => nint,
+}
+
+group1 = (
+	uint => tstr,
+	nint => bstr,
+)
+type3 = 3*3group1
+
+UnorderedMap4 = {
+	3*3group1,
+}

tests/decode/test3_simple/CMakeLists.txt

@@ -35,6 +35,7 @@ set(py_command_serial_recovery
   -d
   ${bit_arg}
   --short-names
+  --unordered-maps
 
   # Testing the --include-prefix option
   --include-prefix serial

tests/decode/test3_simple/src/main.c

@@ -487,19 +487,15 @@ ZTEST(cbor_decode_test3, test_serial1)
 	zassert_equal(ZCBOR_SUCCESS, ret, "decoding failed: %d.", ret);
 	zassert_equal(sizeof(serial_rec_input1), decode_len, NULL);
 
-	zassert_equal(5, upload.members_count,
-		"expect 5 members");
-	zassert_equal(Member_data_c, upload.members[0].members
-		.Member_choice, "expect data 1st");
-	zassert_equal(Member_image_c, upload.members[1].members
-		.Member_choice, "expect image 2nd");
-	zassert_equal(Member_len_c, upload.members[2].members
-		.Member_choice, "was %d\r\n", upload.members[2].members
-		.Member_choice);
-	zassert_equal(Member_off_c, upload.members[3].members
-		.Member_choice, "expect off 4th");
-	zassert_equal(Member_sha_c, upload.members[4].members
-		.Member_choice, "expect sha 5th");
+	zassert_true(upload.data_present, NULL);
+	zassert_equal(0x129, upload.data.data.len, NULL);
+	zassert_true(upload.image_present, NULL);
+	zassert_equal(0, upload.image.image, NULL);
+	zassert_true(upload.len_present, NULL);
+	zassert_equal(0x3b2c, upload.len.len, NULL);
+	zassert_equal(0, upload.off, NULL);
+	zassert_true(upload.sha_present, NULL);
+	zassert_equal(0x20, upload.sha.sha.len, NULL);
 }
 
 ZTEST(cbor_decode_test3, test_serial2)
@@ -511,18 +507,15 @@ ZTEST(cbor_decode_test3, test_serial2)
 	zassert_equal(ZCBOR_SUCCESS, ret, "decoding failed: %d.", ret);
 	zassert_equal(sizeof(serial_rec_input2), decode_len, NULL);
 
-	zassert_equal(5, upload.members_count,
-		"expect 5 members");
-	zassert_equal(Member_data_c, upload.members[0].members
-		.Member_choice, "expect data 1st");
-	zassert_equal(Member_image_c, upload.members[1].members
-		.Member_choice, "expect image 2nd");
-	zassert_equal(Member_len_c, upload.members[2].members
-		.Member_choice, "expect len 3rd");
-	zassert_equal(Member_off_c, upload.members[3].members
-		.Member_choice, "expect off 4th");
-	zassert_equal(Member_sha_c, upload.members[4].members
-		.Member_choice, "expect sha 5th");
+	zassert_true(upload.data_present, NULL);
+	zassert_equal(0x129, upload.data.data.len, NULL);
+	zassert_true(upload.image_present, NULL);
+	zassert_equal(0, upload.image.image, NULL);
+	zassert_true(upload.len_present, NULL);
+	zassert_equal(0x2fe0, upload.len.len, NULL);
+	zassert_equal(0, upload.off, NULL);
+	zassert_true(upload.sha_present, NULL);
+	zassert_equal(0x20, upload.sha.sha.len, NULL);
 }
 
 ZTEST_SUITE(cbor_decode_test3, NULL, NULL, NULL, NULL, NULL);

tests/decode/test9_manifest14/CMakeLists.txt

@@ -8,6 +8,7 @@ cmake_minimum_required(VERSION 3.13.1)
 
 find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
 project(test9_manifest14)
+set(MAP_SEARCH_FLAGS ON)  # Because of --unordered maps
 include(../../cmake/test_template.cmake)
 
 if (NOT MANIFEST)
@@ -30,6 +31,7 @@ set(py_command
   SUIT_Common_Sequence
   -d
   ${bit_arg}
+  --unordered-maps
   )
 
 execute_process(

tests/decode/testA_unordered_map/CMakeLists.txt

@@ -0,0 +1,40 @@
+#
+# Copyright (c) 2021 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+cmake_minimum_required(VERSION 3.13.1)
+
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+project(test9_manifest14)
+set(MAP_SEARCH_FLAGS ON)  # Because of --unordered maps
+set(MAP_SMART_SEARCH ON)
+include(../../cmake/test_template.cmake)
+
+set(py_command
+  ${PYTHON_EXECUTABLE}
+  ${CMAKE_CURRENT_LIST_DIR}/../../../zcbor/zcbor.py
+  code
+  -c ${CMAKE_CURRENT_LIST_DIR}/../../cases/unordered_map.cddl
+  --output-cmake ${PROJECT_BINARY_DIR}/unordered_map.cmake
+  -t
+  UnorderedMap1
+  UnorderedMap2
+  UnorderedMap3
+  UnorderedMap4
+  -d
+  ${bit_arg}
+  --unordered-maps
+  )
+
+execute_process(
+  COMMAND
+  ${py_command}
+  COMMAND_ERROR_IS_FATAL ANY
+)
+
+include(${PROJECT_BINARY_DIR}/unordered_map.cmake)
+
+target_link_libraries(unordered_map PRIVATE zephyr_interface)
+target_link_libraries(app PRIVATE unordered_map)

tests/decode/testA_unordered_map/prj.conf

@@ -0,0 +1,8 @@
+#
+# Copyright (c) 2021 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+CONFIG_ZTEST=y
+CONFIG_ZTEST_STACK_SIZE=131072