From b9e874a25597d5fea65dac5892bdc73b084ab607 Mon Sep 17 00:00:00 2001 From: Rob Blanckaert Date: Sun, 14 Sep 2025 13:57:22 -0700 Subject: [PATCH] WARM --- .gitignore | 1 + SConstruct | 21 +- SCsub | 37 +- config.py | 2 +- src/store.h | 2 +- src/wasi-shim.h | 6 +- src/wasm-memory.cpp | 12 +- src/wasm.cpp | 4 +- src/wasm.h | 4 + utils.py | 1 - wamr/LICENSE | 219 + wamr/include/bh_assert.h | 42 + wamr/include/bh_atomic.h | 300 + wamr/include/bh_bitmap.h | 114 + wamr/include/bh_common.h | 83 + wamr/include/bh_hashmap.h | 168 + wamr/include/bh_leb128.h | 30 + wamr/include/bh_list.h | 109 + wamr/include/bh_log.h | 94 + wamr/include/bh_platform.h | 38 + wamr/include/bh_queue.h | 80 + wamr/include/bh_vector.h | 126 + wamr/include/config.h | 78 + wamr/include/ems_gc.h | 350 + wamr/include/ems_gc_internal.h | 389 + wamr/include/gnuc.h | 14 + wamr/include/lib_export.h | 60 + wamr/include/mem_alloc.h | 94 + wamr/include/platform_api_extension.h | 1708 +++ wamr/include/platform_api_vmcore.h | 192 + wamr/include/platform_common.h | 204 + wamr/include/platform_internal.h | 136 + wamr/include/platform_wasi_types.h | 613 + wamr/include/runtime_timer.h | 51 + wamr/include/version.h | 24 + wamr/include/wasm.h | 1517 +++ wamr/include/wasm_c_api.h | 910 ++ wamr/include/wasm_c_api_internal.h | 246 + wamr/include/wasm_exec_env.h | 328 + wamr/include/wasm_export.h | 2440 ++++ wamr/include/wasm_interp.h | 122 + wamr/include/wasm_loader.h | 80 + wamr/include/wasm_loader_common.h | 68 + wamr/include/wasm_memory.h | 154 + wamr/include/wasm_native.h | 117 + wamr/include/wasm_opcode.h | 1050 ++ wamr/include/wasm_runtime.h | 912 ++ wamr/include/wasm_runtime_common.h | 1390 ++ wamr/include/wasm_shared_memory.h | 63 + wamr/include/wasm_suspend_flags.h | 52 + wamr/src/bh_assert.c | 25 + wamr/src/bh_common.c | 217 + wamr/src/bh_leb128.c | 77 + wamr/src/bh_vector.c | 279 + wamr/src/ems_alloc.c | 1169 ++ wamr/src/ems_kfc.c | 547 + wamr/src/mem_alloc.c | 252 + wamr/src/posix_thread.c | 784 ++ wamr/src/wasm_c_api.c | 5387 ++++++++ wamr/src/wasm_exec_env.c | 330 + wamr/src/wasm_interp_fast.c | 7997 +++++++++++ wamr/src/wasm_loader.c | 16983 ++++++++++++++++++++++++ wamr/src/wasm_loader_common.c | 254 + wamr/src/wasm_memory.c | 2002 +++ wamr/src/wasm_runtime.c | 5140 +++++++ wamr/src/wasm_runtime_common.c | 8007 +++++++++++ wamr/stub.cpp | 116 + wamr/update.py | 248 + 68 files changed, 64639 insertions(+), 30 deletions(-) create mode 100644 wamr/LICENSE create mode 100644 wamr/include/bh_assert.h create mode 100644 wamr/include/bh_atomic.h create mode 100644 wamr/include/bh_bitmap.h create mode 100644 wamr/include/bh_common.h create mode 100644 wamr/include/bh_hashmap.h create mode 100644 wamr/include/bh_leb128.h create mode 100644 wamr/include/bh_list.h create mode 100644 wamr/include/bh_log.h create mode 100644 wamr/include/bh_platform.h create mode 100644 wamr/include/bh_queue.h create mode 100644 wamr/include/bh_vector.h create mode 100644 wamr/include/config.h create mode 100644 wamr/include/ems_gc.h create mode 100644 wamr/include/ems_gc_internal.h create mode 100644 wamr/include/gnuc.h create mode 100644 wamr/include/lib_export.h create mode 100644 wamr/include/mem_alloc.h create mode 100644 wamr/include/platform_api_extension.h create mode 100644 wamr/include/platform_api_vmcore.h create mode 100644 wamr/include/platform_common.h create mode 100644 wamr/include/platform_internal.h create mode 100644 wamr/include/platform_wasi_types.h create mode 100644 wamr/include/runtime_timer.h create mode 100644 wamr/include/version.h create mode 100644 wamr/include/wasm.h create mode 100644 wamr/include/wasm_c_api.h create mode 100644 wamr/include/wasm_c_api_internal.h create mode 100644 wamr/include/wasm_exec_env.h create mode 100644 wamr/include/wasm_export.h create mode 100644 wamr/include/wasm_interp.h create mode 100644 wamr/include/wasm_loader.h create mode 100644 wamr/include/wasm_loader_common.h create mode 100644 wamr/include/wasm_memory.h create mode 100644 wamr/include/wasm_native.h create mode 100644 wamr/include/wasm_opcode.h create mode 100644 wamr/include/wasm_runtime.h create mode 100644 wamr/include/wasm_runtime_common.h create mode 100644 wamr/include/wasm_shared_memory.h create mode 100644 wamr/include/wasm_suspend_flags.h create mode 100644 wamr/src/bh_assert.c create mode 100644 wamr/src/bh_common.c create mode 100644 wamr/src/bh_leb128.c create mode 100644 wamr/src/bh_vector.c create mode 100644 wamr/src/ems_alloc.c create mode 100644 wamr/src/ems_kfc.c create mode 100644 wamr/src/mem_alloc.c create mode 100644 wamr/src/posix_thread.c create mode 100644 wamr/src/wasm_c_api.c create mode 100644 wamr/src/wasm_exec_env.c create mode 100644 wamr/src/wasm_interp_fast.c create mode 100644 wamr/src/wasm_loader.c create mode 100644 wamr/src/wasm_loader_common.c create mode 100644 wamr/src/wasm_memory.c create mode 100644 wamr/src/wasm_runtime.c create mode 100644 wamr/src/wasm_runtime_common.c create mode 100644 wamr/stub.cpp create mode 100644 wamr/update.py diff --git a/.gitignore b/.gitignore index 46bc54a..5a76df3 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ templates/ *.exp *.lib __pycache__ +compile_commands.json diff --git a/SConstruct b/SConstruct index acaf30a..8bb1f08 100644 --- a/SConstruct +++ b/SConstruct @@ -5,7 +5,7 @@ from utils import download_wasmer, download_wasmtime, WASMER_VER_DEFAULT, WASMTI opts = Variables([], ARGUMENTS) # Define options -opts.Add(EnumVariable("wasm_runtime", "Wasm runtime used", "wasmtime", ["wasmer", "wasmtime"])) +opts.Add(EnumVariable("wasm_runtime", "Wasm runtime used", "wasmtime", ["wasmer", "wasmtime", "wamr"])) opts.Add(BoolVariable("download_runtime", "(Re)download runtime library", "no")) opts.Add("runtime_version", "Runtime library version", None) @@ -40,21 +40,26 @@ if env["platform"] == "windows": # Defines for GDExtension specific API env.Append(CPPDEFINES=["GDEXTENSION", "LIBWASM_STATIC"]) +env.Append(CPPDEFINES=["WASM_RUNTIME_" + env["wasm_runtime"]]) + +# Godot Wasm sources +source = ["register_types.cpp", env.Glob("src/*.cpp")] + # Explicit static libraries -runtime_lib = env.File( - "{runtime}/lib/{prefix}{runtime}{suffix}".format( +if env["wasm_runtime"] != "wamr": + runtime_lib = env.File( + "{runtime}/lib/{prefix}{runtime}{suffix}".format( runtime=env["wasm_runtime"], prefix=env["LIBPREFIX"], suffix=env.get("LIBRUNTIMESUFFIX", env["LIBSUFFIX"]), + ) ) -) + env.Append(LIBS=[runtime_lib]) +else: + source += [env.Glob("wamr/src/*.c"), "wamr/stub.cpp"] # CPP includes and libraries env.Append(CPPPATH=[".", "{}/include".format(env["wasm_runtime"])]) -env.Append(LIBS=[runtime_lib]) - -# Godot Wasm sources -source = ["register_types.cpp", env.Glob("src/*.cpp")] # Builders library = env.SharedLibrary(target="addons/godot-wasm/bin/{}/godot-wasm".format(env["platform"]), source=source) diff --git a/SCsub b/SCsub index c4b2d22..5deaf1b 100644 --- a/SCsub +++ b/SCsub @@ -1,13 +1,20 @@ from utils import download_wasmer, download_wasmtime, WASMER_VER_DEFAULT, WASMTIME_VER_DEFAULT +# Import env and create module-specific clone +Import("env") + +default_runtime = "wasmtime" +if env["platform"] in ["web"]: + default_runtime = "wamr" + opts = Variables([], ARGUMENTS) -opts.Add(EnumVariable("wasm_runtime", "Wasm runtime used", "wasmtime", ["wasmer", "wasmtime"])) +opts.Add(EnumVariable("wasm_runtime", "Wasm runtime used", default_runtime, ["wasmer", "wasmtime", "wamr"])) opts.Add(BoolVariable("download_runtime", "(Re)download runtime library", "no")) opts.Add("runtime_version", "Runtime library version", None) -# Import env and create module-specific clone -Import("env") + + module_env = env.Clone() opts.Update(module_env) @@ -33,21 +40,29 @@ elif env["platform"] == "windows": env.Append(LINKFLAGS=["bcrypt.lib", "userenv.lib", "ws2_32.lib", "advapi32.lib", "ntdll.lib"]) # Explicit static libraries -runtime_lib = env.File( - "{runtime}/lib/{prefix}{runtime}{suffix}".format( - runtime=module_env["wasm_runtime"], - prefix=env["LIBPREFIX"], - suffix=env.get("LIBRUNTIMESUFFIX", env["LIBSUFFIX"]), +if module_env["wasm_runtime"] != "wamr": + runtime_lib = env.File( + "{runtime}/lib/{prefix}{runtime}{suffix}".format( + runtime=module_env["wasm_runtime"], + prefix=env["LIBPREFIX"], + suffix=env.get("LIBRUNTIMESUFFIX", env["LIBSUFFIX"]), + ) + ) + env.Append(LIBS=[runtime_lib]) +else: + module_env.add_source_files( + env.modules_sources, [env.Glob("wamr/src/*.c"), "wamr/stub.cpp"] ) -) # Linked libraries (global env) and includes (cloned env) -env.Append(LIBPATH=[env.Dir("{}/lib".format(module_env["wasm_runtime"])).abspath]) -env.Append(LIBS=[runtime_lib]) +if module_env["wasm_runtime"] != "wamr": + env.Append(LIBPATH=[env.Dir("{}/lib".format(module_env["wasm_runtime"])).abspath]) + module_env.Append(CPPPATH=[env.Dir("{}/include".format(module_env["wasm_runtime"])).abspath]) # Defines for module agnosticism module_env.Append(CPPDEFINES=["GODOT_MODULE", "LIBWASM_STATIC"]) +module_env.Append(CPPDEFINES=["WASM_RUNTIME_" + module_env["wasm_runtime"]]) # Module sources module_env.add_source_files( diff --git a/config.py b/config.py index 23eb250..1c0b901 100644 --- a/config.py +++ b/config.py @@ -1,5 +1,5 @@ def can_build(env, platform): - return platform in ["linux", "linuxbsd", "x11", "windows", "osx", "macos"] + return platform in ["linux", "linuxbsd", "x11", "windows", "osx", "macos", "web"] def configure(env): diff --git a/src/store.h b/src/store.h index f6fcff5..a0e232d 100644 --- a/src/store.h +++ b/src/store.h @@ -6,7 +6,7 @@ Singleton Wasm C API store The same store is used between all compiled Wasm modules */ -#include +#include "wasm.h" #define STORE ::godot_wasm::Store::instance().store diff --git a/src/wasi-shim.h b/src/wasi-shim.h index 20783cd..df7dfba 100644 --- a/src/wasi-shim.h +++ b/src/wasi-shim.h @@ -2,7 +2,11 @@ #define WASI_SHIM_H #include -#include +#if defined(WASM_RUNTIME_wamr) +#include "wasm_c_api.h" +#else +#include "wasm.h" +#endif #include "defs.h" namespace godot { diff --git a/src/wasm-memory.cpp b/src/wasm-memory.cpp index 1737606..b056c43 100644 --- a/src/wasm-memory.cpp +++ b/src/wasm-memory.cpp @@ -1,4 +1,4 @@ -#include +#include "wasm.h" #include "wasm-memory.h" #include "store.h" @@ -88,8 +88,9 @@ namespace godot { godot_error WasmMemory::INTERFACE_GET_DATA { FAIL_IF(memory == NULL, "Invalid memory", ERR_INVALID_DATA); - byte_t* data = wasm_memory_data(memory) + pointer; - memcpy(buffer, data, bytes); + byte_t* data = wasm_memory_data(memory); + FAIL_IF(data == NULL, "Invalid memory state", ERR_INVALID_DATA); + memcpy(buffer, data + pointer, bytes); pointer += bytes; #ifndef GODOT_MODULE *received = bytes; @@ -109,8 +110,9 @@ namespace godot { godot_error WasmMemory::INTERFACE_PUT_DATA { FAIL_IF(memory == NULL, "Invalid memory", ERR_INVALID_DATA); if (bytes <= 0) return OK; - byte_t* data = wasm_memory_data(memory) + pointer; - memcpy(data, buffer, bytes); + byte_t* data = wasm_memory_data(memory); + FAIL_IF(data == NULL, "Invalid memory state", ERR_INVALID_DATA); + memcpy(data + pointer, buffer, bytes); pointer += bytes; #ifndef GODOT_MODULE *sent = bytes; diff --git a/src/wasm.cpp b/src/wasm.cpp index 12553d4..32def49 100644 --- a/src/wasm.cpp +++ b/src/wasm.cpp @@ -455,12 +455,12 @@ namespace godot { } wasm_val_vec_t f_args; DEFER(wasm_val_vec_delete(&f_args)); - wasm_val_vec_new(&f_args, args_vec.size(), args_vec.data()); + wasm_val_vec_new(&f_args, args_vec.size(), args_vec.data()); // Construct return values wasm_val_vec_t f_results; DEFER(wasm_val_vec_delete(&f_results)); - wasm_val_vec_new_uninitialized(&f_results, context.return_count); + wasm_val_vec_new_uninitialized(&f_results, context.return_count); // Call function FAIL_IF(wasm_func_call(func, &f_args, &f_results), "Failed calling function " + name, NULL_VARIANT); diff --git a/src/wasm.h b/src/wasm.h index 8e644c1..3bd863c 100644 --- a/src/wasm.h +++ b/src/wasm.h @@ -2,7 +2,11 @@ #define GODOT_WASM_H #include +#if defined(WASM_RUNTIME_wamr) +#include "wasm_c_api.h" +#else #include +#endif #include "defs.h" #include "wasm-memory.h" diff --git a/utils.py b/utils.py index bd76a8a..7048853 100644 --- a/utils.py +++ b/utils.py @@ -10,7 +10,6 @@ WASMTIME_BASE_URL = "https://github.com/bytecodealliance/wasmtime/releases/download/{0}/wasmtime-{0}-{1}-c-api.{2}" WASMTIME_VER_DEFAULT = "v36.0.2" - def _validate_version(v): """Validate semver string""" if not re.fullmatch(r"v\d+\.\d+\.\d+(-.+)?", v): diff --git a/wamr/LICENSE b/wamr/LICENSE new file mode 100644 index 0000000..c6bd7e0 --- /dev/null +++ b/wamr/LICENSE @@ -0,0 +1,219 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +--- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + diff --git a/wamr/include/bh_assert.h b/wamr/include/bh_assert.h new file mode 100644 index 0000000..339e349 --- /dev/null +++ b/wamr/include/bh_assert.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_ASSERT_H +#define _BH_ASSERT_H + +#include "bh_platform.h" // bh_platform.h + +#ifdef __cplusplus +extern "C" { +#endif + +#if BH_DEBUG != 0 +void +bh_assert_internal(int64 v, const char *file_name, int line_number, + const char *expr_string); +#define bh_assert(expr) \ + bh_assert_internal((int64)(uintptr_t)(expr), __FILE__, __LINE__, #expr) +#else +#define bh_assert(expr) (void)0 +#endif /* end of BH_DEBUG */ + +#if !defined(__has_extension) +#define __has_extension(a) 0 +#endif + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || (defined(__GNUC__) && __GNUC__ * 0x100 + __GNUC_MINOR__ >= 0x406) \ + || __has_extension(c_static_assert) + +#define bh_static_assert(expr) _Static_assert(expr, #expr) +#else +#define bh_static_assert(expr) /* nothing */ +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* end of _BH_ASSERT_H */ diff --git a/wamr/include/bh_atomic.h b/wamr/include/bh_atomic.h new file mode 100644 index 0000000..28189d5 --- /dev/null +++ b/wamr/include/bh_atomic.h @@ -0,0 +1,300 @@ +/* + * Copyright (C) 2023 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_ATOMIC_H +#define _BH_ATOMIC_H + +#include "bh_platform.h" // bh_platform.h +#include "gnuc.h" // gnuc.h + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Why don't we use C11 stdatomics here? + * + * Unlike C11 stdatomics, + * + * - bh_atomic_xxx_t is guaranteed to have the same size as the base type. + * Thus more friendly to our AOT conventions. + * + * - It's available for C++. + * Although C++23 will have C-compatible stdatomics.h, it isn't widely + * available yet. + */ + +/* + * Note about BH_ATOMIC_32_IS_ATOMIC + * + * If BH_ATOMIC_32_IS_ATOMIC == 0, BH_ATOMIC_xxx operations defined below + * are not really atomic and require an external lock. + * + * Expected usage is: + * + * bh_atomic_32_t var = 0; + * uint32 old; + * #if BH_ATOMIC_32_IS_ATOMIC == 0 + * lock(&some_lock); + * #endif + * old = BH_ATOMIC_32_FETCH_AND(var, 1); + * #if BH_ATOMIC_32_IS_ATOMIC == 0 + * unlock(&some_lock); + * #endif + */ + +typedef uint64 bh_atomic_64_t; +typedef uint32 bh_atomic_32_t; +typedef uint16 bh_atomic_16_t; + +/* The flag can be defined by the user if the platform + * supports atomic 32-bit operations. + * If left undefined, it will be automatically defined + * according to the platform. + */ +#ifdef WASM_UINT64_IS_ATOMIC +#define BH_ATOMIC_64_IS_ATOMIC WASM_UINT64_IS_ATOMIC +#endif /* WASM_UINT64_IS_ATOMIC */ + +#ifdef WASM_UINT32_IS_ATOMIC +#define BH_ATOMIC_32_IS_ATOMIC WASM_UINT32_IS_ATOMIC +#endif /* WASM_UINT32_IS_ATOMIC */ + +#ifdef WASM_UINT16_IS_ATOMIC +#define BH_ATOMIC_16_IS_ATOMIC WASM_UINT16_IS_ATOMIC +#endif /* WASM_UINT16_IS_ATOMIC */ + +#if defined(__GNUC_PREREQ) +#if __GNUC_PREREQ(4, 7) +#define CLANG_GCC_HAS_ATOMIC_BUILTIN +#endif +#elif defined(__clang__) +#if __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 0) +#define CLANG_GCC_HAS_ATOMIC_BUILTIN +#endif +#endif + +#if defined(CLANG_GCC_HAS_ATOMIC_BUILTIN) +#ifndef BH_ATOMIC_64_IS_ATOMIC +#define BH_ATOMIC_64_IS_ATOMIC 1 +#endif +#ifndef BH_ATOMIC_32_IS_ATOMIC +#define BH_ATOMIC_32_IS_ATOMIC 1 +#endif +#ifndef BH_ATOMIC_16_IS_ATOMIC +#define BH_ATOMIC_16_IS_ATOMIC 1 +#endif +#else +#ifndef BH_ATOMIC_64_IS_ATOMIC +#define BH_ATOMIC_64_IS_ATOMIC 0 +#endif +#ifndef BH_ATOMIC_32_IS_ATOMIC +#define BH_ATOMIC_32_IS_ATOMIC 0 +#endif +#ifndef BH_ATOMIC_16_IS_ATOMIC +#define BH_ATOMIC_16_IS_ATOMIC 0 +#endif +#endif + +/* Force disable atomic 16-bit operations on bare-metal RISC-V + * because the 16-bit atomic operations is emulated by 32-bit + * atomic operations, which has linkage problem on current toolchain: + * in function `shared_memory_inc_reference': + * wasm_shared_memory.c:85:(.text.shared_memory_inc_reference+0x10): undefined + * reference to `__atomic_fetch_add_2' + */ +#ifndef WASM_UINT16_IS_ATOMIC +#if !defined(__linux__) && !defined(__FreeBSD__) && !defined(__NetBSD__) \ + && !defined(__OpenBSD__) && defined(__riscv) +#undef BH_ATOMIC_16_IS_ATOMIC +#define BH_ATOMIC_16_IS_ATOMIC 0 +#endif +#endif + +/* On some 32-bit platform, disable 64-bit atomic operations, otherwise + * undefined reference to `__atomic_load_8', if on Zephyr, can add board related + * macro in autoconf.h to control */ +#ifndef WASM_UINT64_IS_ATOMIC +#if !defined(__linux__) && !defined(__FreeBSD__) && !defined(__NetBSD__) \ + && !defined(__OpenBSD__) && (defined(__riscv) || defined(__arm__)) \ + && UINT32_MAX == UINTPTR_MAX +#undef BH_ATOMIC_64_IS_ATOMIC +#define BH_ATOMIC_64_IS_ATOMIC 0 +#endif +#endif + +#if BH_ATOMIC_64_IS_ATOMIC != 0 + +#define BH_ATOMIC_64_LOAD(v) __atomic_load_n(&(v), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_64_STORE(v, val) __atomic_store_n(&(v), val, __ATOMIC_SEQ_CST) +#define BH_ATOMIC_64_FETCH_OR(v, val) \ + __atomic_fetch_or(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_64_FETCH_AND(v, val) \ + __atomic_fetch_and(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_64_FETCH_ADD(v, val) \ + __atomic_fetch_add(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_64_FETCH_SUB(v, val) \ + __atomic_fetch_sub(&(v), (val), __ATOMIC_SEQ_CST) + +#else /* else of BH_ATOMIC_64_IS_ATOMIC != 0 */ + +#define BH_ATOMIC_64_LOAD(v) (v) +#define BH_ATOMIC_64_STORE(v, val) (v) = val +#define BH_ATOMIC_64_FETCH_OR(v, val) nonatomic_64_fetch_or(&(v), val) +#define BH_ATOMIC_64_FETCH_AND(v, val) nonatomic_64_fetch_and(&(v), val) +#define BH_ATOMIC_64_FETCH_ADD(v, val) nonatomic_64_fetch_add(&(v), val) +#define BH_ATOMIC_64_FETCH_SUB(v, val) nonatomic_64_fetch_sub(&(v), val) + +static inline uint64 +nonatomic_64_fetch_or(bh_atomic_64_t *p, uint64 val) +{ + uint64 old = *p; + *p |= val; + return old; +} + +static inline uint64 +nonatomic_64_fetch_and(bh_atomic_64_t *p, uint64 val) +{ + uint64 old = *p; + *p &= val; + return old; +} + +static inline uint64 +nonatomic_64_fetch_add(bh_atomic_64_t *p, uint64 val) +{ + uint64 old = *p; + *p += val; + return old; +} + +static inline uint64 +nonatomic_64_fetch_sub(bh_atomic_64_t *p, uint64 val) +{ + uint64 old = *p; + *p -= val; + return old; +} +#endif + +#if BH_ATOMIC_32_IS_ATOMIC != 0 + +#define BH_ATOMIC_32_LOAD(v) __atomic_load_n(&(v), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_32_STORE(v, val) __atomic_store_n(&(v), val, __ATOMIC_SEQ_CST) +#define BH_ATOMIC_32_FETCH_OR(v, val) \ + __atomic_fetch_or(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_32_FETCH_AND(v, val) \ + __atomic_fetch_and(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_32_FETCH_ADD(v, val) \ + __atomic_fetch_add(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_32_FETCH_SUB(v, val) \ + __atomic_fetch_sub(&(v), (val), __ATOMIC_SEQ_CST) + +#else /* else of BH_ATOMIC_32_IS_ATOMIC != 0 */ + +#define BH_ATOMIC_32_LOAD(v) (v) +#define BH_ATOMIC_32_STORE(v, val) (v) = val +#define BH_ATOMIC_32_FETCH_OR(v, val) nonatomic_32_fetch_or(&(v), val) +#define BH_ATOMIC_32_FETCH_AND(v, val) nonatomic_32_fetch_and(&(v), val) +#define BH_ATOMIC_32_FETCH_ADD(v, val) nonatomic_32_fetch_add(&(v), val) +#define BH_ATOMIC_32_FETCH_SUB(v, val) nonatomic_32_fetch_sub(&(v), val) + +static inline uint32 +nonatomic_32_fetch_or(bh_atomic_32_t *p, uint32 val) +{ + uint32 old = *p; + *p |= val; + return old; +} + +static inline uint32 +nonatomic_32_fetch_and(bh_atomic_32_t *p, uint32 val) +{ + uint32 old = *p; + *p &= val; + return old; +} + +static inline uint32 +nonatomic_32_fetch_add(bh_atomic_32_t *p, uint32 val) +{ + uint32 old = *p; + *p += val; + return old; +} + +static inline uint32 +nonatomic_32_fetch_sub(bh_atomic_32_t *p, uint32 val) +{ + uint32 old = *p; + *p -= val; + return old; +} + +#endif + +#if BH_ATOMIC_16_IS_ATOMIC != 0 + +#define BH_ATOMIC_16_IS_ATOMIC 1 +#define BH_ATOMIC_16_LOAD(v) __atomic_load_n(&(v), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_16_STORE(v, val) __atomic_store_n(&(v), val, __ATOMIC_SEQ_CST) +#define BH_ATOMIC_16_FETCH_OR(v, val) \ + __atomic_fetch_or(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_16_FETCH_AND(v, val) \ + __atomic_fetch_and(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_16_FETCH_ADD(v, val) \ + __atomic_fetch_add(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_16_FETCH_SUB(v, val) \ + __atomic_fetch_sub(&(v), (val), __ATOMIC_SEQ_CST) + +#else /* else of BH_ATOMIC_16_IS_ATOMIC != 0 */ + +#define BH_ATOMIC_16_LOAD(v) (v) +#define BH_ATOMIC_16_STORE(v) (v) = val +#define BH_ATOMIC_16_FETCH_OR(v, val) nonatomic_16_fetch_or(&(v), val) +#define BH_ATOMIC_16_FETCH_AND(v, val) nonatomic_16_fetch_and(&(v), val) +#define BH_ATOMIC_16_FETCH_ADD(v, val) nonatomic_16_fetch_add(&(v), val) +#define BH_ATOMIC_16_FETCH_SUB(v, val) nonatomic_16_fetch_sub(&(v), val) + +static inline uint16 +nonatomic_16_fetch_or(bh_atomic_16_t *p, uint16 val) +{ + uint16 old = *p; + *p |= val; + return old; +} + +static inline uint16 +nonatomic_16_fetch_and(bh_atomic_16_t *p, uint16 val) +{ + uint16 old = *p; + *p &= val; + return old; +} + +static inline uint16 +nonatomic_16_fetch_add(bh_atomic_16_t *p, uint16 val) +{ + uint16 old = *p; + *p += val; + return old; +} + +static inline uint16 +nonatomic_16_fetch_sub(bh_atomic_16_t *p, uint16 val) +{ + uint16 old = *p; + *p -= val; + return old; +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* end of _BH_ATOMIC_H */ diff --git a/wamr/include/bh_bitmap.h b/wamr/include/bh_bitmap.h new file mode 100644 index 0000000..285c69e --- /dev/null +++ b/wamr/include/bh_bitmap.h @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_BITMAP_H +#define _BH_BITMAP_H + +#include "bh_platform.h" // bh_platform.h + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * A simple fixed size bitmap. + */ +typedef struct bh_bitmap { + /* The first valid bit index. */ + uintptr_t begin_index; + + /* The last valid bit index plus one. */ + uintptr_t end_index; + + /* The bitmap. */ + uint8 map[1]; +} bh_bitmap; + +/** + * Create a new bitmap. + * + * @param begin_index the first valid bit index + * @param bitnum maximal bit number of the bitmap. + * + * @return the new bitmap if succeeds, NULL otherwise. + */ +bh_bitmap * +bh_bitmap_new(uintptr_t begin_index, unsigned bitnum); + +/** + * Delete a bitmap. + * + * @param bitmap the bitmap to be deleted + */ +static inline void +bh_bitmap_delete(bh_bitmap *bitmap) +{ + if (bitmap != NULL) + BH_FREE(bitmap); +} + +/** + * Check whether the given index is in the range of the bitmap. + * + * @param bitmap the bitmap + * @param n the bit index + * + * @return true if the index is in range, false otherwise + */ +static inline bool +bh_bitmap_is_in_range(bh_bitmap *bitmap, uintptr_t n) +{ + return n >= bitmap->begin_index && n < bitmap->end_index; +} + +/** + * Get a bit in the bitmap + * + * @param bitmap the bitmap + * @param n the n-th bit to be get + * + * @return value of the bit + */ +static inline int +bh_bitmap_get_bit(bh_bitmap *bitmap, uintptr_t n) +{ + uintptr_t idx = n - bitmap->begin_index; + bh_assert(n >= bitmap->begin_index && n < bitmap->end_index); + return (bitmap->map[idx / 8] >> (idx % 8)) & 1; +} + +/** + * Set a bit in the bitmap. + * + * @param bitmap the bitmap + * @param n the n-th bit to be set + */ +static inline void +bh_bitmap_set_bit(bh_bitmap *bitmap, uintptr_t n) +{ + uintptr_t idx = n - bitmap->begin_index; + bh_assert(n >= bitmap->begin_index && n < bitmap->end_index); + bitmap->map[idx / 8] |= 1 << (idx % 8); +} + +/** + * Clear a bit in the bitmap. + * + * @param bitmap the bitmap + * @param n the n-th bit to be cleared + */ +static inline void +bh_bitmap_clear_bit(bh_bitmap *bitmap, uintptr_t n) +{ + uintptr_t idx = n - bitmap->begin_index; + bh_assert(n >= bitmap->begin_index && n < bitmap->end_index); + bitmap->map[idx / 8] &= ~(1 << (idx % 8)); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wamr/include/bh_common.h b/wamr/include/bh_common.h new file mode 100644 index 0000000..725894d --- /dev/null +++ b/wamr/include/bh_common.h @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_COMMON_H +#define _BH_COMMON_H + +#include "bh_platform.h" // bh_platform.h + +#ifdef __cplusplus +extern "C" { +#endif + +#define bh_memcpy_s(dest, dlen, src, slen) \ + do { \ + int _ret = b_memcpy_s(dest, dlen, src, slen); \ + (void)_ret; \ + bh_assert(_ret == 0); \ + } while (0) + +#define bh_memcpy_wa(dest, dlen, src, slen) \ + do { \ + int _ret = b_memcpy_wa(dest, dlen, src, slen); \ + (void)_ret; \ + bh_assert(_ret == 0); \ + } while (0) + +#define bh_memmove_s(dest, dlen, src, slen) \ + do { \ + int _ret = b_memmove_s(dest, dlen, src, slen); \ + (void)_ret; \ + bh_assert(_ret == 0); \ + } while (0) + +#define bh_strcat_s(dest, dlen, src) \ + do { \ + int _ret = b_strcat_s(dest, dlen, src); \ + (void)_ret; \ + bh_assert(_ret == 0); \ + } while (0) + +#define bh_strcpy_s(dest, dlen, src) \ + do { \ + int _ret = b_strcpy_s(dest, dlen, src); \ + (void)_ret; \ + bh_assert(_ret == 0); \ + } while (0) + +int +b_memcpy_s(void *s1, unsigned int s1max, const void *s2, unsigned int n); +int +b_memcpy_wa(void *s1, unsigned int s1max, const void *s2, unsigned int n); +int +b_memmove_s(void *s1, unsigned int s1max, const void *s2, unsigned int n); +int +b_strcat_s(char *s1, unsigned int s1max, const char *s2); +int +b_strcpy_s(char *s1, unsigned int s1max, const char *s2); + +/* strdup with string allocated by BH_MALLOC */ +char * +bh_strdup(const char *s); + +/* strdup with string allocated by WA_MALLOC */ +char * +wa_strdup(const char *s); + +#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 +/* Executes a system command in bash/cmd.exe */ +int +bh_system(const char *cmd); + +/* Tests whether can create a temporary file with the given name */ +bool +bh_mkstemp(char *filename, size_t name_len); +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wamr/include/bh_hashmap.h b/wamr/include/bh_hashmap.h new file mode 100644 index 0000000..9bce58d --- /dev/null +++ b/wamr/include/bh_hashmap.h @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef WASM_HASHMAP_H +#define WASM_HASHMAP_H + +#include "bh_platform.h" // bh_platform.h + +#ifdef __cplusplus +extern "C" { +#endif + +/* Minimum initial size of hash map */ +#define HASH_MAP_MIN_SIZE 4 + +/* Maximum initial size of hash map */ +#define HASH_MAP_MAX_SIZE 65536 + +struct HashMap; +typedef struct HashMap HashMap; + +/* Hash function: to get the hash value of key. */ +typedef uint32 (*HashFunc)(const void *key); + +/* Key equal function: to check whether two keys are equal. */ +typedef bool (*KeyEqualFunc)(void *key1, void *key2); + +/* Key destroy function: to destroy the key, auto called + for each key when the hash map is destroyed. */ +typedef void (*KeyDestroyFunc)(void *key); + +/* Value destroy function: to destroy the value, auto called + for each value when the hash map is destroyed. */ +typedef void (*ValueDestroyFunc)(void *value); + +/* traverse callback function: + auto called when traverse every hash element */ +typedef void (*TraverseCallbackFunc)(void *key, void *value, void *user_data); + +/** + * Create a hash map. + * + * @param size: the initial size of the hash map + * @param use_lock whether to lock the hash map when operating on it + * @param hash_func hash function of the key, must be specified + * @param key_equal_func key equal function, check whether two keys + * are equal, must be specified + * @param key_destroy_func key destroy function, called for each key if not NULL + * when the hash map is destroyed + * @param value_destroy_func value destroy function, called for each value if + * not NULL when the hash map is destroyed + * + * @return the hash map created, NULL if failed + */ +HashMap * +bh_hash_map_create(uint32 size, bool use_lock, HashFunc hash_func, + KeyEqualFunc key_equal_func, KeyDestroyFunc key_destroy_func, + ValueDestroyFunc value_destroy_func); + +/** + * Insert an element to the hash map + * + * @param map the hash map to insert element + * @key the key of the element + * @value the value of the element + * + * @return true if success, false otherwise + * Note: fail if key is NULL or duplicated key exists in the hash map, + */ +bool +bh_hash_map_insert(HashMap *map, void *key, void *value); + +/** + * Find an element in the hash map + * + * @param map the hash map to find element + * @key the key of the element + * + * @return the value of the found element if success, NULL otherwise + */ +void * +bh_hash_map_find(HashMap *map, void *key); + +/** + * Update an element in the hash map with new value + * + * @param map the hash map to update element + * @key the key of the element + * @value the new value of the element + * @p_old_value if not NULL, copies the old value to it + * + * @return true if success, false otherwise + * Note: the old value won't be destroyed by value destroy function, + * it will be copied to p_old_value for user to process. + */ +bool +bh_hash_map_update(HashMap *map, void *key, void *value, void **p_old_value); + +/** + * Remove an element from the hash map + * + * @param map the hash map to remove element + * @key the key of the element + * @p_old_key if not NULL, copies the old key to it + * @p_old_value if not NULL, copies the old value to it + * + * @return true if success, false otherwise + * Note: the old key and old value won't be destroyed by key destroy + * function and value destroy function, they will be copied to + * p_old_key and p_old_value for user to process. + */ +bool +bh_hash_map_remove(HashMap *map, void *key, void **p_old_key, + void **p_old_value); + +/** + * Destroy the hashmap + * + * @param map the hash map to destroy + * + * @return true if success, false otherwise + * Note: the key destroy function and value destroy function will be + * called to destroy each element's key and value if they are + * not NULL. + */ +bool +bh_hash_map_destroy(HashMap *map); + +/** + * Get the structure size of HashMap + * + * @param map the hash map to calculate + * + * @return the memory space occupied by HashMap structure + */ +uint32 +bh_hash_map_get_struct_size(HashMap *hashmap); + +/** + * Get the structure size of HashMap Element + * + * @return the memory space occupied by HashMapElem structure + */ +uint32 +bh_hash_map_get_elem_struct_size(void); + +/** + * Traverse the hash map and call the callback function + * + * @param map the hash map to traverse + * @param callback the function to be called for every element + * @param user_data the argument to be passed to the callback function + * + * @return true if success, false otherwise + * Note: if the hash map has lock, the map will be locked during traverse, + * keep the callback function as simple as possible. + */ +bool +bh_hash_map_traverse(HashMap *map, TraverseCallbackFunc callback, + void *user_data); + +#ifdef __cplusplus +} +#endif + +#endif /* endof WASM_HASHMAP_H */ diff --git a/wamr/include/bh_leb128.h b/wamr/include/bh_leb128.h new file mode 100644 index 0000000..21e44a2 --- /dev/null +++ b/wamr/include/bh_leb128.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_LEB128_H +#define _BH_LEB128_H + +#include "bh_platform.h" // bh_platform.h + +typedef enum { + BH_LEB_READ_SUCCESS, + BH_LEB_READ_TOO_LONG, + BH_LEB_READ_OVERFLOW, + BH_LEB_READ_UNEXPECTED_END, +} bh_leb_read_status_t; + +#ifdef __cplusplus +extern "C" { +#endif + +bh_leb_read_status_t +bh_leb_read(const uint8 *buf, const uint8 *buf_end, uint32 maxbits, bool sign, + uint64 *p_result, size_t *p_offset); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/wamr/include/bh_list.h b/wamr/include/bh_list.h new file mode 100644 index 0000000..0b28fcc --- /dev/null +++ b/wamr/include/bh_list.h @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_LIST_H +#define _BH_LIST_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "bh_platform.h" // bh_platform.h + +/* List user should embedded bh_list_link into list elem data structure + * definition. And bh_list_link data field should be the first field. + * For example, if we would like to use bh_list for our own data type A, + * A must be defined as a structure like below: + * struct A { + * bh_list_link l; + * ... + * }; + * + * bh_list_link is defined as a structure (not typedef void*). + * It will make extend list into bi-direction easy. + */ +typedef struct bh_list_link { + struct bh_list_link *next; +} bh_list_link; + +typedef struct bh_list { + bh_list_link head; + uint32 len; +} bh_list; + +/* list operation return value */ +typedef enum bh_list_status { + BH_LIST_SUCCESS = 0, + BH_LIST_ERROR = -1 +} bh_list_status; + +/** + * Initialize a list. + * + * @param list pointer to list. + * @return BH_LIST_ERROR if OK; + * BH_LIST_ERROR if list pointer is NULL. + */ +bh_list_status +bh_list_init(bh_list *list); + +/** + * Insert an elem pointer into list. The list node memory is maintained by list + * while elem memory is the responsibility of list user. + * + * @param list pointer to list. + * @param elem pointer to elem that will be inserted into list. + * @return BH_LIST_ERROR if OK; + * BH_LIST_ERROR if input is invalid or no memory + * available. + */ +bh_list_status +bh_list_insert(bh_list *list, void *elem); + +/** + * Remove an elem pointer from list. The list node memory is maintained by list + * while elem memory is the responsibility of list user. + * + * @param list pointer to list. + * @param elem pointer to elem that will be inserted into list. + * @return BH_LIST_ERROR if OK; + * BH_LIST_ERROR if element does not exist in given + * list. + */ +bh_list_status +bh_list_remove(bh_list *list, void *elem); + +/** + * Get the list length. + * + * @param list pointer to list. + * @return the length of the list. + */ +uint32 +bh_list_length(bh_list *list); + +/** + * Get the first elem in the list. + * + * @param list pointer to list. + * @return pointer to the first node. + */ +void * +bh_list_first_elem(bh_list *list); + +/** + * Get the next elem of given list input elem. + * + * @param node pointer to list node. + * @return pointer to next list node. + */ +void * +bh_list_elem_next(void *node); + +#ifdef __cplusplus +} +#endif + +#endif /* #ifndef _BH_LIST_H */ diff --git a/wamr/include/bh_log.h b/wamr/include/bh_log.h new file mode 100644 index 0000000..7291878 --- /dev/null +++ b/wamr/include/bh_log.h @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +/** + * @file bh_log.h + * @date Tue Nov 8 18:19:10 2011 + * + * @brief This log system supports wrapping multiple outputs into one + * log message. This is useful for outputting variable-length logs + * without additional memory overhead (the buffer for concatenating + * the message), e.g. exception stack trace, which cannot be printed + * by a single log calling without the help of an additional buffer. + * Avoiding additional memory buffer is useful for resource-constraint + * systems. It can minimize the impact of log system on applications + * and logs can be printed even when no enough memory is available. + * Functions with prefix "_" are private functions. Only macros that + * are not start with "_" are exposed and can be used. + */ + +#ifndef _BH_LOG_H +#define _BH_LOG_H + +#include "bh_platform.h" // bh_platform.h + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + BH_LOG_LEVEL_FATAL = 0, + BH_LOG_LEVEL_ERROR = 1, + BH_LOG_LEVEL_WARNING = 2, + BH_LOG_LEVEL_DEBUG = 3, + BH_LOG_LEVEL_VERBOSE = 4 +} LogLevel; + +void +bh_log_set_verbose_level(uint32 level); + +#ifndef BH_LOG +void +bh_log(LogLevel log_level, const char *file, int line, const char *fmt, ...); +#else +void +BH_LOG(uint32 log_level, const char *file, int line, const char *fmt, ...); +#define bh_log BH_LOG +#endif + +#ifdef BH_PLATFORM_NUTTX + +#undef LOG_FATAL +#undef LOG_ERROR +#undef LOG_WARNING +#undef LOG_VERBOSE +#undef LOG_DEBUG + +#endif + +#if BH_DEBUG != 0 +#define LOG_FATAL(...) \ + bh_log(BH_LOG_LEVEL_FATAL, __FILE__, __LINE__, __VA_ARGS__) +#else +#define LOG_FATAL(...) \ + bh_log(BH_LOG_LEVEL_FATAL, __FUNCTION__, __LINE__, __VA_ARGS__) +#endif + +#define LOG_ERROR(...) bh_log(BH_LOG_LEVEL_ERROR, NULL, 0, __VA_ARGS__) +#define LOG_WARNING(...) bh_log(BH_LOG_LEVEL_WARNING, NULL, 0, __VA_ARGS__) +#define LOG_VERBOSE(...) bh_log(BH_LOG_LEVEL_VERBOSE, NULL, 0, __VA_ARGS__) + +#if BH_DEBUG != 0 +#define LOG_DEBUG(...) \ + bh_log(BH_LOG_LEVEL_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#else +#define LOG_DEBUG(...) (void)0 +#endif + +void +bh_print_time(const char *prompt); + +void +bh_print_proc_mem(const char *prompt); + +void +bh_log_proc_mem(const char *function, uint32 line); + +#define LOG_PROC_MEM(...) bh_log_proc_mem(__FUNCTION__, __LINE__) + +#ifdef __cplusplus +} +#endif + +#endif /* _BH_LOG_H */ diff --git a/wamr/include/bh_platform.h b/wamr/include/bh_platform.h new file mode 100644 index 0000000..65ffc3f --- /dev/null +++ b/wamr/include/bh_platform.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_PLATFORM_H +#define _BH_PLATFORM_H + +#include "platform_common.h" // ../platform/include/platform_common.h +#include "platform_api_vmcore.h" // ../platform/include/platform_api_vmcore.h +#include "platform_api_extension.h" // ../platform/include/platform_api_extension.h +#include "bh_assert.h" // bh_assert.h +#include "bh_common.h" // bh_common.h +#include "bh_hashmap.h" // bh_hashmap.h +#include "bh_list.h" // bh_list.h +#include "bh_log.h" // bh_log.h +#include "bh_queue.h" // bh_queue.h +#include "bh_vector.h" // bh_vector.h +#include "runtime_timer.h" // runtime_timer.h + +/** + * WA_MALLOC/WA_FREE need to be redefined for both + * runtime native and WASM app respectively. + * + * Some source files are shared for building native and WASM, + * and this the mem allocator API for these files. + * + * Here we define it for the native world + */ +#ifndef WA_MALLOC +#define WA_MALLOC wasm_runtime_malloc +#endif + +#ifndef WA_FREE +#define WA_FREE wasm_runtime_free +#endif + +#endif /* #ifndef _BH_PLATFORM_H */ diff --git a/wamr/include/bh_queue.h b/wamr/include/bh_queue.h new file mode 100644 index 0000000..341fd81 --- /dev/null +++ b/wamr/include/bh_queue.h @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_QUEUE_H +#define _BH_QUEUE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "bh_platform.h" // bh_platform.h + +struct bh_queue_node; +typedef struct bh_queue_node *bh_message_t; +struct bh_queue; +typedef struct bh_queue bh_queue; + +typedef void (*bh_queue_handle_msg_callback)(void *message, void *arg); + +#define bh_queue_malloc BH_MALLOC +#define bh_queue_free BH_FREE + +#define bh_queue_mutex korp_mutex +#define bh_queue_cond korp_cond + +#define bh_queue_mutex_init os_mutex_init +#define bh_queue_mutex_destroy os_mutex_destroy +#define bh_queue_mutex_lock os_mutex_lock +#define bh_queue_mutex_unlock os_mutex_unlock + +#define bh_queue_cond_init os_cond_init +#define bh_queue_cond_destroy os_cond_destroy +#define bh_queue_cond_wait os_cond_wait +#define bh_queue_cond_timedwait os_cond_reltimedwait +#define bh_queue_cond_signal os_cond_signal +#define bh_queue_cond_broadcast os_cond_broadcast + +typedef void (*bh_msg_cleaner)(void *msg); + +bh_queue * +bh_queue_create(void); + +void +bh_queue_destroy(bh_queue *queue); + +char * +bh_message_payload(bh_message_t message); +uint32 +bh_message_payload_len(bh_message_t message); +int +bh_message_type(bh_message_t message); + +bh_message_t +bh_new_msg(unsigned short tag, void *body, unsigned int len, void *handler); +void +bh_free_msg(bh_message_t msg); +bool +bh_post_msg(bh_queue *queue, unsigned short tag, void *body, unsigned int len); +bool +bh_post_msg2(bh_queue *queue, bh_message_t msg); + +bh_message_t +bh_get_msg(bh_queue *queue, uint64 timeout_us); + +unsigned +bh_queue_get_message_count(bh_queue *queue); + +void +bh_queue_enter_loop_run(bh_queue *queue, bh_queue_handle_msg_callback handle_cb, + void *arg); +void +bh_queue_exit_loop_run(bh_queue *queue); + +#ifdef __cplusplus +} +#endif + +#endif /* #ifndef _BH_QUEUE_H */ diff --git a/wamr/include/bh_vector.h b/wamr/include/bh_vector.h new file mode 100644 index 0000000..143fc66 --- /dev/null +++ b/wamr/include/bh_vector.h @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_VECTOR_H +#define _WASM_VECTOR_H + +#include "bh_platform.h" // bh_platform.h + +#ifdef __cplusplus +extern "C" { +#endif + +#define DEFAULT_VECTOR_INIT_SIZE 8 + +typedef struct Vector { + /* max element number */ + size_t max_elems; + /* vector data allocated */ + uint8 *data; + /* current element num */ + size_t num_elems; + /* size of each element */ + size_t size_elem; + void *lock; +} Vector; + +/** + * Initialize vector + * + * @param vector the vector to init + * @param init_length the initial length of the vector + * @param size_elem size of each element + * + * @return true if success, false otherwise + */ +bool +bh_vector_init(Vector *vector, size_t init_length, size_t size_elem, + bool use_lock); + +/** + * Set element of vector + * + * @param vector the vector to set + * @param index the index of the element to set + * @param elem_buf the element buffer which stores the element data + * + * @return true if success, false otherwise + */ +bool +bh_vector_set(Vector *vector, uint32 index, const void *elem_buf); + +/** + * Get element of vector + * + * @param vector the vector to get + * @param index the index of the element to get + * @param elem_buf the element buffer to store the element data, + * whose length must be no less than element size + * + * @return true if success, false otherwise + */ +bool +bh_vector_get(Vector *vector, uint32 index, void *elem_buf); + +/** + * Insert element of vector + * + * @param vector the vector to insert + * @param index the index of the element to insert + * @param elem_buf the element buffer which stores the element data + * + * @return true if success, false otherwise + */ +bool +bh_vector_insert(Vector *vector, uint32 index, const void *elem_buf); + +/** + * Append element to the end of vector + * + * @param vector the vector to append + * @param elem_buf the element buffer which stores the element data + * + * @return true if success, false otherwise + */ +bool +bh_vector_append(Vector *vector, const void *elem_buf); + +/** + * Remove element from vector + * + * @param vector the vector to remove element + * @param index the index of the element to remove + * @param old_elem_buf if not NULL, copies the element data to the buffer + * + * @return true if success, false otherwise + */ +bool +bh_vector_remove(Vector *vector, uint32 index, void *old_elem_buf); + +/** + * Return the size of the vector + * + * @param vector the vector to get size + * + * @return return the size of the vector + */ +size_t +bh_vector_size(const Vector *vector); + +/** + * Destroy the vector + * + * @param vector the vector to destroy + * + * @return true if success, false otherwise + */ +bool +bh_vector_destroy(Vector *vector); + +#ifdef __cplusplus +} +#endif + +#endif /* endof _WASM_VECTOR_H */ diff --git a/wamr/include/config.h b/wamr/include/config.h new file mode 100644 index 0000000..6921ab3 --- /dev/null +++ b/wamr/include/config.h @@ -0,0 +1,78 @@ + +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* Replaced by godot-wasm */ + +#define WASM_ENABLE_INTERP 1 +#define WASM_ENABLE_FAST_INTERP 1 +#define WASM_ENABLE_SHARED_HEAP 0 +#define WASM_ENABLE_SHARED_MEMORY 0 +#define WASM_ENABLE_LIBC_WASI 0 +#define WASM_ENABLE_GC 0 +#define WASM_ENABLE_WASM_CACHE 0 +#define WASM_ENABLE_INVOKE_NATIVE 0 +#define WASM_ENABLE_JIT 0 +#define WASM_ENABLE_FAST_JIT 0 +#define WASM_ENABLE_SIMDE 0 + +#define BH_MALLOC wasm_runtime_malloc +#define BH_FREE wasm_runtime_free + +#ifndef WASM_TABLE_MAX_SIZE +#define WASM_TABLE_MAX_SIZE 1024 +#endif + +/* Default wasm block address cache size and conflict list size */ +#ifndef BLOCK_ADDR_CACHE_SIZE +#define BLOCK_ADDR_CACHE_SIZE 64 +#endif +#define BLOCK_ADDR_CONFLICT_SIZE 2 + +#ifndef DEFAULT_QUEUE_LENGTH +#define DEFAULT_QUEUE_LENGTH 50 +#endif + +/* Default min/max heap size of each app */ +#ifndef APP_HEAP_SIZE_DEFAULT +#define APP_HEAP_SIZE_DEFAULT (8 * 1024) +#endif +#define APP_HEAP_SIZE_MIN (256) + +/* The ems memory allocator supports maximal heap size 1GB, + see ems_gc_internal.h */ +#define APP_HEAP_SIZE_MAX (1024 * 1024 * 1024) + +/* Default/min native stack size of each app thread */ +#if defined(PTHREAD_STACK_DEFAULT) && defined(PTHREAD_STACK_MIN) +#define APP_THREAD_STACK_SIZE_DEFAULT PTHREAD_STACK_DEFAULT +#define APP_THREAD_STACK_SIZE_MIN PTHREAD_STACK_MIN +#else +#define APP_THREAD_STACK_SIZE_DEFAULT (128 * 1024) +#define APP_THREAD_STACK_SIZE_MIN (24 * 1024) +#endif + +/* Max native stack size of each app thread */ +#if !defined(APP_THREAD_STACK_SIZE_MAX) +#define APP_THREAD_STACK_SIZE_MAX (8 * 1024 * 1024) +#endif + +#ifndef WASM_CONST_EXPR_STACK_SIZE +#if WASM_ENABLE_GC != 0 +#define WASM_CONST_EXPR_STACK_SIZE 8 +#else +#define WASM_CONST_EXPR_STACK_SIZE 4 +#endif +#endif + +#define WASM_STACK_GUARD_SIZE (4096 * 6) +#define DEFAULT_WASM_STACK_SIZE (16 * 1024) +#define AOT_CURRENT_VERSION 5 + + +#ifndef _CONFIG_H_ +#define _CONFIG_H_ + +#endif /* end of _CONFIG_H_ */ diff --git a/wamr/include/ems_gc.h b/wamr/include/ems_gc.h new file mode 100644 index 0000000..755fc7a --- /dev/null +++ b/wamr/include/ems_gc.h @@ -0,0 +1,350 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/** + * @file ems_gc.h + * @date Wed Aug 3 10:46:38 2011 + * + * @brief This file defines GC modules types and interfaces. + */ + +#ifndef _EMS_GC_H +#define _EMS_GC_H + +#include "bh_platform.h" // bh_platform.h + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef GC_STAT_DATA +#define GC_STAT_DATA 0 +#endif + +#ifndef GC_STAT_SHOW +#define GC_STAT_SHOW 0 +#endif + +#ifndef GC_IN_EVERY_ALLOCATION +#define GC_IN_EVERY_ALLOCATION 0 +#endif + +#ifndef GC_MANUALLY +#define GC_MANUALLY 0 +#endif + +#define GC_HEAD_PADDING 4 + +#ifndef NULL_REF +#define NULL_REF ((gc_object_t)NULL) +#endif + +#define GC_SUCCESS (0) +#define GC_ERROR (-1) + +#define GC_TRUE (1) +#define GC_FALSE (0) + +#define GC_MAX_HEAP_SIZE (256 * BH_KB) + +typedef void *gc_handle_t; +typedef void *gc_object_t; +typedef uint64 gc_uint64; +typedef int64 gc_int64; +typedef uint32 gc_uint32; +typedef int32 gc_int32; +typedef uint16 gc_uint16; +typedef int16 gc_int16; +typedef uint8 gc_uint8; +typedef int8 gc_int8; +typedef uint32 gc_size_t; + +typedef enum { + GC_STAT_TOTAL = 0, + GC_STAT_FREE, + GC_STAT_HIGHMARK, + GC_STAT_COUNT, + GC_STAT_TIME, + GC_STAT_MAX +} GC_STAT_INDEX; + +#ifndef GC_FINALIZER_T_DEFINED +#define GC_FINALIZER_T_DEFINED +typedef void (*gc_finalizer_t)(void *obj, void *data); +#endif + +#ifndef EXTRA_INFO_NORMAL_NODE_CNT +#define EXTRA_INFO_NORMAL_NODE_CNT 32 +#endif + +/* extra information attached to specific object */ +typedef struct extra_info_node { + gc_object_t obj; + gc_finalizer_t finalizer; + void *data; +} extra_info_node_t; + +/** + * GC initialization from a buffer, which is separated into + * two parts: the beginning of the buffer is used to create + * the heap structure, and the left is used to create the + * actual pool data + * + * @param buf the buffer to be initialized to a heap + * @param buf_size the size of buffer + * + * @return gc handle if success, NULL otherwise + */ +gc_handle_t +gc_init_with_pool(char *buf, gc_size_t buf_size); + +/** + * GC initialization from heap struct buffer and pool buffer + * + * @param struct_buf the struct buffer to create the heap structure + * @param struct_buf_size the size of struct buffer + * @param pool_buf the pool buffer to create pool data + * @param pool_buf_size the size of poll buffer + * + * @return gc handle if success, NULL otherwise + */ +gc_handle_t +gc_init_with_struct_and_pool(char *struct_buf, gc_size_t struct_buf_size, + char *pool_buf, gc_size_t pool_buf_size); + +/** + * Destroy heap which is initialized from a buffer + * + * @param handle handle to heap needed destroy + * + * @return GC_SUCCESS if success + * GC_ERROR for bad parameters or failed system resource freeing. + */ +int +gc_destroy_with_pool(gc_handle_t handle); + +#if WASM_ENABLE_GC != 0 +/** + * Enable or disable GC reclaim for a heap + * + * @param handle handle of the heap + * @param exec_env the exec_env of current module instance + */ +#if WASM_ENABLE_THREAD_MGR == 0 +void +gc_enable_gc_reclaim(gc_handle_t handle, void *exec_env); +#else +/** + * Enable or disable GC reclaim for a heap + * + * @param handle handle of the heap + * @param cluster the tread cluster of current module instance + */ +void +gc_enable_gc_reclaim(gc_handle_t handle, void *cluster); +#endif +#endif + +/** + * Return heap struct size + */ +uint32 +gc_get_heap_struct_size(void); + +/** + * Migrate heap from one pool buf to another pool buf + * + * @param handle handle of the new heap + * @param pool_buf_new the new pool buffer + * @param pool_buf_size the size of new pool buffer + * + * @return GC_SUCCESS if success, GC_ERROR otherwise + */ +int +gc_migrate(gc_handle_t handle, char *pool_buf_new, gc_size_t pool_buf_size); + +/** + * Check whether the heap is corrupted + * + * @param handle handle of the heap + * + * @return true if success, false otherwise + */ +bool +gc_is_heap_corrupted(gc_handle_t handle); + +/** + * Get Heap Stats + * + * @param stats [out] integer array to save heap stats + * @param size [in] the size of stats + * @param mmt [in] type of heap, MMT_SHARED or MMT_INSTANCE + */ +void * +gc_heap_stats(void *heap, uint32 *stats, int size); + +#if BH_ENABLE_GC_VERIFY == 0 + +gc_object_t +gc_alloc_vo(void *heap, gc_size_t size); + +gc_object_t +gc_realloc_vo(void *heap, void *ptr, gc_size_t size); + +int +gc_free_vo(void *heap, gc_object_t obj); + +#if WASM_ENABLE_GC != 0 +gc_object_t +gc_alloc_wo(void *heap, gc_size_t size); + +void +gc_free_wo(void *vheap, void *ptr); +#endif + +#else /* else of BH_ENABLE_GC_VERIFY */ + +gc_object_t +gc_alloc_vo_internal(void *heap, gc_size_t size, const char *file, int line); + +gc_object_t +gc_realloc_vo_internal(void *heap, void *ptr, gc_size_t size, const char *file, + int line); + +int +gc_free_vo_internal(void *heap, gc_object_t obj, const char *file, int line); + +#if WASM_ENABLE_GC != 0 +gc_object_t +gc_alloc_wo_internal(void *heap, gc_size_t size, const char *file, int line); + +void +gc_free_wo_internal(void *vheap, void *ptr, const char *file, int line); +#endif + +/* clang-format off */ +#define gc_alloc_vo(heap, size) \ + gc_alloc_vo_internal(heap, size, __FILE__, __LINE__) + +#define gc_realloc_vo(heap, ptr, size) \ + gc_realloc_vo_internal(heap, ptr, size, __FILE__, __LINE__) + +#define gc_free_vo(heap, obj) \ + gc_free_vo_internal(heap, obj, __FILE__, __LINE__) + +#if WASM_ENABLE_GC != 0 +#define gc_alloc_wo(heap, size) \ + gc_alloc_wo_internal(heap, size, __FILE__, __LINE__) + +#define gc_free_wo(heap, obj) \ + gc_free_wo_internal(heap, obj, __FILE__, __LINE__) +#endif +/* clang-format on */ + +#endif /* end of BH_ENABLE_GC_VERIFY */ + +#if WASM_ENABLE_GC != 0 +/** + * Add gc object ref to the rootset of a gc heap. + * + * @param heap the heap to add the gc object to its rootset + * @param obj pointer to a valid WASM object managed by the gc heap. + * + * @return GC_SUCCESS if success, GC_ERROR otherwise + */ +int +gc_add_root(void *heap, gc_object_t obj); + +int +gci_gc_heap(void *heap); + +extra_info_node_t * +gc_search_extra_info_node(gc_handle_t handle, gc_object_t obj, + gc_size_t *p_index); + +/** + * Set finalizer to the given object, if another finalizer is set to the same + * object, the previous one will be cancelled + * + * @param handle handle of the heap + * @param obj object to set finalizer + * @param cb finalizer function to be called before this object is freed + * @param data custom data to be passed to finalizer function + * + * @return true if success, false otherwise + */ +bool +gc_set_finalizer(gc_handle_t handle, gc_object_t obj, gc_finalizer_t cb, + void *data); + +/** + * Unset finalizer to the given object + * + * @param handle handle of the heap + * @param obj object to unset finalizer + */ +void +gc_unset_finalizer(gc_handle_t handle, gc_object_t obj); + +#if WASM_ENABLE_THREAD_MGR == 0 +bool +wasm_runtime_traverse_gc_rootset(void *exec_env, void *heap); +#else +bool +wasm_runtime_traverse_gc_rootset(void *cluster, void *heap); +#endif + +bool +wasm_runtime_get_wasm_object_ref_list(gc_object_t obj, bool *p_is_compact_mode, + gc_uint32 *p_ref_num, + gc_uint16 **p_ref_list, + gc_uint32 *p_ref_start_offset); + +bool +wasm_runtime_get_wasm_object_extra_info_flag(gc_object_t obj); + +void +wasm_runtime_set_wasm_object_extra_info_flag(gc_object_t obj, bool set); + +void +wasm_runtime_gc_prepare(void *exec_env); + +void +wasm_runtime_gc_finalize(void *exec_env); +#endif /* end of WASM_ENABLE_GC != 0 */ + +#define GC_HEAP_STAT_SIZE (128 / 4) + +typedef struct { + int usage; + int usage_block; + int vo_usage; + int wo_usage; + int free; + int free_block; + int vo_free; + int wo_free; + int usage_sizes[GC_HEAP_STAT_SIZE]; + int free_sizes[GC_HEAP_STAT_SIZE]; +} gc_stat_t; + +void +gc_show_stat(gc_handle_t handle); + +#if WASM_ENABLE_GC != 0 +void +gc_show_fragment(gc_handle_t handle); + +#if WASM_ENABLE_GC_PERF_PROFILING != 0 +void +gc_dump_perf_profiling(gc_handle_t *handle); +#endif +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wamr/include/ems_gc_internal.h b/wamr/include/ems_gc_internal.h new file mode 100644 index 0000000..b5c2106 --- /dev/null +++ b/wamr/include/ems_gc_internal.h @@ -0,0 +1,389 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _EMS_GC_INTERNAL_H +#define _EMS_GC_INTERNAL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "bh_platform.h" // bh_platform.h +#include "ems_gc.h" // ems_gc.h + +/* HMU (heap memory unit) basic block type */ +typedef enum hmu_type_enum { + HMU_TYPE_MIN = 0, + HMU_TYPE_MAX = 3, + HMU_WO = 3, /* WASM Object */ + HMU_VO = 2, /* VM Object */ + HMU_FC = 1, + HMU_FM = 0 +} hmu_type_t; + +typedef struct hmu_struct { + gc_uint32 header; +} hmu_t; + +#if BH_ENABLE_GC_VERIFY != 0 + +#if UINTPTR_MAX > UINT32_MAX +/* 2 prefix paddings for 64-bit pointer */ +#define GC_OBJECT_PREFIX_PADDING_CNT 2 +#else +/* 3 prefix paddings for 32-bit pointer */ +#define GC_OBJECT_PREFIX_PADDING_CNT 3 +#endif +#define GC_OBJECT_SUFFIX_PADDING_CNT 4 +#define GC_OBJECT_PADDING_VALUE (0x12345678) + +typedef struct gc_object_prefix { + const char *file_name; + gc_int32 line_no; + gc_int32 size; + gc_uint32 padding[GC_OBJECT_PREFIX_PADDING_CNT]; +} gc_object_prefix_t; + +typedef struct gc_object_suffix { + gc_uint32 padding[GC_OBJECT_SUFFIX_PADDING_CNT]; +} gc_object_suffix_t; + +#define OBJ_PREFIX_SIZE (sizeof(gc_object_prefix_t)) +#define OBJ_SUFFIX_SIZE (sizeof(gc_object_suffix_t)) + +void +hmu_init_prefix_and_suffix(hmu_t *hmu, gc_size_t tot_size, + const char *file_name, int line_no); + +void +hmu_verify(void *vheap, hmu_t *hmu); + +#define SKIP_OBJ_PREFIX(p) ((void *)((gc_uint8 *)(p) + OBJ_PREFIX_SIZE)) +#define SKIP_OBJ_SUFFIX(p) ((void *)((gc_uint8 *)(p) + OBJ_SUFFIX_SIZE)) + +#define OBJ_EXTRA_SIZE (HMU_SIZE + OBJ_PREFIX_SIZE + OBJ_SUFFIX_SIZE) + +#else /* else of BH_ENABLE_GC_VERIFY */ + +#define OBJ_PREFIX_SIZE 0 +#define OBJ_SUFFIX_SIZE 0 + +#define SKIP_OBJ_PREFIX(p) ((void *)((gc_uint8 *)(p) + OBJ_PREFIX_SIZE)) +#define SKIP_OBJ_SUFFIX(p) ((void *)((gc_uint8 *)(p) + OBJ_SUFFIX_SIZE)) + +#define OBJ_EXTRA_SIZE (HMU_SIZE + OBJ_PREFIX_SIZE + OBJ_SUFFIX_SIZE) + +#endif /* end of BH_ENABLE_GC_VERIFY */ + +#define hmu_obj_size(s) ((s)-OBJ_EXTRA_SIZE) + +#define GC_ALIGN_8(s) (((uint32)(s) + 7) & (uint32)~7) + +#define GC_SMALLEST_SIZE \ + GC_ALIGN_8(HMU_SIZE + OBJ_PREFIX_SIZE + OBJ_SUFFIX_SIZE + 8) +#define GC_GET_REAL_SIZE(x) \ + GC_ALIGN_8(HMU_SIZE + OBJ_PREFIX_SIZE + OBJ_SUFFIX_SIZE \ + + (((x) > 8) ? (x) : 8)) + +/** + * hmu bit operation + */ + +#define SETBIT(v, offset) (v) |= ((uint32)1 << (offset)) +#define GETBIT(v, offset) ((v) & ((uint32)1 << (offset)) ? 1 : 0) +#define CLRBIT(v, offset) (v) &= (~((uint32)1 << (offset))) + +/* clang-format off */ +#define SETBITS(v, offset, size, value) \ + do { \ + (v) &= ~((((uint32)1 << size) - 1) << offset); \ + (v) |= ((uint32)value << offset); \ + } while (0) +#define CLRBITS(v, offset, size) \ + (v) &= ~((((uint32)1 << size) - 1) << offset) +#define GETBITS(v, offset, size) \ + (((v) & (((((uint32)1 << size) - 1) << offset))) >> offset) +/* clang-format on */ + +/** + * gc object layout definition + */ + +#define HMU_SIZE (sizeof(hmu_t)) + +#define hmu_to_obj(hmu) (gc_object_t)(SKIP_OBJ_PREFIX((hmu_t *)(hmu) + 1)) +#define obj_to_hmu(obj) ((hmu_t *)((gc_uint8 *)(obj)-OBJ_PREFIX_SIZE) - 1) + +#define HMU_UT_SIZE 2 +#define HMU_UT_OFFSET 30 + +/* clang-format off */ +#define hmu_get_ut(hmu) \ + GETBITS((hmu)->header, HMU_UT_OFFSET, HMU_UT_SIZE) +#define hmu_set_ut(hmu, type) \ + SETBITS((hmu)->header, HMU_UT_OFFSET, HMU_UT_SIZE, type) +#define hmu_is_ut_valid(tp) \ + (tp >= HMU_TYPE_MIN && tp <= HMU_TYPE_MAX) +/* clang-format on */ + +/* P in use bit means the previous chunk is in use */ +#define HMU_P_OFFSET 29 + +#define hmu_mark_pinuse(hmu) SETBIT((hmu)->header, HMU_P_OFFSET) +#define hmu_unmark_pinuse(hmu) CLRBIT((hmu)->header, HMU_P_OFFSET) +#define hmu_get_pinuse(hmu) GETBIT((hmu)->header, HMU_P_OFFSET) + +#define HMU_WO_VT_SIZE 27 +#define HMU_WO_VT_OFFSET 0 +#define HMU_WO_MB_OFFSET 28 + +#define hmu_mark_wo(hmu) SETBIT((hmu)->header, HMU_WO_MB_OFFSET) +#define hmu_unmark_wo(hmu) CLRBIT((hmu)->header, HMU_WO_MB_OFFSET) +#define hmu_is_wo_marked(hmu) GETBIT((hmu)->header, HMU_WO_MB_OFFSET) + +/** + * The hmu size is divisible by 8, its lowest 3 bits are 0, so we only + * store its higher bits of bit [29..3], and bit [2..0] are not stored. + * After that, the maximal heap size can be enlarged from (1<<27) = 128MB + * to (1<<27) * 8 = 1GB. + */ +#define HMU_SIZE_SIZE 27 +#define HMU_SIZE_OFFSET 0 + +#define HMU_VO_FB_OFFSET 28 + +#define hmu_is_vo_freed(hmu) GETBIT((hmu)->header, HMU_VO_FB_OFFSET) +#define hmu_unfree_vo(hmu) CLRBIT((hmu)->header, HMU_VO_FB_OFFSET) + +#define hmu_get_size(hmu) \ + (GETBITS((hmu)->header, HMU_SIZE_OFFSET, HMU_SIZE_SIZE) << 3) +#define hmu_set_size(hmu, size) \ + SETBITS((hmu)->header, HMU_SIZE_OFFSET, HMU_SIZE_SIZE, ((size) >> 3)) + +/** + * HMU free chunk management + */ + +#ifndef HMU_NORMAL_NODE_CNT +#define HMU_NORMAL_NODE_CNT 32 +#endif +#define HMU_FC_NORMAL_MAX_SIZE ((HMU_NORMAL_NODE_CNT - 1) << 3) +#define HMU_IS_FC_NORMAL(size) ((size) < HMU_FC_NORMAL_MAX_SIZE) +#if HMU_FC_NORMAL_MAX_SIZE >= GC_MAX_HEAP_SIZE +#error "Too small GC_MAX_HEAP_SIZE" +#endif + +typedef struct hmu_normal_node { + hmu_t hmu_header; + gc_int32 next_offset; +} hmu_normal_node_t; + +typedef struct hmu_normal_list { + hmu_normal_node_t *next; +} hmu_normal_list_t; + +static inline hmu_normal_node_t * +get_hmu_normal_node_next(hmu_normal_node_t *node) +{ + return node->next_offset + ? (hmu_normal_node_t *)((uint8 *)node + node->next_offset) + : NULL; +} + +static inline void +set_hmu_normal_node_next(hmu_normal_node_t *node, hmu_normal_node_t *next) +{ + if (next) { + bh_assert((uint8 *)next - (uint8 *)node < INT32_MAX); + node->next_offset = (gc_int32)(intptr_t)((uint8 *)next - (uint8 *)node); + } + else { + node->next_offset = 0; + } +} + +/** + * Define hmu_tree_node as a packed struct, since it is at the 4-byte + * aligned address and the size of hmu_head is 4, so in 64-bit target, + * the left/right/parent fields will be at 8-byte aligned address, + * we can access them directly. + */ +#if UINTPTR_MAX == UINT64_MAX +#if defined(_MSC_VER) +__pragma(pack(push, 1)); +#define __attr_packed +#define __attr_aligned(a) +#elif defined(__GNUC__) || defined(__clang__) +#define __attr_packed __attribute__((packed)) +#define __attr_aligned(a) __attribute__((aligned(a))) +#else +#error "packed attribute isn't used to define struct hmu_tree_node" +#endif +#else /* else of UINTPTR_MAX == UINT64_MAX */ +#define __attr_packed +#define __attr_aligned(a) +#endif + +typedef struct hmu_tree_node { + hmu_t hmu_header; + struct hmu_tree_node *left; + struct hmu_tree_node *right; + struct hmu_tree_node *parent; + gc_size_t size; +} __attr_packed __attr_aligned(4) hmu_tree_node_t; + +#if UINTPTR_MAX == UINT64_MAX +#if defined(_MSC_VER) +__pragma(pack(pop)); +#endif +#endif + +bh_static_assert(sizeof(hmu_tree_node_t) == 8 + 3 * sizeof(void *)); +bh_static_assert(offsetof(hmu_tree_node_t, left) == 4); + +#define ASSERT_TREE_NODE_ALIGNED_ACCESS(tree_node) \ + do { \ + bh_assert((((uintptr_t)&tree_node->left) & (sizeof(uintptr_t) - 1)) \ + == 0); \ + } while (0) + +typedef struct gc_heap_struct { + /* for double checking*/ + gc_handle_t heap_id; + + gc_uint8 *base_addr; + gc_size_t current_size; + + korp_mutex lock; + + hmu_normal_list_t kfc_normal_list[HMU_NORMAL_NODE_CNT]; + +#if UINTPTR_MAX == UINT64_MAX + /* make kfc_tree_root_buf 4-byte aligned and not 8-byte aligned, + so kfc_tree_root's left/right/parent fields are 8-byte aligned + and we can access them directly */ + uint32 __padding; +#endif + uint8 kfc_tree_root_buf[sizeof(hmu_tree_node_t)]; + /* point to kfc_tree_root_buf, the order in kfc_tree is: + size[left] <= size[cur] < size[right] */ + hmu_tree_node_t *kfc_tree_root; + +#if WASM_ENABLE_GC != 0 + /* for rootset enumeration of private heap*/ + void *root_set; + +#if WASM_ENABLE_THREAD_MGR == 0 + /* exec_env of current wasm module instance */ + void *exec_env; +#else + /* thread cluster of current module instances */ + void *cluster; +#endif + + /* whether the fast mode of marking process that requires + additional memory fails. When the fast mode fails, the + marking process can still be done in the slow mode, which + doesn't need additional memory (by walking through all + blocks and marking successors of marked nodes until no new + node is marked). TODO: slow mode is not implemented. */ + unsigned is_fast_marking_failed : 1; + + /* whether the heap is doing reclaim */ + unsigned is_doing_reclaim : 1; + + /* Whether the heap can do reclaim */ + unsigned is_reclaim_enabled : 1; +#endif + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + /* whether heap is corrupted, e.g. the hmu nodes are modified + by user */ + bool is_heap_corrupted; +#endif + + gc_size_t init_size; + gc_size_t highmark_size; + gc_size_t total_free_size; + +#if WASM_ENABLE_GC != 0 + gc_size_t gc_threshold; + gc_size_t gc_threshold_factor; + gc_size_t total_gc_count; + gc_size_t total_gc_time; + gc_size_t max_gc_time; + /* Usually there won't be too many extra info node, so we try to use a fixed + * array to store them, if the fixed array don't have enough space to store + * the nodes, a new space will be allocated from heap */ + extra_info_node_t *extra_info_normal_nodes[EXTRA_INFO_NORMAL_NODE_CNT]; + /* Used to store extra information such as finalizer for specified nodes, we + * introduce a separate space to store these information so only nodes who + * really require extra information will occupy additional memory spaces. */ + extra_info_node_t **extra_info_nodes; + gc_size_t extra_info_node_cnt; + gc_size_t extra_info_node_capacity; +#endif +#if GC_STAT_DATA != 0 + gc_uint64 total_size_allocated; + gc_uint64 total_size_freed; +#endif +} gc_heap_t; + +#if WASM_ENABLE_GC != 0 + +#define GC_DEFAULT_THRESHOLD_FACTOR 300 + +static inline void +gc_update_threshold(gc_heap_t *heap) +{ + uint64_t result = (uint64_t)heap->total_free_size + * (uint64_t)heap->gc_threshold_factor / 1000; + /* heap->total_free_size * heap->gc_threshold_factor won't exceed + * 6^32(GC_HEAP_SIZE_MAX * GC_DEFAULT_THRESHOLD_FACTOR), so casting result + * to uint32_t is safe + */ + heap->gc_threshold = (uint32_t)result; +} + +#define gct_vm_mutex_init os_mutex_init +#define gct_vm_mutex_destroy os_mutex_destroy +#define gct_vm_mutex_lock os_mutex_lock +#define gct_vm_mutex_unlock os_mutex_unlock +#define gct_vm_gc_prepare wasm_runtime_gc_prepare +#define gct_vm_gc_finished wasm_runtime_gc_finalize +#define gct_vm_begin_rootset_enumeration wasm_runtime_traverse_gc_rootset +#define gct_vm_get_wasm_object_ref_list wasm_runtime_get_wasm_object_ref_list +#define gct_vm_get_extra_info_flag wasm_runtime_get_wasm_object_extra_info_flag +#define gct_vm_set_extra_info_flag wasm_runtime_set_wasm_object_extra_info_flag + +#endif /* end of WAMS_ENABLE_GC != 0 */ + +/** + * MISC internal used APIs + */ + +bool +gci_add_fc(gc_heap_t *heap, hmu_t *hmu, gc_size_t size); + +int +gci_is_heap_valid(gc_heap_t *heap); + +/** + * Verify heap integrity + */ +void +gci_verify_heap(gc_heap_t *heap); + +/** + * Dump heap nodes + */ +void +gci_dump(gc_heap_t *heap); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _EMS_GC_INTERNAL_H */ diff --git a/wamr/include/gnuc.h b/wamr/include/gnuc.h new file mode 100644 index 0000000..70000ae --- /dev/null +++ b/wamr/include/gnuc.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2023 Amazon.com, Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#if !defined(__GNUC_PREREQ) && (defined(__GNUC__) || defined(__GNUG__)) \ + && !defined(__clang__) && defined(__GNUC_MINOR__) +/* Depending on the platform the macro is defined in sys/features.h or + features.h Given the macro is simple, we re-implement it here instead of + dealing with two different paths. + */ +#define __GNUC_PREREQ(maj, min) \ + ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +#endif diff --git a/wamr/include/lib_export.h b/wamr/include/lib_export.h new file mode 100644 index 0000000..0ca668f --- /dev/null +++ b/wamr/include/lib_export.h @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/** + * @file lib_export.h + * + */ + +#ifndef _LIB_EXPORT_H_ +#define _LIB_EXPORT_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct NativeSymbol { + const char *symbol; + void *func_ptr; + const char *signature; + /* attachment which can be retrieved in native API by + calling wasm_runtime_get_function_attachment(exec_env) */ + void *attachment; +} NativeSymbol; + +/* clang-format off */ +#define EXPORT_WASM_API(symbol) \ + { #symbol, (void *)symbol, NULL, NULL } +#define EXPORT_WASM_API2(symbol) \ + { #symbol, (void *)symbol##_wrapper, NULL, NULL } + +#define EXPORT_WASM_API_WITH_SIG(symbol, signature) \ + { #symbol, (void *)symbol, signature, NULL } +#define EXPORT_WASM_API_WITH_SIG2(symbol, signature) \ + { #symbol, (void *)symbol##_wrapper, signature, NULL } + +#define EXPORT_WASM_API_WITH_ATT(symbol, signature, attachment) \ + { #symbol, (void *)symbol, signature, attachment } +#define EXPORT_WASM_API_WITH_ATT2(symbol, signature, attachment) \ + { #symbol, (void *)symbol##_wrapper, signature, attachment } +/* clang-format on */ + +/** + * Get the exported APIs of base lib + * + * @param p_base_lib_apis return the exported API array of base lib + * + * @return the number of the exported API + */ +uint32_t +get_base_lib_export_apis(NativeSymbol **p_base_lib_apis); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _LIB_EXPORT_H_ */ diff --git a/wamr/include/mem_alloc.h b/wamr/include/mem_alloc.h new file mode 100644 index 0000000..80a842b --- /dev/null +++ b/wamr/include/mem_alloc.h @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __MEM_ALLOC_H +#define __MEM_ALLOC_H + +#include "bh_platform.h" // bh_platform.h + + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void *mem_allocator_t; + +#ifndef GC_FINALIZER_T_DEFINED +#define GC_FINALIZER_T_DEFINED +typedef void (*gc_finalizer_t)(void *obj, void *data); +#endif + +mem_allocator_t +mem_allocator_create(void *mem, uint32_t size); + +mem_allocator_t +mem_allocator_create_with_struct_and_pool(void *struct_buf, + uint32_t struct_buf_size, + void *pool_buf, + uint32_t pool_buf_size); + +int +mem_allocator_destroy(mem_allocator_t allocator); + +uint32 +mem_allocator_get_heap_struct_size(void); + +void * +mem_allocator_malloc(mem_allocator_t allocator, uint32_t size); + +void * +mem_allocator_realloc(mem_allocator_t allocator, void *ptr, uint32_t size); + +void +mem_allocator_free(mem_allocator_t allocator, void *ptr); + +int +mem_allocator_migrate(mem_allocator_t allocator, char *pool_buf_new, + uint32 pool_buf_size); + +bool +mem_allocator_is_heap_corrupted(mem_allocator_t allocator); + +#if WASM_ENABLE_GC != 0 +void * +mem_allocator_malloc_with_gc(mem_allocator_t allocator, uint32_t size); + +#if WASM_GC_MANUALLY != 0 +void +mem_allocator_free_with_gc(mem_allocator_t allocator, void *ptr); +#endif + +#if WASM_ENABLE_THREAD_MGR == 0 +void +mem_allocator_enable_gc_reclaim(mem_allocator_t allocator, void *exec_env); +#else +void +mem_allocator_enable_gc_reclaim(mem_allocator_t allocator, void *cluster); +#endif + +int +mem_allocator_add_root(mem_allocator_t allocator, WASMObjectRef obj); + +bool +mem_allocator_set_gc_finalizer(mem_allocator_t allocator, void *obj, + gc_finalizer_t cb, void *data); + +void +mem_allocator_unset_gc_finalizer(mem_allocator_t allocator, void *obj); + +#if WASM_ENABLE_GC_PERF_PROFILING != 0 +void +mem_allocator_dump_perf_profiling(mem_allocator_t allocator); +#endif +#endif /* end of WASM_ENABLE_GC != 0 */ + +bool +mem_allocator_get_alloc_info(mem_allocator_t allocator, void *mem_alloc_info); + +#ifdef __cplusplus +} +#endif + +#endif /* #ifndef __MEM_ALLOC_H */ diff --git a/wamr/include/platform_api_extension.h b/wamr/include/platform_api_extension.h new file mode 100644 index 0000000..7f27af1 --- /dev/null +++ b/wamr/include/platform_api_extension.h @@ -0,0 +1,1708 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef PLATFORM_API_EXTENSION_H +#define PLATFORM_API_EXTENSION_H + +#include "platform_common.h" // platform_common.h + +#if !defined(__wasm32__) && !defined(__EMSCRIPTEN__) +#include "platform_wasi_types.h" // platform_wasi_types.h +#else +#include +#define __WASI_ESUCCESS (0) +#define __WASI_EINVAL (28) +#define __WASI_CLOCK_REALTIME (0) +#define __WASI_CLOCK_MONOTONIC (1) +#define __WASI_CLOCK_PROCESS_CPUTIME_ID (2) +#define __WASI_CLOCK_THREAD_CPUTIME_ID (3) +#endif + +/** + * The related data structures should be defined + * in platform_internal.h + **/ +#include "platform_internal.h" // platform_internal.h + +#ifdef __cplusplus +extern "C" { +#endif + +/*************************************************** + * * + * Extension interface * + * * + ***************************************************/ + +/**************************************************** + * Section 1 * + * Multi thread support * + ****************************************************/ + +/** + * NOTES: + * 1. If you are building VM core only, it must be implemented to + * enable multi-thread support, otherwise no need to implement it + * 2. To build the app-mgr and app-framework, you must implement it + */ + +/** + * Creates a thread + * + * @param p_tid [OUTPUT] the pointer of tid + * @param start main routine of the thread + * @param arg argument passed to main routine + * @param stack_size bytes of stack size + * + * @return 0 if success. + */ +int +os_thread_create(korp_tid *p_tid, thread_start_routine_t start, void *arg, + unsigned int stack_size); + +/** + * Creates a thread with priority + * + * @param p_tid [OUTPUT] the pointer of tid + * @param start main routine of the thread + * @param arg argument passed to main routine + * @param stack_size bytes of stack size + * @param prio the priority + * + * @return 0 if success. + */ +int +os_thread_create_with_prio(korp_tid *p_tid, thread_start_routine_t start, + void *arg, unsigned int stack_size, int prio); + +/** + * Waits for the thread specified by thread to terminate + * + * @param thread the thread to wait + * @param retval if not NULL, output the exit status of the terminated thread + * + * @return return 0 if success + */ +int +os_thread_join(korp_tid thread, void **retval); + +/** + * Detach the thread specified by thread + * + * @param thread the thread to detach + * + * @return return 0 if success + */ +int os_thread_detach(korp_tid); + +/** + * Exit current thread + * + * @param retval the return value of the current thread + */ +void +os_thread_exit(void *retval); + +/* Try to define os_atomic_thread_fence if it isn't defined in + platform's platform_internal.h */ +#ifndef os_atomic_thread_fence + +#if !defined(__GNUC_PREREQ) && (defined(__GNUC__) || defined(__GNUG__)) \ + && !defined(__clang__) && defined(__GNUC_MINOR__) +#define __GNUC_PREREQ(maj, min) \ + ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +#endif + +/* Clang's __GNUC_PREREQ macro has a different meaning than GCC one, + so we have to handle this case specially(except the CCAC compiler + provided by MetaWare, which doesn't support atomic operations) */ +#if defined(__clang__) && !defined(__CCAC__) +/* Clang provides stdatomic.h since 3.6.0 + See https://releases.llvm.org/3.6.0/tools/clang/docs/ReleaseNotes.html */ +#if __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 6) +#define BH_HAS_STD_ATOMIC +#endif +#elif defined(__GNUC_PREREQ) +/* Even though older versions of GCC support C11, atomics were + not implemented until 4.9. See + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58016 */ +#if __GNUC_PREREQ(4, 9) +#define BH_HAS_STD_ATOMIC +#elif __GNUC_PREREQ(4, 7) +#define os_memory_order_acquire __ATOMIC_ACQUIRE +#define os_memory_order_release __ATOMIC_RELEASE +#define os_memory_order_seq_cst __ATOMIC_SEQ_CST +#define os_atomic_thread_fence __atomic_thread_fence +#endif /* end of __GNUC_PREREQ(4, 9) */ +#endif /* end of defined(__GNUC_PREREQ) */ + +#if defined(BH_HAS_STD_ATOMIC) && !defined(__cplusplus) +#include +#define os_memory_order_acquire memory_order_acquire +#define os_memory_order_release memory_order_release +#define os_memory_order_seq_cst memory_order_seq_cst +#define os_atomic_thread_fence atomic_thread_fence +#define os_atomic_cmpxchg atomic_compare_exchange_strong +#endif + +#endif /* end of os_atomic_thread_fence */ + +/** + * Initialize current thread environment if current thread + * is created by developer but not runtime + * + * @return 0 if success, -1 otherwise + */ +int +os_thread_env_init(void); + +/** + * Destroy current thread environment + */ +void +os_thread_env_destroy(void); + +/** + * Whether the thread environment is initialized + */ +bool +os_thread_env_inited(void); + +/** + * Suspend execution of the calling thread for (at least) + * usec microseconds + * + * @return 0 if success, -1 otherwise + */ +int +os_usleep(uint32 usec); + +/** + * Creates a recursive mutex + * + * @param mutex [OUTPUT] pointer to mutex initialized. + * + * @return 0 if success + */ +int +os_recursive_mutex_init(korp_mutex *mutex); + +/** + * This function creates a condition variable + * + * @param cond [OUTPUT] pointer to condition variable + * + * @return 0 if success + */ +int +os_cond_init(korp_cond *cond); + +/** + * This function destroys condition variable + * + * @param cond pointer to condition variable + * + * @return 0 if success + */ +int +os_cond_destroy(korp_cond *cond); + +/** + * Wait a condition variable. + * + * @param cond pointer to condition variable + * @param mutex pointer to mutex to protect the condition variable + * + * @return 0 if success + */ +int +os_cond_wait(korp_cond *cond, korp_mutex *mutex); + +/** + * Wait a condition variable or return if time specified passes. + * + * @param cond pointer to condition variable + * @param mutex pointer to mutex to protect the condition variable + * @param useconds microseconds to wait + * + * @return 0 if success + */ +int +os_cond_reltimedwait(korp_cond *cond, korp_mutex *mutex, uint64 useconds); + +/** + * Signals the condition variable + * + * @param cond condition variable + * + * @return 0 if success + */ +int +os_cond_signal(korp_cond *cond); + +/** + * Broadcast the condition variable + * + * @param cond condition variable + * + * @return 0 if success + */ +int +os_cond_broadcast(korp_cond *cond); + +/** + * Initialize readwrite lock object + * + * @param cond [OUTPUT] pointer to a lock object variable + * + * @return 0 if success + */ +int +os_rwlock_init(korp_rwlock *lock); + +/** + * Acquire the read lock + * + * @param lock lock variable + * + * @return 0 if success + */ +int +os_rwlock_rdlock(korp_rwlock *lock); + +/** + * Acquire the write lock + * + * @param lock lock variable + * + * @return 0 if success + */ +int +os_rwlock_wrlock(korp_rwlock *lock); + +/** + * Unlocks the lock object + * + * @param lock lock variable + * + * @return 0 if success + */ +int +os_rwlock_unlock(korp_rwlock *lock); + +/** + * Destroy a lock object + * + * @param lock lock variable + * + * @return 0 if success + */ +int +os_rwlock_destroy(korp_rwlock *lock); + +/** + * Creates a new POSIX-like semaphore or opens an existing + * semaphore. The semaphore is identified by name. For details of + * the construction of name, please refer to + * https://man7.org/linux/man-pages/man3/sem_open.3.html. + * + * @param name semaphore name + * @param oflasg specifies flags that control the operation of the call + * @param mode permission flags + * @param val initial value of the named semaphore. + * + * @return korp_sem * if success, NULL otherwise + */ +korp_sem * +os_sem_open(const char *name, int oflags, int mode, int val); + +/** + * Closes the named semaphore referred to by sem, + * allowing any resources that the system has allocated to the + * calling process for this semaphore to be freed. + * + * @param sem + * + * @return 0 if success + */ +int +os_sem_close(korp_sem *sem); + +/** + * Decrements (locks) the semaphore pointed to by sem. + * If the semaphore's value is greater than zero, then the decrement + * proceeds, and the function returns, immediately. If the + * semaphore currently has the value zero, then the call blocks + * until either it becomes possible to perform the decrement (i.e., + * the semaphore value rises above zero), or a signal handler + * interrupts the call. + * + * @return 0 if success + */ +int +os_sem_wait(korp_sem *sem); + +/** + * Is the same as sem_wait(), except that if the + * decrement cannot be immediately performed, then call returns an + * error (errno set to EAGAIN) instead of blocking. + * + * @return 0 if success + */ +int +os_sem_trywait(korp_sem *sem); + +/** + * Increments (unlocks) the semaphore pointed to by sem. + * If the semaphore's value consequently becomes greater than zero, + * then another process or thread blocked in a sem_wait(3) call will + * be woken up and proceed to lock the semaphore. + * + * @return 0 if success + */ +int +os_sem_post(korp_sem *sem); + +/** + * Places the current value of the semaphore pointed + * to sem into the integer pointed to by sval. + * + * @return 0 if success + */ +int +os_sem_getvalue(korp_sem *sem, int *sval); + +/** + * Remove the named semaphore referred to by name. + * The semaphore name is removed immediately. The semaphore is + * destroyed once all other processes that have the semaphore open + * close it. + * + * @param name semaphore name + * + * @return 0 if success + */ +int +os_sem_unlink(const char *name); + +/** + * Initialize process-global state for os_wakeup_blocking_op. + */ +int +os_blocking_op_init(void); + +/** + * Start accepting os_wakeup_blocking_op requests for the calling thread. + */ +void +os_begin_blocking_op(void); + +/** + * Stop accepting os_wakeup_blocking_op requests for the calling thread. + */ +void +os_end_blocking_op(void); + +/** + * Wake up the specified thread. + * + * For example, on posix-like platforms, this can be implemented by + * sending a signal (w/o SA_RESTART) which interrupts a blocking + * system call. + */ +int +os_wakeup_blocking_op(korp_tid tid); + +/**************************************************** + * Section 2 * + * Socket support * + ****************************************************/ + +/** + * NOTES: + * Socket APIs are required by source debugging feature. + * If you don't need source debugging feature, then no + * need to implement these APIs + */ + +typedef union { + uint32 ipv4; + uint16 ipv6[8]; + uint8 data[1]; +} bh_ip_addr_buffer_t; + +typedef struct { + bh_ip_addr_buffer_t addr_buffer; + uint16 port; + bool is_ipv4; +} bh_sockaddr_t; + +/** + * Create a socket + * + * @param sock [OUTPUT] the pointer of socket + * @param is_ipv4 true for IPv4, false for IPv6 + * @param is_tcp true for tcp, false for udp + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_create(bh_socket_t *sock, bool is_ipv4, bool is_tcp); + +/** + * Assign the address and port to the socket + * + * @param socket the socket to bind + * @param addr the ip address, only IPv4 supported currently + * @param port [INPUT/OUTPUT] the port number, if the value is 0, + * it will use a port assigned by OS. On return it will + * contain the actual bound port number + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_bind(bh_socket_t socket, const char *addr, int *port); + +/** + * Set timeout for the given socket + * + * @param socket the socket to set timeout + * @param timeout_us timeout in microseconds + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_settimeout(bh_socket_t socket, uint64 timeout_us); + +/** + * Make the socket as a passive socket to accept incoming connection requests + * + * @param socket the socket to listen + * @param max_client maximum clients + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_listen(bh_socket_t socket, int max_client); + +/** + * Accept an incoming connection + * + * @param server_sock the socket to accept new connections + * @param sock [OUTPUT] the connected socket + * @param addr [OUTPUT] the address of the peer socket. If addr is NULL, + * nothing is filled in, and addrlen will not be used + * @param addrlen [INPUT/OUTPUT] the size (in bytes) of the structure + * pointed to by addr, on return it will contain the actual + * size of the peer address + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_accept(bh_socket_t server_sock, bh_socket_t *sock, void *addr, + unsigned int *addrlen); + +/** + * initiate a connection on a socket + * + * @param socket the socket to connect with + * @param addr the ip address, only IPv4 supported currently + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_connect(bh_socket_t socket, const char *addr, int port); + +/** + * Blocking receive message from a socket. + * + * @param socket the socket to receive message from + * @param buf the buffer to store the data + * @param len length of the buffer, this API does not guarantee that + * [len] bytes are received + * + * @return number of bytes received if success, -1 otherwise + */ +int +os_socket_recv(bh_socket_t socket, void *buf, unsigned int len); + +/** + * Blocking receive message from a socket. + * + * @param socket the socket to send message + * @param buf the buffer to store the data + * @param len length of the buffer, this API does not guarantee that + * [len] bytes are received + * @param flags control the operation + * @param src_addr source address + * + * @return number of bytes sent if success, -1 otherwise + */ +int +os_socket_recv_from(bh_socket_t socket, void *buf, unsigned int len, int flags, + bh_sockaddr_t *src_addr); + +/** + * Blocking send message on a socket + * + * @param socket the socket to send message + * @param buf the buffer of data to be sent + * @param len length of the buffer + * + * @return number of bytes sent if success, -1 otherwise + */ +int +os_socket_send(bh_socket_t socket, const void *buf, unsigned int len); + +/** + * Blocking send message on a socket to the target address + * + * @param socket the socket to send message + * @param buf the buffer of data to be sent + * @param len length of the buffer + * @param flags control the operation + * @param dest_addr target address + * + * @return number of bytes sent if success, -1 otherwise + */ +int +os_socket_send_to(bh_socket_t socket, const void *buf, unsigned int len, + int flags, const bh_sockaddr_t *dest_addr); + +/** + * Close a socket + * + * @param socket the socket to be closed + * + * @return always return 0 + */ +int +os_socket_close(bh_socket_t socket); + +/** + * Shutdown a socket + * + * @param socket the socket to be shutdown + * + * @return returns corresponding error code + */ +__wasi_errno_t +os_socket_shutdown(bh_socket_t socket); + +/** + * converts cp into a number in host byte order suitable for use as + * an Internet network address + * + * @param is_ipv4 a flag that indicates whether the string is an IPv4 or + * IPv6 address + * + * @param cp a string in IPv4 numbers-and-dots notation or IPv6 + * numbers-and-colons notation + * + * @param out an output buffer to store binary address + * + * @return On success, the function returns 0. + * If the input is invalid, -1 is returned + */ +int +os_socket_inet_network(bool is_ipv4, const char *cp, bh_ip_addr_buffer_t *out); + +typedef struct { + bh_sockaddr_t sockaddr; + uint8_t is_tcp; +} bh_addr_info_t; + +/** + * Resolve a host a hostname and a service to one or more IP addresses + * + * @param host a host to resolve + * + * @param service a service to find a port for + * + * @param hint_is_tcp an optional flag that determines a preferred socket type + (TCP or UDP). + * + * @param hint_is_ipv4 an optional flag that determines a preferred address + family (IPv4 or IPv6) + * + * @param addr_info a buffer for resolved addresses + * + * @param addr_info_size a size of the buffer for resolved addresses + + * @param max_info_size a maximum number of addresses available (can be bigger + or smaller than buffer size) + + * @return On success, the function returns 0; otherwise, it returns -1 + */ +int +os_socket_addr_resolve(const char *host, const char *service, + uint8_t *hint_is_tcp, uint8_t *hint_is_ipv4, + bh_addr_info_t *addr_info, size_t addr_info_size, + size_t *max_info_size); + +/** + * Returns an binary address and a port of the local socket + * + * @param socket the local socket + * + * @param sockaddr a buffer for storing the address + * + * @return On success, returns 0; otherwise, it returns -1. + */ +int +os_socket_addr_local(bh_socket_t socket, bh_sockaddr_t *sockaddr); + +/** + * Returns an binary address and a port of the remote socket + * + * @param socket the remote socket + * + * @param sockaddr a buffer for storing the address + * + * @return On success, returns 0; otherwise, it returns -1. + */ +int +os_socket_addr_remote(bh_socket_t socket, bh_sockaddr_t *sockaddr); + +/** + * Set the maximum send buffer size. + * + * @param socket the socket to set + * @param bufsiz requested kernel buffer size + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_send_buf_size(bh_socket_t socket, size_t bufsiz); + +/** + * Get the maximum send buffer size. + * + * @param socket the socket to set + * @param bufsiz the returned kernel buffer size + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_send_buf_size(bh_socket_t socket, size_t *bufsiz); + +/** + * Set the maximum receive buffer size. + * + * @param socket the socket to set + * @param bufsiz requested kernel buffer size + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_recv_buf_size(bh_socket_t socket, size_t bufsiz); + +/** + * Get the maximum receive buffer size. + * + * @param socket the socket to set + * @param bufsiz the returned kernel buffer size + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_recv_buf_size(bh_socket_t socket, size_t *bufsiz); + +/** + * Enable sending of keep-alive messages on connection-oriented sockets + * + * @param socket the socket to set the flag + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_keep_alive(bh_socket_t socket, bool is_enabled); + +/** + * Get if sending of keep-alive messages on connection-oriented sockets is + * enabled + * + * @param socket the socket to check + * @param is_enabled 1 if enabled or 0 if disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_keep_alive(bh_socket_t socket, bool *is_enabled); + +/** + * Set the send timeout until reporting an error + * + * @param socket the socket to set + * @param time_us microseconds until timeout + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_send_timeout(bh_socket_t socket, uint64 timeout_us); + +/** + * Get the send timeout until reporting an error + * + * @param socket the socket to set + * @param time_us the returned microseconds until timeout + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_send_timeout(bh_socket_t socket, uint64 *timeout_us); + +/** + * Set the recv timeout until reporting an error + * + * @param socket the socket to set + * @param time_us microseconds until timeout + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_recv_timeout(bh_socket_t socket, uint64 timeout_us); + +/** + * Get the recv timeout until reporting an error + * + * @param socket the socket to set + * @param time_us the returned microseconds until timeout + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_recv_timeout(bh_socket_t socket, uint64 *timeout_us); + +/** + * Enable reuse of local addresses + * + * @param socket the socket to set + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_reuse_addr(bh_socket_t socket, bool is_enabled); + +/** + * Get whether reuse of local addresses is enabled + * + * @param socket the socket to set + * @param is_enabled 1 for enabled or 0 for disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_reuse_addr(bh_socket_t socket, bool *is_enabled); + +/** + * Enable reuse of local ports + * + * @param socket the socket to set + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_reuse_port(bh_socket_t socket, bool is_enabled); + +/** + * Get whether reuse of local ports is enabled + * + * @param socket the socket to set + * @param is_enabled 1 for enabled or 0 for disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_reuse_port(bh_socket_t socket, bool *is_enabled); + +/** + * Set the linger options for the given socket + * + * @param socket the socket to set + * @param is_enabled whether linger is enabled + * @param linger_s linger time (seconds) + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_linger(bh_socket_t socket, bool is_enabled, int linger_s); + +/** + * Get the linger options for the given socket + * + * @param socket the socket to get + * @param is_enabled whether linger is enabled + * @param linger_s linger time (seconds) + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_linger(bh_socket_t socket, bool *is_enabled, int *linger_s); + +/** + * Set no delay TCP + * If set, disable the Nagle algorithm. + * This means that segments are always sent as soon as possible, + * even if there is only a small amount of data + * + * @param socket the socket to set the flag + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_tcp_no_delay(bh_socket_t socket, bool is_enabled); + +/** + * Get no delay TCP + * If set, disable the Nagle algorithm. + * This means that segments are always sent as soon as possible, + * even if there is only a small amount of data + * + * @param socket the socket to check + * @param is_enabled 1 if enabled or 0 if disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_tcp_no_delay(bh_socket_t socket, bool *is_enabled); + +/** + * Enable/Disable tcp quickack mode + * In quickack mode, acks are sent immediately, rather than delayed if needed in + * accordance to normal TCP operation + * + * @param socket the socket to set the flag + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_tcp_quick_ack(bh_socket_t socket, bool is_enabled); + +/** + * Enable/Disable tcp quickack mode + * In quickack mode, acks are sent immediately, rather than delayed if needed in + * accordance to normal TCP operation + * + * @param socket the socket to check + * @param is_enabled 1 if enabled or 0 if disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_tcp_quick_ack(bh_socket_t socket, bool *is_enabled); + +/** + * Set the time the connection needs to remain idle before sending keepalive + * probes + * + * @param socket the socket to set + * @param time_s seconds until keepalive probes are sent + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_tcp_keep_idle(bh_socket_t socket, uint32_t time_s); + +/** + * Gets the time the connection needs to remain idle before sending keepalive + * probes + * + * @param socket the socket to check + * @param time_s seconds until keepalive probes are sent + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_tcp_keep_idle(bh_socket_t socket, uint32_t *time_s); + +/** + * Set the time between individual keepalive probes + * + * @param socket the socket to set + * @param time_us seconds between individual keepalive probes + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_tcp_keep_intvl(bh_socket_t socket, uint32_t time_s); + +/** + * Get the time between individual keepalive probes + * + * @param socket the socket to get + * @param time_s seconds between individual keepalive probes + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_tcp_keep_intvl(bh_socket_t socket, uint32_t *time_s); + +/** + * Set use of TCP Fast Open + * + * @param socket the socket to set + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_tcp_fastopen_connect(bh_socket_t socket, bool is_enabled); + +/** + * Get whether use of TCP Fast Open is enabled + * + * @param socket the socket to get + * @param is_enabled 1 to enabled or 0 to disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_tcp_fastopen_connect(bh_socket_t socket, bool *is_enabled); + +/** + * Set enable or disable IPv4 or IPv6 multicast loopback. + * + * @param socket the socket to set + * @param ipv6 true to set ipv6 loopback or false for ipv4 + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_ip_multicast_loop(bh_socket_t socket, bool ipv6, bool is_enabled); + +/** + * Get enable or disable IPv4 or IPv6 multicast loopback. + * + * @param socket the socket to check + * @param ipv6 true to set ipv6 loopback or false for ipv4 + * @param is_enabled 1 for enabled or 0 for disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_ip_multicast_loop(bh_socket_t socket, bool ipv6, + bool *is_enabled); + +/** + * Add membership to a group + * + * @param socket the socket to add membership to + * @param imr_multiaddr the group multicast address (IPv4 or IPv6) + * @param imr_interface the interface to join on + * @param is_ipv6 whether the imr_multiaddr is IPv4 or IPv6 (true for IPv6) + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_ip_add_membership(bh_socket_t socket, + bh_ip_addr_buffer_t *imr_multiaddr, + uint32_t imr_interface, bool is_ipv6); + +/** + * Drop membership of a group + * + * @param socket the socket to drop membership to + * @param imr_multiaddr the group multicast address (IPv4 or IPv6) + * @param imr_interface the interface to join on + * @param is_ipv6 whether the imr_multiaddr is IPv4 or IPv6 (true for IPv6) + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_ip_drop_membership(bh_socket_t socket, + bh_ip_addr_buffer_t *imr_multiaddr, + uint32_t imr_interface, bool is_ipv6); + +/** + * Set the current time-to-live field that is + * used in every packet sent from this socket. + * @param socket the socket to set the flag + * @param ttl_s time to live (seconds) + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_ip_ttl(bh_socket_t socket, uint8_t ttl_s); + +/** + * Retrieve the current time-to-live field that is + * used in every packet sent from this socket. + * @param socket the socket to set the flag + * @param ttl_s time to live (seconds) + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_ip_ttl(bh_socket_t socket, uint8_t *ttl_s); + +/** + * Set the time-to-live value of outgoing multicast + * packets for this socket + * @param socket the socket to set the flag + * @param ttl_s time to live (seconds) + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_ip_multicast_ttl(bh_socket_t socket, uint8_t ttl_s); + +/** + * Read the time-to-live value of outgoing multicast + * packets for this socket + * @param socket the socket to set the flag + * @param ttl_s time to live (seconds) + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_ip_multicast_ttl(bh_socket_t socket, uint8_t *ttl_s); + +/** + * Restrict to sending and receiving IPv6 packets only + * + * @param socket the socket to set + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_ipv6_only(bh_socket_t socket, bool is_enabled); + +/** + * Get whether only sending and receiving IPv6 packets + * + * @param socket the socket to check + * @param is_enabled 1 for enabled or 0 for disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_ipv6_only(bh_socket_t socket, bool *is_enabled); + +/** + * Set whether broadcast is enabled + * When enabled, datagram sockets are allowed + * to send packets to a broadcast address. + * + * @param socket the socket to set the flag + * @param is_enabled 1 to enable or 0 to disable + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_set_broadcast(bh_socket_t socket, bool is_enabled); + +/** + * Get whether broadcast is enabled + * When enabled, datagram sockets are allowed + * to send packets to a broadcast address. + * + * @param socket the socket to check + * @param is_enabled 1 if enabled or 0 if disabled + * + * @return 0 if success, -1 otherwise + */ +int +os_socket_get_broadcast(bh_socket_t socket, bool *is_enabled); + +/** + * Dump memory information of the current process + * It may have variant implementations in different platforms + * + * @param out the output buffer. It is for sure the return content + * is a c-string which ends up with '\0' + * @param size the size of the output buffer + * + * @return 0 if success, -1 otherwise + */ +int +os_dumps_proc_mem_info(char *out, unsigned int size); + +/**************************************************** + * Section 3 * + * Filesystem support * + ****************************************************/ + +/** + * NOTES: + * Filesystem APIs are required for WASI libc support. If you don't need to + * support WASI libc, there is no need to implement these APIs. With a + * few exceptions, each filesystem function has been named after the equivalent + * POSIX filesystem function with an os_ prefix. + * + * Filesystem types + * + * os_raw_file_handle: the underlying OS file handle type e.g. int on POSIX + * systems and HANDLE on Windows. This type exists to allow embedders to provide + * custom file handles for stdout/stdin/stderr. + * + * os_file_handle: the file handle type used in the WASI libc fd + * table. Filesystem implementations can use it as a means to store any + * necessary platform-specific information which may not be directly available + * through the raw OS file handle. Similar to POSIX file descriptors, file + * handles may also refer to sockets, directories, symbolic links or character + * devices and any of the filesystem operations which make sense for these + * resource types should be supported as far as possible. + * + * os_dir_stream: a directory stream type in which filesystem implementations + * can store any necessary state to iterate over the entries in a directory. + */ + +/** + * Obtain information about an open file associated with the given handle. + * + * @param handle the handle for which to obtain file information + * @param buf a buffer in which to store the information + */ +__wasi_errno_t +os_fstat(os_file_handle handle, struct __wasi_filestat_t *buf); + +/** + * Obtain information about an open file or directory. + * @param handle the directory handle from which to resolve the file/directory + * path + * @param path the relative path of the file or directory for which to obtain + * information + * @param buf a buffer in which to store the information + * @param follow_symlink whether to follow symlinks when resolving the path + */ +__wasi_errno_t +os_fstatat(os_file_handle handle, const char *path, + struct __wasi_filestat_t *buf, __wasi_lookupflags_t lookup_flags); + +/** + * Obtain the file status flags for the provided handle. This is similar to the + * POSIX function fcntl called with the F_GETFL command. + * + * @param handle the handle for which to obtain the file status flags + * @param flags a pointer in which to store the output + */ +__wasi_errno_t +os_file_get_fdflags(os_file_handle handle, __wasi_fdflags_t *flags); + +/** + * Set the file status flags for the provided handle. This is similar to the + * POSIX function fcntl called with the F_SETFL command. + * + * @param handle the handle for which to set the file status flags + * @param flags the flags to set + */ +__wasi_errno_t +os_file_set_fdflags(os_file_handle handle, __wasi_fdflags_t flags); + +/** + * Synchronize the data of a file to disk. + * + * @param handle + */ +__wasi_errno_t +os_fdatasync(os_file_handle handle); + +/** + * Synchronize the data and metadata of a file to disk. + * + * @param handle + */ +__wasi_errno_t +os_fsync(os_file_handle handle); + +/** + * Open a preopen directory. The path provided must refer to a directory and the + * returned handle will allow only readonly operations. + * + * @param path the path of the preopen directory to open + * @param out a pointer in which to store the newly opened handle + */ +__wasi_errno_t +os_open_preopendir(const char *path, os_file_handle *out); + +typedef uint8 wasi_libc_file_access_mode; +#define WASI_LIBC_ACCESS_MODE_READ_ONLY 0 +#define WASI_LIBC_ACCESS_MODE_WRITE_ONLY 1 +#define WASI_LIBC_ACCESS_MODE_READ_WRITE 2 + +/** + * Open a file or directory at the given path. + * + * @param handle a handle to the directory in which to open the new file or + * directory + * @param path the relative path of the file or directory to open + * @param oflags the flags to determine how the file or directory is opened + * @param fd_flags the flags to set on the returned handle + * @param lookup_flags whether to follow symlinks when resolving the path + * @param access_mode whether the file is opened as read only, write only or + * both + * @param out a pointer in which to store the newly opened handle + */ +__wasi_errno_t +os_openat(os_file_handle handle, const char *path, __wasi_oflags_t oflags, + __wasi_fdflags_t fd_flags, __wasi_lookupflags_t lookup_flags, + wasi_libc_file_access_mode access_mode, os_file_handle *out); + +/** + * Obtain the file access mode for the provided handle. This is similar to the + * POSIX function fcntl called with the F_GETFL command combined with the + * O_ACCMODE mask. + * + * @param handle the handle for which to obtain the access mode + * @param access_mode a pointer in which to store the access mode + */ +__wasi_errno_t +os_file_get_access_mode(os_file_handle handle, + wasi_libc_file_access_mode *access_mode); + +/** + * Close the provided handle. If is_stdio is true, the raw file handle + * associated with the given file handle will not be closed. + * + * @param handle the handle to close + * @param is_stdio whether the provided handle refers to a stdio device + */ +__wasi_errno_t +os_close(os_file_handle handle, bool is_stdio); + +/** + * Read data from the provided handle at the given offset into multiple buffers. + * + * @param handle the handle to read from + * @param iov the buffers to read into + * @param iovcnt the number of buffers to read into + * @param offset the offset to read from + * @param nread a pointer in which to store the number of bytes read + */ +__wasi_errno_t +os_preadv(os_file_handle handle, const struct __wasi_iovec_t *iov, int iovcnt, + __wasi_filesize_t offset, size_t *nread); + +/** + * Write data from multiple buffers at the given offset to the provided handle. + * + * @param handle the handle to write to + * @param iov the buffers to write from + * @param iovcnt the number of buffers to write from + * @param offset the offset to write from + * @param nwritten a pointer in which to store the number of bytes written + */ +__wasi_errno_t +os_pwritev(os_file_handle handle, const struct __wasi_ciovec_t *iov, int iovcnt, + __wasi_filesize_t offset, size_t *nwritten); + +/** + * Read data from the provided handle into multiple buffers. + * + * @param handle the handle to read from + * @param iov the buffers to read into + * @param iovcnt the number of buffers to read into + * @param nread a pointer in which to store the number of bytes read + */ +__wasi_errno_t +os_readv(os_file_handle handle, const struct __wasi_iovec_t *iov, int iovcnt, + size_t *nread); + +/** + * Write data from multiple buffers to the provided handle. + * + * @param handle the handle to write to + * @param iov the buffers to write from + * @param iovcnt the number of buffers to write from + * @param nwritten a pointer in which to store the number of bytes written + */ +__wasi_errno_t +os_writev(os_file_handle handle, const struct __wasi_ciovec_t *iov, int iovcnt, + size_t *nwritten); + +/** + * Allocate storage space for the file associated with the provided handle. This + * is similar to the POSIX function posix_fallocate. + * + * @param handle the handle to allocate space for + * @param offset the offset to allocate space at + * @param length the amount of space to allocate + */ +__wasi_errno_t +os_fallocate(os_file_handle handle, __wasi_filesize_t offset, + __wasi_filesize_t length); + +/** + * Adjust the size of an open file. + * + * @param handle the associated file handle for which to adjust the size + * @param size the new size of the file + */ +__wasi_errno_t +os_ftruncate(os_file_handle handle, __wasi_filesize_t size); + +/** + * Set file access and modification times on an open file or directory. + * + * @param handle the associated file handle for which to adjust the + * access/modification times + * @param access_time the timestamp for the new access time + * @param modification_time the timestamp for the new modification time + * @param fstflags a bitmask to indicate which timestamps to adjust + */ +__wasi_errno_t +os_futimens(os_file_handle handle, __wasi_timestamp_t access_time, + __wasi_timestamp_t modification_time, __wasi_fstflags_t fstflags); + +/** + * Set file access and modification times on an open file or directory. + * + * @param handle the directory handle from which to resolve the path + * @param path the relative path of the file or directory for which to adjust + * the access/modification times + * @param access_time the timestamp for the new access time + * @param modification_time the timestamp for the new modification time + * @param fstflags a bitmask to indicate which timestamps to adjust + * @param lookup_flags whether to follow symlinks when resolving the path + */ +__wasi_errno_t +os_utimensat(os_file_handle handle, const char *path, + __wasi_timestamp_t access_time, + __wasi_timestamp_t modification_time, __wasi_fstflags_t fstflags, + __wasi_lookupflags_t lookup_flags); + +/** + * Read the contents of a symbolic link relative to the provided directory + * handle. + * + * @param handle the directory handle + * @param path the relative path of the symbolic link from which to read + * @param buf the buffer to read the link contents into + * @param bufsize the size of the provided buffer + * @param nread a pointer in which to store the number of bytes read into the + * buffer + */ +__wasi_errno_t +os_readlinkat(os_file_handle handle, const char *path, char *buf, + size_t bufsize, size_t *nread); + +/** + * Create a link from one path to another path. + * + * @param from_handle the directory handle from which to resolve the origin path + * @param from_path the origin path to link from + * @param to_handle the directory handle from which to resolve the destination + * path + * @param to_path the destination path at which to create the link + * @param lookup_flags whether to follow symlinks when resolving the origin path + */ +__wasi_errno_t +os_linkat(os_file_handle from_handle, const char *from_path, + os_file_handle to_handle, const char *to_path, + __wasi_lookupflags_t lookup_flags); + +/** + * Create a symbolic link from one path to another path. + * + * @param old_path the symbolic link contents + * @param handle the directory handle from which to resolve the destination path + * @param new_path the destination path at which to create the symbolic link + */ +__wasi_errno_t +os_symlinkat(const char *old_path, os_file_handle handle, const char *new_path); + +/** + * Create a directory relative to the provided directory handle. + * + * @param handle the directory handle + * @param path the relative path of the directory to create + */ +__wasi_errno_t +os_mkdirat(os_file_handle handle, const char *path); + +/** + * Rename a file or directory. + * + * @param old_handle the directory handle from which to resolve the old path + * @param old_path the source path to rename + * @param new_handle the directory handle from which to resolve the destination + * path + * @param new_path the destination path to which to rename the file or directory + */ +__wasi_errno_t +os_renameat(os_file_handle old_handle, const char *old_path, + os_file_handle new_handle, const char *new_path); + +/** + * Unlink a file or directory. + * + * @param handle the directory handle from which to resolve the path + * @param path the relative path of the file or directory to unlink + * @param is_dir whether the provided handle refers to a directory or file + */ +__wasi_errno_t +os_unlinkat(os_file_handle handle, const char *path, bool is_dir); + +/** + * Move the read/write offset of an open file. + * + * @param handle the associated file handle for which to adjust the offset + * @param offset the number of bytes to adjust the offset by + * @param whence the position whence to adjust the offset + * @param new_offset a pointer in which to store the new offset + */ +__wasi_errno_t +os_lseek(os_file_handle handle, __wasi_filedelta_t offset, + __wasi_whence_t whence, __wasi_filesize_t *new_offset); + +/** + * Provide file advisory information for the given handle. This is similar to + * the POSIX function posix_fadvise. + * + * @param handle the associated file handle for which to provide advisory + * information + * @param offset the offset within the file to which the advisory + * information applies + * @param length the length of the region for which the advisory information + * applies + * @param advice the advice to provide + */ +__wasi_errno_t +os_fadvise(os_file_handle handle, __wasi_filesize_t offset, + __wasi_filesize_t length, __wasi_advice_t advice); + +/** + * Determine if the given handle refers to a terminal device. __WASI_ESUCCESS + * will be returned if the handle is associated with a terminal device, + * otherwise an appropriate error code will be returned. + * + * @param handle + */ +__wasi_errno_t +os_isatty(os_file_handle handle); + +/** + * Converts a raw file handle to STDIN to a corresponding file handle to STDIN. + * If the provided raw file handle is invalid, the platform-default raw handle + * for STDIN will be used. + * + * @param raw_stdin a raw file handle to STDIN + * + * @return a handle to STDIN + */ +os_file_handle +os_convert_stdin_handle(os_raw_file_handle raw_stdin); + +/** + * Converts a raw file handle to STDOUT to a corresponding file handle to + * STDOUT. If the provided raw file handle is invalid, the platform-default raw + * handle for STDOUT will be used. + * + * @param raw_stdout a raw file handle to STDOUT + * + * @return a handle to STDOUT + */ +os_file_handle +os_convert_stdout_handle(os_raw_file_handle raw_stdout); + +/** + * Converts a raw file handle to STDERR to a corresponding file handle to + * STDERR. If the provided raw file handle is invalid, the platform-default raw + * handle for STDERR will be used. + * + * @param raw_stderr a raw file handle to STDERR + * + * @return a handle to STDERR + */ +os_file_handle +os_convert_stderr_handle(os_raw_file_handle raw_stderr); + +/** + * + * @param fd a file handle + * + * @return true if it is stdin + */ +bool +os_is_stdin_handle(os_file_handle fd); + +/** + * + * @param fd a file handle + * + * @return true if it is stdout + */ +bool +os_is_stdout_handle(os_file_handle fd); + +/** + * + * @param fd a file handle + * + * @return true if it is stderr + */ +bool +os_is_stderr_handle(os_file_handle fd); + +/** + * Open a directory stream for the provided directory handle. The returned + * directory stream will be positioned at the first entry in the directory. + * + * @param handle the directory handle + * @param dir_stream a pointer in which to store the new directory stream + */ +__wasi_errno_t +os_fdopendir(os_file_handle handle, os_dir_stream *dir_stream); + +/** + * Reset the position of a directory stream to the beginning of the directory. + * + * @param dir_stream the directory stream for which to reset the position + */ +__wasi_errno_t +os_rewinddir(os_dir_stream dir_stream); + +/** + * Set the position of the given directory stream. + * + * @param dir_stream the directory stream for which to set the position + * @param position the position to set + */ +__wasi_errno_t +os_seekdir(os_dir_stream dir_stream, __wasi_dircookie_t position); + +/** + * Read a directory entry from the given directory stream. The directory name + * will be NULL if the end of the directory is reached or an error is + * encountered. + * + * @param dir_stream the directory stream from which to read the entry + * @param entry a pointer in which to store the directory entry + * @param d_name a pointer in which to store the directory entry name + */ +__wasi_errno_t +os_readdir(os_dir_stream dir_stream, __wasi_dirent_t *entry, + const char **d_name); + +/** + * Close the given directory stream. The handle associated with the directory + * stream will also be closed. + * + * @param dir_stream the directory stream to close + */ +__wasi_errno_t +os_closedir(os_dir_stream dir_stream); + +/** + * Returns an invalid directory stream that is guaranteed to cause failure when + * called with any directory filesystem operation. + * + * @return the invalid directory stream + */ +os_dir_stream +os_get_invalid_dir_stream(void); + +/** + * Checks whether the given directory stream is valid. An invalid directory + * stream is guaranteed to cause failure when called with any directory + * filesystem operation. + * + * @param dir_stream a pointer to a directory stream + */ +bool +os_is_dir_stream_valid(os_dir_stream *dir_stream); + +/** + * Returns an invalid handle that is guaranteed to cause failure when + * called with any filesystem operation. + * + * @return the invalid handle + */ +os_file_handle +os_get_invalid_handle(void); + +/** + * Returns an invalid raw file handle that is guaranteed to cause failure when + * called with any filesystem operation. + * + * @return the invalid raw file handle + */ +os_raw_file_handle +os_invalid_raw_handle(void); + +/** + * Checks whether the given file handle is valid. An invalid handle is + * guaranteed to cause failure when called with any filesystem operation. + * + * @param handle a pointer to a file handle + */ +bool +os_is_handle_valid(os_file_handle *handle); + +/** + * Resolve a pathname. The generated pathname will be stored as a + * null-terminated string, with a maximum length of PATH_MAX bytes. + * + * @param path the path to resolve + * @param resolved_path the buffer to store the resolved path in + * + * @return the resolved path if success, NULL otherwise + */ +char * +os_realpath(const char *path, char *resolved_path); + +/**************************************************** + * Section 4 * + * Clock functions * + ****************************************************/ + +/** + * NOTES: + * Clock functions are required for WASI libc support. If you don't need to + * support WASI libc, there is no need to implement these APIs. + */ + +/** + * Get the resolution of the specified clock. + * + * @param clock_id clock identifier + * @param resolution output variable to store the clock resolution + */ +__wasi_errno_t +os_clock_res_get(__wasi_clockid_t clock_id, __wasi_timestamp_t *resolution); + +/** + * Get the current time of the specified clock. + * + * @param clock_id clock identifier + * @param precision the maximum lag that the returned time value may have, + * compared to its actual value. + * @param time output variable to store the clock time + */ +__wasi_errno_t +os_clock_time_get(__wasi_clockid_t clock_id, __wasi_timestamp_t precision, + __wasi_timestamp_t *time); + +#ifdef __cplusplus +} +#endif + +/* Experimental */ + +/* Used in posix.c around L2259 and expect the return code + * of ioctl() directly. + */ +int +os_ioctl(os_file_handle handle, int request, ...); + +/* Higher level API: + * __wasi_errno_t + * blocking_op_poll(wasm_exec_env_t exec_env, os_poll_file_handle *pfds, + * os_nfds_t nfds, int timeout_ms, int *retp) + * Already format the errno and expect the return code of poll() directly. + */ +int +os_poll(os_poll_file_handle *pfds, os_nfds_t nfs, int timeout); + +bool +os_compare_file_handle(os_file_handle handle1, os_file_handle handle2); + +#endif /* #ifndef PLATFORM_API_EXTENSION_H */ diff --git a/wamr/include/platform_api_vmcore.h b/wamr/include/platform_api_vmcore.h new file mode 100644 index 0000000..1db9650 --- /dev/null +++ b/wamr/include/platform_api_vmcore.h @@ -0,0 +1,192 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_API_VMCORE_H +#define _PLATFORM_API_VMCORE_H + +#include "platform_common.h" // platform_common.h +#include "platform_internal.h" // platform_internal.h + +#ifdef __cplusplus +extern "C" { +#endif + +/**************************************************** + * Section 1 * + * Interfaces required by the runtime * + ****************************************************/ + +/** + * Initialize the platform internal resources if needed, + * this function is called by wasm_runtime_init() and + * wasm_runtime_full_init() + * + * @return 0 if success + */ +int +bh_platform_init(void); + +/** + * Destroy the platform internal resources if needed, + * this function is called by wasm_runtime_destroy() + */ +void +bh_platform_destroy(void); + +/** + ******** memory allocator APIs ********** + */ + +void * +os_malloc(unsigned size); + +void * +os_realloc(void *ptr, unsigned size); + +void +os_free(void *ptr); + +/** + * Note: the above APIs can simply return NULL if wasm runtime + * isn't initialized with Alloc_With_System_Allocator. + * Refer to wasm_runtime_full_init(). + */ + +int +os_printf(const char *format, ...); + +int +os_vprintf(const char *format, va_list ap); + +/** + * Get microseconds after boot. + */ +uint64 +os_time_get_boot_us(void); + +/** + * Get thread-specific CPU-time clock in microseconds + */ +uint64 +os_time_thread_cputime_us(void); + +/** + * Get current thread id. + * Implementation optional: Used by runtime for logging only. + */ +korp_tid +os_self_thread(void); + +/** + * Get current thread's stack boundary address, used for runtime + * to check the native stack overflow. Return NULL if it is not + * easy to implement, but may have potential issue. + */ +uint8 * +os_thread_get_stack_boundary(void); + +/** + * Set whether the MAP_JIT region write protection is enabled for this thread. + * Pass true to make the region executable, false to make it writable. + */ +void +os_thread_jit_write_protect_np(bool enabled); + +/** + ************** mutext APIs *********** + * vmcore: Not required until pthread is supported by runtime + * app-mgr: Must be implemented + */ + +int +os_mutex_init(korp_mutex *mutex); + +int +os_mutex_destroy(korp_mutex *mutex); + +int +os_mutex_lock(korp_mutex *mutex); + +int +os_mutex_unlock(korp_mutex *mutex); + +/************************************************** + * Section 2 * + * APIs required by WAMR AOT * + **************************************************/ + +/* Memory map modes */ +enum { + MMAP_PROT_NONE = 0, + MMAP_PROT_READ = 1, + MMAP_PROT_WRITE = 2, + MMAP_PROT_EXEC = 4 +}; + +/* Memory map flags */ +enum { + MMAP_MAP_NONE = 0, + /* Put the mapping into 0 to 2 G, supported only on x86_64 */ + MMAP_MAP_32BIT = 1, + /* Don't interpret addr as a hint: place the mapping at exactly + that address. */ + MMAP_MAP_FIXED = 2, +}; + +void * +os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file); +void +os_munmap(void *addr, size_t size); +int +os_mprotect(void *addr, size_t size, int prot); + +static inline void * +os_mremap_slow(void *old_addr, size_t old_size, size_t new_size) +{ + void *new_memory = os_mmap(NULL, new_size, MMAP_PROT_WRITE | MMAP_PROT_READ, + 0, os_get_invalid_handle()); + if (!new_memory) { + return NULL; + } + /* + * bh_memcpy_s can't be used as it doesn't support values bigger than + * UINT32_MAX + */ + memcpy(new_memory, old_addr, new_size < old_size ? new_size : old_size); + os_munmap(old_addr, old_size); + + return new_memory; +} + +/* Doesn't guarantee that protection flags will be preserved. + os_mprotect() must be called after remapping. */ +void * +os_mremap(void *old_addr, size_t old_size, size_t new_size); + +#if (WASM_MEM_DUAL_BUS_MIRROR != 0) +void * +os_get_dbus_mirror(void *ibus); +#endif + +/** + * Flush cpu data cache, in some CPUs, after applying relocation to the + * AOT code, the code may haven't been written back to the cpu data cache, + * which may cause unexpected behaviour when executing the AOT code. + * Implement this function if required, or just leave it empty. + */ +void +os_dcache_flush(void); + +/** + * Flush instruction cache. + */ +void +os_icache_flush(void *start, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif /* #ifndef _PLATFORM_API_VMCORE_H */ diff --git a/wamr/include/platform_common.h b/wamr/include/platform_common.h new file mode 100644 index 0000000..6dd9976 --- /dev/null +++ b/wamr/include/platform_common.h @@ -0,0 +1,204 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_COMMON_H +#define _PLATFORM_COMMON_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "platform_internal.h" // platform_internal.h +#include "config.h" // ../../../config.h + +#define BH_MAX_THREAD 32 + +#define BHT_ERROR (-1) +#define BHT_TIMED_OUT (1) +#define BHT_OK (0) + +#define BHT_WAIT_FOREVER ((uint64)-1LL) + +#define BH_KB (1024) +#define BH_MB ((BH_KB)*1024) +#define BH_GB ((BH_MB)*1024) + +#ifndef BH_MALLOC +#define BH_MALLOC os_malloc +#endif + +#ifndef BH_FREE +#define BH_FREE os_free +#endif + +#ifndef BH_TIME_T_MAX +#define BH_TIME_T_MAX LONG_MAX +#endif + +#if defined(_MSC_BUILD) +#if defined(COMPILING_WASM_RUNTIME_API) +__declspec(dllexport) void *BH_MALLOC(unsigned int size); +__declspec(dllexport) void BH_FREE(void *ptr); +#else +__declspec(dllimport) void *BH_MALLOC(unsigned int size); +__declspec(dllimport) void BH_FREE(void *ptr); +#endif +#else +void * +BH_MALLOC(unsigned int size); +void +BH_FREE(void *ptr); +#endif + +#if defined(BH_VPRINTF) +#if defined(MSVC) +__declspec(dllimport) int BH_VPRINTF(const char *format, va_list ap); +#else +int +BH_VPRINTF(const char *format, va_list ap); +#endif +#endif + +#ifndef NULL +#define NULL (void *)0 +#endif + +#if !defined(BH_HAS_DLFCN) +#if defined(_POSIX_SOURCE) || defined(_POSIX_C_SOURCE) +#define BH_HAS_DLFCN 1 +#else +#define BH_HAS_DLFCN 0 +#endif +#endif + +#ifndef __cplusplus + +#ifndef true +#define true 1 +#endif + +#ifndef false +#define false 0 +#endif + +#ifndef inline +#define inline __inline +#endif + +#endif + +/* Return the offset of the given field in the given type */ +#ifndef offsetof +/* GCC 4.0 and later has the builtin. */ +#if defined(__GNUC__) && __GNUC__ >= 4 +#define offsetof(Type, field) __builtin_offsetof(Type, field) +#else +#define offsetof(Type, field) ((size_t)(&((Type *)0)->field)) +#endif +#endif + +typedef uint8_t uint8; +typedef int8_t int8; +typedef uint16_t uint16; +typedef int16_t int16; +typedef uint32_t uint32; +typedef int32_t int32; +typedef float float32; +typedef double float64; +typedef uint64_t uint64; +typedef int64_t int64; + +typedef void *(*thread_start_routine_t)(void *); + +#ifndef bh_socket_t +/* If no socket defined on current platform, + give a fake definition to make the compiler happy */ +#define bh_socket_t int +#endif + +/* Format specifiers macros in case + they are not provided by compiler */ +#ifndef __PRI64_PREFIX +#if UINTPTR_MAX == UINT64_MAX +#define __PRI64_PREFIX "l" +#define __PRIPTR_PREFIX "l" +#else +#define __PRI64_PREFIX "ll" +#define __PRIPTR_PREFIX +#endif +#endif /* #ifndef __PRI64_PREFIX */ + +/* Macros for printing format specifiers */ +#ifndef PRId32 +#define PRId32 "d" +#endif +#ifndef PRIi32 +#define PRIi32 "i" +#endif +#ifndef PRIu32 +#define PRIu32 "u" +#endif +#ifndef PRIx32 +#define PRIx32 "x" +#endif +#ifndef PRIX32 +#define PRIX32 "X" +#endif + +#ifndef PRId64 +#define PRId64 __PRI64_PREFIX "d" +#endif +#ifndef PRIu64 +#define PRIu64 __PRI64_PREFIX "u" +#endif +#ifndef PRIx64 +#define PRIx64 __PRI64_PREFIX "x" +#endif +#ifndef PRIX64 +#define PRIX64 __PRI64_PREFIX "X" +#endif +#ifndef PRIxPTR +#define PRIxPTR __PRIPTR_PREFIX "x" +#endif +#ifndef PRIXPTR +#define PRIXPTR __PRIPTR_PREFIX "X" +#endif + +/* Macros for scanning format specifiers */ +#ifndef SCNd32 +#define SCNd32 "d" +#endif +#ifndef SCNi32 +#define SCNi32 "i" +#endif +#ifndef SCNu32 +#define SCNu32 "u" +#endif +#ifndef SCNx32 +#define SCNx32 "x" +#endif + +#ifndef SCNd64 +#define SCNd64 __PRI64_PREFIX "d" +#endif +#ifndef SCNu64 +#define SCNu64 __PRI64_PREFIX "u" +#endif +#ifndef SCNx64 +#define SCNx64 __PRI64_PREFIX "x" +#endif +#ifndef SCNxPTR +#define SCNxPTR __PRIPTR_PREFIX "x" +#endif + +#ifndef NAN +#define NAN (0.0 / 0.0) +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* #ifndef _PLATFORM_COMMON_H */ diff --git a/wamr/include/platform_internal.h b/wamr/include/platform_internal.h new file mode 100644 index 0000000..10de33f --- /dev/null +++ b/wamr/include/platform_internal.h @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * Copyright (C) 2023 Dylibso. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _PLATFORM_INTERNAL_H +#define _PLATFORM_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef BH_PLATFORM_COSMOPOLITAN +#define BH_PLATFORM_COSMOPOLITAN +#endif + +/* Stack size of applet threads's native part. */ +#define BH_APPLET_PRESERVED_STACK_SIZE (32 * 1024) + +/* Default thread priority */ +#define BH_THREAD_DEFAULT_PRIORITY 0 + +typedef pthread_t korp_tid; +typedef pthread_mutex_t korp_mutex; +typedef pthread_cond_t korp_cond; +typedef pthread_t korp_thread; +typedef pthread_rwlock_t korp_rwlock; +typedef sem_t korp_sem; + +#define OS_THREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER + +#define os_thread_local_attribute __thread + +#define bh_socket_t int + +typedef int os_file_handle; +typedef DIR *os_dir_stream; +typedef int os_raw_file_handle; +typedef struct pollfd os_poll_file_handle; +typedef nfds_t os_nfds_t; +typedef struct timespec os_timespec; + +static inline os_file_handle +os_get_invalid_handle(void) +{ + return -1; +} + +#if WASM_DISABLE_WRITE_GS_BASE == 0 +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) +#define os_writegsbase(base_addr) \ + do { \ + uint64 __gs_value = (uint64)(uintptr_t)base_addr; \ + asm volatile("wrgsbase %0" ::"r"(__gs_value) : "memory"); \ + } while (0) +#if 0 +/* _writegsbase_u64 also works, but need to add -mfsgsbase flag for gcc */ +#include +#define os_writegsbase(base_addr) \ + _writegsbase_u64(((uint64)(uintptr_t)base_addr)) +#endif +#endif +#endif + +#if WASM_DISABLE_HW_BOUND_CHECK == 0 +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_AARCH64) || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) + +#include + +#define OS_ENABLE_HW_BOUND_CHECK + +typedef jmp_buf korp_jmpbuf; + +#define os_setjmp setjmp +#define os_longjmp longjmp +#define os_alloca alloca + +typedef void (*os_signal_handler)(void *sig_addr); + +int +os_thread_signal_init(os_signal_handler handler); + +void +os_thread_signal_destroy(); + +bool +os_thread_signal_inited(); + +void +os_signal_unmask(); + +void +os_sigreturn(); +#endif /* end of BUILD_TARGET_X86_64/AMD_64/AARCH64/RISCV64 */ +#endif /* end of WASM_DISABLE_HW_BOUND_CHECK */ + +#define os_getpagesize getpagesize + +#ifdef __cplusplus +} +#endif + +#endif /* end of _PLATFORM_INTERNAL_H */ diff --git a/wamr/include/platform_wasi_types.h b/wamr/include/platform_wasi_types.h new file mode 100644 index 0000000..96b76dc --- /dev/null +++ b/wamr/include/platform_wasi_types.h @@ -0,0 +1,613 @@ +/* + * Copyright (C) 2023 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* + * This file declares the WASI interface. The definitions of types, macros and + * structures in this file should be consistent with those in wasi-libc: + * https://github.com/WebAssembly/wasi-libc/blob/main/libc-bottom-half/headers/public/wasi/api.h + */ + +#ifndef _PLATFORM_WASI_TYPES_H +#define _PLATFORM_WASI_TYPES_H + +#include "config.h" // ../../../config.h + +#include +#include + +/* clang-format off */ + +#ifdef __cplusplus +#ifndef _Static_assert +#define _Static_assert static_assert +#endif /* _Static_assert */ + +#ifndef _Alignof +#define _Alignof alignof +#endif /* _Alignof */ + +extern "C" { +#endif + +/* There is no need to check the WASI layout if we're using uvwasi or libc-wasi + * is not enabled at all. */ +#if WASM_ENABLE_UVWASI != 0 || WASM_ENABLE_LIBC_WASI == 0 +#define assert_wasi_layout(expr, message) /* nothing */ +#else +#define assert_wasi_layout(expr, message) _Static_assert(expr, message) +#endif + +assert_wasi_layout(_Alignof(int8_t) == 1, "non-wasi data layout"); +assert_wasi_layout(_Alignof(uint8_t) == 1, "non-wasi data layout"); +assert_wasi_layout(_Alignof(int16_t) == 2, "non-wasi data layout"); +assert_wasi_layout(_Alignof(uint16_t) == 2, "non-wasi data layout"); +assert_wasi_layout(_Alignof(int32_t) == 4, "non-wasi data layout"); +assert_wasi_layout(_Alignof(uint32_t) == 4, "non-wasi data layout"); +#if 0 +assert_wasi_layout(_Alignof(int64_t) == 8, "non-wasi data layout"); +assert_wasi_layout(_Alignof(uint64_t) == 8, "non-wasi data layout"); +#endif + +typedef uint32_t __wasi_size_t; +assert_wasi_layout(_Alignof(__wasi_size_t) == 4, "non-wasi data layout"); + +typedef uint8_t __wasi_advice_t; +#define __WASI_ADVICE_NORMAL (0) +#define __WASI_ADVICE_SEQUENTIAL (1) +#define __WASI_ADVICE_RANDOM (2) +#define __WASI_ADVICE_WILLNEED (3) +#define __WASI_ADVICE_DONTNEED (4) +#define __WASI_ADVICE_NOREUSE (5) + +typedef uint32_t __wasi_clockid_t; +#define __WASI_CLOCK_REALTIME (0) +#define __WASI_CLOCK_MONOTONIC (1) +#define __WASI_CLOCK_PROCESS_CPUTIME_ID (2) +#define __WASI_CLOCK_THREAD_CPUTIME_ID (3) + +typedef uint64_t __wasi_device_t; + +typedef uint64_t __wasi_dircookie_t; +#define __WASI_DIRCOOKIE_START (0) + +typedef uint32_t __wasi_dirnamlen_t; + +typedef uint16_t __wasi_errno_t; +#define __WASI_ESUCCESS (0) +#define __WASI_E2BIG (1) +#define __WASI_EACCES (2) +#define __WASI_EADDRINUSE (3) +#define __WASI_EADDRNOTAVAIL (4) +#define __WASI_EAFNOSUPPORT (5) +#define __WASI_EAGAIN (6) +#define __WASI_EALREADY (7) +#define __WASI_EBADF (8) +#define __WASI_EBADMSG (9) +#define __WASI_EBUSY (10) +#define __WASI_ECANCELED (11) +#define __WASI_ECHILD (12) +#define __WASI_ECONNABORTED (13) +#define __WASI_ECONNREFUSED (14) +#define __WASI_ECONNRESET (15) +#define __WASI_EDEADLK (16) +#define __WASI_EDESTADDRREQ (17) +#define __WASI_EDOM (18) +#define __WASI_EDQUOT (19) +#define __WASI_EEXIST (20) +#define __WASI_EFAULT (21) +#define __WASI_EFBIG (22) +#define __WASI_EHOSTUNREACH (23) +#define __WASI_EIDRM (24) +#define __WASI_EILSEQ (25) +#define __WASI_EINPROGRESS (26) +#define __WASI_EINTR (27) +#define __WASI_EINVAL (28) +#define __WASI_EIO (29) +#define __WASI_EISCONN (30) +#define __WASI_EISDIR (31) +#define __WASI_ELOOP (32) +#define __WASI_EMFILE (33) +#define __WASI_EMLINK (34) +#define __WASI_EMSGSIZE (35) +#define __WASI_EMULTIHOP (36) +#define __WASI_ENAMETOOLONG (37) +#define __WASI_ENETDOWN (38) +#define __WASI_ENETRESET (39) +#define __WASI_ENETUNREACH (40) +#define __WASI_ENFILE (41) +#define __WASI_ENOBUFS (42) +#define __WASI_ENODEV (43) +#define __WASI_ENOENT (44) +#define __WASI_ENOEXEC (45) +#define __WASI_ENOLCK (46) +#define __WASI_ENOLINK (47) +#define __WASI_ENOMEM (48) +#define __WASI_ENOMSG (49) +#define __WASI_ENOPROTOOPT (50) +#define __WASI_ENOSPC (51) +#define __WASI_ENOSYS (52) +#define __WASI_ENOTCONN (53) +#define __WASI_ENOTDIR (54) +#define __WASI_ENOTEMPTY (55) +#define __WASI_ENOTRECOVERABLE (56) +#define __WASI_ENOTSOCK (57) +#define __WASI_ENOTSUP (58) +#define __WASI_ENOTTY (59) +#define __WASI_ENXIO (60) +#define __WASI_EOVERFLOW (61) +#define __WASI_EOWNERDEAD (62) +#define __WASI_EPERM (63) +#define __WASI_EPIPE (64) +#define __WASI_EPROTO (65) +#define __WASI_EPROTONOSUPPORT (66) +#define __WASI_EPROTOTYPE (67) +#define __WASI_ERANGE (68) +#define __WASI_EROFS (69) +#define __WASI_ESPIPE (70) +#define __WASI_ESRCH (71) +#define __WASI_ESTALE (72) +#define __WASI_ETIMEDOUT (73) +#define __WASI_ETXTBSY (74) +#define __WASI_EXDEV (75) +#define __WASI_ENOTCAPABLE (76) + +#if defined(_MSC_VER) +#define ALIGNED_(x) __declspec(align(x)) +#define WARN_UNUSED _Check_return_ +#elif defined(__GNUC__) +#define ALIGNED_(x) __attribute__ ((aligned(x))) +#define WARN_UNUSED __attribute__((__warn_unused_result__)) +#endif + +#define ALIGNED_TYPE(t,x) typedef t ALIGNED_(x) + +typedef uint16_t __wasi_eventrwflags_t; +#define __WASI_EVENT_FD_READWRITE_HANGUP (0x0001) + +typedef uint8_t __wasi_eventtype_t; +#define __WASI_EVENTTYPE_CLOCK (0) +#define __WASI_EVENTTYPE_FD_READ (1) +#define __WASI_EVENTTYPE_FD_WRITE (2) + +typedef uint32_t __wasi_exitcode_t; + +typedef int32_t __wasi_fd_t; + +typedef uint16_t __wasi_fdflags_t; +#define __WASI_FDFLAG_APPEND (0x0001) +#define __WASI_FDFLAG_DSYNC (0x0002) +#define __WASI_FDFLAG_NONBLOCK (0x0004) +#define __WASI_FDFLAG_RSYNC (0x0008) +#define __WASI_FDFLAG_SYNC (0x0010) + +typedef int64_t __wasi_filedelta_t; + +typedef uint64_t __wasi_filesize_t; + +typedef uint8_t __wasi_filetype_t; +#define __WASI_FILETYPE_UNKNOWN (0) +#define __WASI_FILETYPE_BLOCK_DEVICE (1) +#define __WASI_FILETYPE_CHARACTER_DEVICE (2) +#define __WASI_FILETYPE_DIRECTORY (3) +#define __WASI_FILETYPE_REGULAR_FILE (4) +#define __WASI_FILETYPE_SOCKET_DGRAM (5) +#define __WASI_FILETYPE_SOCKET_STREAM (6) +#define __WASI_FILETYPE_SYMBOLIC_LINK (7) + +typedef uint16_t __wasi_fstflags_t; +#define __WASI_FILESTAT_SET_ATIM (0x0001) +#define __WASI_FILESTAT_SET_ATIM_NOW (0x0002) +#define __WASI_FILESTAT_SET_MTIM (0x0004) +#define __WASI_FILESTAT_SET_MTIM_NOW (0x0008) + +typedef uint64_t __wasi_inode_t; + +ALIGNED_TYPE(uint64_t, 8) __wasi_linkcount_t; + +typedef uint32_t __wasi_lookupflags_t; +#define __WASI_LOOKUP_SYMLINK_FOLLOW (0x00000001) + +typedef uint16_t __wasi_oflags_t; +#define __WASI_O_CREAT (0x0001) +#define __WASI_O_DIRECTORY (0x0002) +#define __WASI_O_EXCL (0x0004) +#define __WASI_O_TRUNC (0x0008) + +typedef uint16_t __wasi_riflags_t; +#define __WASI_SOCK_RECV_PEEK (0x0001) +#define __WASI_SOCK_RECV_WAITALL (0x0002) + +typedef uint64_t __wasi_rights_t; + +/** + * Observe that WASI defines rights in the plural form + * TODO: refactor to use RIGHTS instead of RIGHT + */ +#define __WASI_RIGHT_FD_DATASYNC ((__wasi_rights_t)(UINT64_C(1) << 0)) +#define __WASI_RIGHT_FD_READ ((__wasi_rights_t)(UINT64_C(1) << 1)) +#define __WASI_RIGHT_FD_SEEK ((__wasi_rights_t)(UINT64_C(1) << 2)) +#define __WASI_RIGHT_FD_FDSTAT_SET_FLAGS ((__wasi_rights_t)(UINT64_C(1) << 3)) +#define __WASI_RIGHT_FD_SYNC ((__wasi_rights_t)(UINT64_C(1) << 4)) +#define __WASI_RIGHT_FD_TELL ((__wasi_rights_t)(UINT64_C(1) << 5)) +#define __WASI_RIGHT_FD_WRITE ((__wasi_rights_t)(UINT64_C(1) << 6)) +#define __WASI_RIGHT_FD_ADVISE ((__wasi_rights_t)(UINT64_C(1) << 7)) +#define __WASI_RIGHT_FD_ALLOCATE ((__wasi_rights_t)(UINT64_C(1) << 8)) +#define __WASI_RIGHT_PATH_CREATE_DIRECTORY ((__wasi_rights_t)(UINT64_C(1) << 9)) +#define __WASI_RIGHT_PATH_CREATE_FILE ((__wasi_rights_t)(UINT64_C(1) << 10)) +#define __WASI_RIGHT_PATH_LINK_SOURCE ((__wasi_rights_t)(UINT64_C(1) << 11)) +#define __WASI_RIGHT_PATH_LINK_TARGET ((__wasi_rights_t)(UINT64_C(1) << 12)) +#define __WASI_RIGHT_PATH_OPEN ((__wasi_rights_t)(UINT64_C(1) << 13)) +#define __WASI_RIGHT_FD_READDIR ((__wasi_rights_t)(UINT64_C(1) << 14)) +#define __WASI_RIGHT_PATH_READLINK ((__wasi_rights_t)(UINT64_C(1) << 15)) +#define __WASI_RIGHT_PATH_RENAME_SOURCE ((__wasi_rights_t)(UINT64_C(1) << 16)) +#define __WASI_RIGHT_PATH_RENAME_TARGET ((__wasi_rights_t)(UINT64_C(1) << 17)) +#define __WASI_RIGHT_PATH_FILESTAT_GET ((__wasi_rights_t)(UINT64_C(1) << 18)) +#define __WASI_RIGHT_PATH_FILESTAT_SET_SIZE ((__wasi_rights_t)(UINT64_C(1) << 19)) +#define __WASI_RIGHT_PATH_FILESTAT_SET_TIMES ((__wasi_rights_t)(UINT64_C(1) << 20)) +#define __WASI_RIGHT_FD_FILESTAT_GET ((__wasi_rights_t)(UINT64_C(1) << 21)) +#define __WASI_RIGHT_FD_FILESTAT_SET_SIZE ((__wasi_rights_t)(UINT64_C(1) << 22)) +#define __WASI_RIGHT_FD_FILESTAT_SET_TIMES ((__wasi_rights_t)(UINT64_C(1) << 23)) +#define __WASI_RIGHT_PATH_SYMLINK ((__wasi_rights_t)(UINT64_C(1) << 24)) +#define __WASI_RIGHT_PATH_REMOVE_DIRECTORY ((__wasi_rights_t)(UINT64_C(1) << 25)) +#define __WASI_RIGHT_PATH_UNLINK_FILE ((__wasi_rights_t)(UINT64_C(1) << 26)) +#define __WASI_RIGHT_POLL_FD_READWRITE ((__wasi_rights_t)(UINT64_C(1) << 27)) +#define __WASI_RIGHT_SOCK_CONNECT ((__wasi_rights_t)(UINT64_C(1) << 28)) +#define __WASI_RIGHT_SOCK_LISTEN ((__wasi_rights_t)(UINT64_C(1) << 29)) +#define __WASI_RIGHT_SOCK_BIND ((__wasi_rights_t)(UINT64_C(1) << 30)) +#define __WASI_RIGHT_SOCK_ACCEPT ((__wasi_rights_t)(UINT64_C(1) << 31)) +#define __WASI_RIGHT_SOCK_RECV ((__wasi_rights_t)(UINT64_C(1) << 32)) +#define __WASI_RIGHT_SOCK_SEND ((__wasi_rights_t)(UINT64_C(1) << 33)) +#define __WASI_RIGHT_SOCK_ADDR_LOCAL ((__wasi_rights_t)(UINT64_C(1) << 34)) +#define __WASI_RIGHT_SOCK_ADDR_REMOTE ((__wasi_rights_t)(UINT64_C(1) << 35)) +#define __WASI_RIGHT_SOCK_RECV_FROM ((__wasi_rights_t)(UINT64_C(1) << 36)) +#define __WASI_RIGHT_SOCK_SEND_TO ((__wasi_rights_t)(UINT64_C(1) << 37)) + +typedef uint16_t __wasi_roflags_t; +#define __WASI_SOCK_RECV_DATA_TRUNCATED (0x0001) + +typedef uint8_t __wasi_sdflags_t; +#define __WASI_SHUT_RD (0x01) +#define __WASI_SHUT_WR (0x02) + +typedef uint16_t __wasi_siflags_t; + +typedef uint8_t __wasi_signal_t; + +typedef uint16_t __wasi_subclockflags_t; +#define __WASI_SUBSCRIPTION_CLOCK_ABSTIME (0x0001) + +typedef uint64_t __wasi_timestamp_t; + +typedef uint64_t __wasi_userdata_t; + +typedef uint8_t __wasi_whence_t; +#define __WASI_WHENCE_SET (0) +#define __WASI_WHENCE_CUR (1) +#define __WASI_WHENCE_END (2) + +typedef uint8_t __wasi_preopentype_t; +#define __WASI_PREOPENTYPE_DIR (0) + +struct fd_table; +struct fd_prestats; +struct argv_environ_values; +struct addr_pool; + +typedef struct ALIGNED_(8) __wasi_dirent_t { + __wasi_dircookie_t d_next; + __wasi_inode_t d_ino; + __wasi_dirnamlen_t d_namlen; + __wasi_filetype_t d_type; +} __wasi_dirent_t; +assert_wasi_layout(offsetof(__wasi_dirent_t, d_next) == 0, "non-wasi data layout"); +assert_wasi_layout(offsetof(__wasi_dirent_t, d_ino) == 8, "non-wasi data layout"); +assert_wasi_layout(offsetof(__wasi_dirent_t, d_namlen) == 16, "non-wasi data layout"); +assert_wasi_layout(offsetof(__wasi_dirent_t, d_type) == 20, "non-wasi data layout"); +assert_wasi_layout(sizeof(__wasi_dirent_t) == 24, "non-wasi data layout"); +assert_wasi_layout(_Alignof(__wasi_dirent_t) == 8, "non-wasi data layout"); + +typedef struct ALIGNED_(8) __wasi_event_t { + __wasi_userdata_t userdata; + __wasi_errno_t error; + __wasi_eventtype_t type; + uint8_t __paddings[5]; + union __wasi_event_u { + struct __wasi_event_u_fd_readwrite_t { + __wasi_filesize_t nbytes; + __wasi_eventrwflags_t flags; + uint8_t __paddings[6]; + } fd_readwrite; + } u; +} __wasi_event_t; +assert_wasi_layout(offsetof(__wasi_event_t, userdata) == 0, "non-wasi data layout"); +assert_wasi_layout(offsetof(__wasi_event_t, error) == 8, "non-wasi data layout"); +assert_wasi_layout(offsetof(__wasi_event_t, type) == 10, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_event_t, u.fd_readwrite.nbytes) == 16, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_event_t, u.fd_readwrite.flags) == 24, "non-wasi data layout"); +assert_wasi_layout(sizeof(__wasi_event_t) == 32, "non-wasi data layout"); +assert_wasi_layout(_Alignof(__wasi_event_t) == 8, "non-wasi data layout"); + +typedef struct __wasi_prestat_t { + __wasi_preopentype_t pr_type; + union __wasi_prestat_u { + struct __wasi_prestat_u_dir_t { + size_t pr_name_len; + } dir; + } u; +} __wasi_prestat_t; +assert_wasi_layout(offsetof(__wasi_prestat_t, pr_type) == 0, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + offsetof(__wasi_prestat_t, u.dir.pr_name_len) == 4, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + offsetof(__wasi_prestat_t, u.dir.pr_name_len) == 8, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + sizeof(__wasi_prestat_t) == 8, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + sizeof(__wasi_prestat_t) == 16, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + _Alignof(__wasi_prestat_t) == 4, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + _Alignof(__wasi_prestat_t) == 8, "non-wasi data layout"); + +typedef struct ALIGNED_(8) __wasi_fdstat_t { + __wasi_filetype_t fs_filetype; + __wasi_fdflags_t fs_flags; + uint8_t __paddings[4]; + __wasi_rights_t fs_rights_base; + __wasi_rights_t fs_rights_inheriting; +} __wasi_fdstat_t; +assert_wasi_layout( + offsetof(__wasi_fdstat_t, fs_filetype) == 0, "non-wasi data layout"); +assert_wasi_layout(offsetof(__wasi_fdstat_t, fs_flags) == 2, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_fdstat_t, fs_rights_base) == 8, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_fdstat_t, fs_rights_inheriting) == 16, + "non-wasi data layout"); +assert_wasi_layout(sizeof(__wasi_fdstat_t) == 24, "non-wasi data layout"); +assert_wasi_layout(_Alignof(__wasi_fdstat_t) == 8, "non-wasi data layout"); + +typedef struct ALIGNED_(8) __wasi_filestat_t { + __wasi_device_t st_dev; + __wasi_inode_t st_ino; + __wasi_filetype_t st_filetype; + __wasi_linkcount_t st_nlink; + __wasi_filesize_t st_size; + __wasi_timestamp_t st_atim; + __wasi_timestamp_t st_mtim; + __wasi_timestamp_t st_ctim; +} __wasi_filestat_t; +assert_wasi_layout(offsetof(__wasi_filestat_t, st_dev) == 0, "non-wasi data layout"); +assert_wasi_layout(offsetof(__wasi_filestat_t, st_ino) == 8, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_filestat_t, st_filetype) == 16, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_filestat_t, st_nlink) == 24, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_filestat_t, st_size) == 32, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_filestat_t, st_atim) == 40, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_filestat_t, st_mtim) == 48, "non-wasi data layout"); +assert_wasi_layout( + offsetof(__wasi_filestat_t, st_ctim) == 56, "non-wasi data layout"); +assert_wasi_layout(sizeof(__wasi_filestat_t) == 64, "non-wasi data layout"); +assert_wasi_layout(_Alignof(__wasi_filestat_t) == 8, "non-wasi data layout"); + +typedef struct __wasi_ciovec_t { + const void *buf; + size_t buf_len; +} __wasi_ciovec_t; +assert_wasi_layout(offsetof(__wasi_ciovec_t, buf) == 0, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + offsetof(__wasi_ciovec_t, buf_len) == 4, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + offsetof(__wasi_ciovec_t, buf_len) == 8, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + sizeof(__wasi_ciovec_t) == 8, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + sizeof(__wasi_ciovec_t) == 16, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + _Alignof(__wasi_ciovec_t) == 4, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + _Alignof(__wasi_ciovec_t) == 8, "non-wasi data layout"); + +typedef struct __wasi_iovec_t { + void *buf; + size_t buf_len; +} __wasi_iovec_t; +assert_wasi_layout(offsetof(__wasi_iovec_t, buf) == 0, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + offsetof(__wasi_iovec_t, buf_len) == 4, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + offsetof(__wasi_iovec_t, buf_len) == 8, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + sizeof(__wasi_iovec_t) == 8, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + sizeof(__wasi_iovec_t) == 16, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 4 || + _Alignof(__wasi_iovec_t) == 4, "non-wasi data layout"); +assert_wasi_layout(sizeof(void *) != 8 || + _Alignof(__wasi_iovec_t) == 8, "non-wasi data layout"); + +/** + * The contents of a `subscription` when type is `eventtype::clock`. + */ +typedef struct ALIGNED_(8) __wasi_subscription_clock_t { + /** + * The clock against which to compare the timestamp. + */ + __wasi_clockid_t clock_id; + + uint8_t __paddings1[4]; + + /** + * The absolute or relative timestamp. + */ + __wasi_timestamp_t timeout; + + /** + * The amount of time that the implementation may wait additionally + * to coalesce with other events. + */ + __wasi_timestamp_t precision; + + /** + * Flags specifying whether the timeout is absolute or relative + */ + __wasi_subclockflags_t flags; + + uint8_t __paddings2[4]; + +} __wasi_subscription_clock_t; + +assert_wasi_layout(sizeof(__wasi_subscription_clock_t) == 32, "witx calculated size"); +assert_wasi_layout(_Alignof(__wasi_subscription_clock_t) == 8, "witx calculated align"); +assert_wasi_layout(offsetof(__wasi_subscription_clock_t, clock_id) == 0, "witx calculated offset"); +assert_wasi_layout(offsetof(__wasi_subscription_clock_t, timeout) == 8, "witx calculated offset"); +assert_wasi_layout(offsetof(__wasi_subscription_clock_t, precision) == 16, "witx calculated offset"); +assert_wasi_layout(offsetof(__wasi_subscription_clock_t, flags) == 24, "witx calculated offset"); + +/** + * The contents of a `subscription` when type is type is + * `eventtype::fd_read` or `eventtype::fd_write`. + */ +typedef struct __wasi_subscription_fd_readwrite_t { + /** + * The file descriptor on which to wait for it to become ready for reading or writing. + */ + __wasi_fd_t fd; + +} __wasi_subscription_fd_readwrite_t; + +assert_wasi_layout(sizeof(__wasi_subscription_fd_readwrite_t) == 4, "witx calculated size"); +assert_wasi_layout(_Alignof(__wasi_subscription_fd_readwrite_t) == 4, "witx calculated align"); +assert_wasi_layout(offsetof(__wasi_subscription_fd_readwrite_t, fd) == 0, "witx calculated offset"); + +/** + * The contents of a `subscription`. + */ +typedef union __wasi_subscription_u_u_t { + __wasi_subscription_clock_t clock; + __wasi_subscription_fd_readwrite_t fd_readwrite; +} __wasi_subscription_u_u_t ; + +typedef struct ALIGNED_(8) __wasi_subscription_u_t { + __wasi_eventtype_t type; + __wasi_subscription_u_u_t u; +} __wasi_subscription_u_t; + +assert_wasi_layout(sizeof(__wasi_subscription_u_t) == 40, "witx calculated size"); +assert_wasi_layout(_Alignof(__wasi_subscription_u_t) == 8, "witx calculated align"); +assert_wasi_layout(offsetof(__wasi_subscription_u_t, u) == 8, "witx calculated union offset"); +assert_wasi_layout(sizeof(__wasi_subscription_u_u_t) == 32, "witx calculated union size"); +assert_wasi_layout(_Alignof(__wasi_subscription_u_u_t) == 8, "witx calculated union align"); + +/** + * Subscription to an event. + */ +typedef struct __wasi_subscription_t { + /** + * User-provided value that is attached to the subscription in the + * implementation and returned through `event::userdata`. + */ + __wasi_userdata_t userdata; + + /** + * The type of the event to which to subscribe, and its contents + */ + __wasi_subscription_u_t u; + +} __wasi_subscription_t; + +assert_wasi_layout(sizeof(__wasi_subscription_t) == 48, "witx calculated size"); +assert_wasi_layout(_Alignof(__wasi_subscription_t) == 8, "witx calculated align"); +assert_wasi_layout(offsetof(__wasi_subscription_t, userdata) == 0, "witx calculated offset"); +assert_wasi_layout(offsetof(__wasi_subscription_t, u) == 8, "witx calculated offset"); + +/* keep syncing with wasi_socket_ext.h */ +typedef enum { + /* Used only for sock_addr_resolve hints */ + SOCKET_ANY = -1, + SOCKET_DGRAM = 0, + SOCKET_STREAM, +} __wasi_sock_type_t; + +typedef uint16_t __wasi_ip_port_t; + +/* Ensure that __wasi_addr_type_t has a size of 4 byte (I32). + However, it will not have the type safety of enum. */ +typedef uint32_t __wasi_addr_type_t; +enum { IPv4 = 0, IPv6 }; + +/* n0.n1.n2.n3 */ +typedef struct __wasi_addr_ip4_t { + uint8_t n0; + uint8_t n1; + uint8_t n2; + uint8_t n3; +} __wasi_addr_ip4_t; + +typedef struct __wasi_addr_ip4_port_t { + __wasi_addr_ip4_t addr; + __wasi_ip_port_t port; +} __wasi_addr_ip4_port_t; + +typedef struct __wasi_addr_ip6_t { + uint16_t n0; + uint16_t n1; + uint16_t n2; + uint16_t n3; + uint16_t h0; + uint16_t h1; + uint16_t h2; + uint16_t h3; +} __wasi_addr_ip6_t; + +typedef struct __wasi_addr_ip6_port_t { + __wasi_addr_ip6_t addr; + __wasi_ip_port_t port; +} __wasi_addr_ip6_port_t; + +typedef struct __wasi_addr_ip_t { + __wasi_addr_type_t kind; + union { + __wasi_addr_ip4_t ip4; + __wasi_addr_ip6_t ip6; + } addr; +} __wasi_addr_ip_t; + +typedef struct __wasi_addr_t { + __wasi_addr_type_t kind; + union { + __wasi_addr_ip4_port_t ip4; + __wasi_addr_ip6_port_t ip6; + } addr; +} __wasi_addr_t; + +typedef enum { INET4 = 0, INET6, INET_UNSPEC } __wasi_address_family_t; + +typedef struct __wasi_addr_info_t { + __wasi_addr_t addr; + __wasi_sock_type_t type; +} __wasi_addr_info_t; + +typedef struct __wasi_addr_info_hints_t { + __wasi_sock_type_t type; + __wasi_address_family_t family; + // this is to workaround lack of optional parameters + uint8_t hints_enabled; +} __wasi_addr_info_hints_t; + +#undef assert_wasi_layout + +/* clang-format on */ +#ifdef __cplusplus +} +#endif + +#endif /* end of _PLATFORM_WASI_TYPES_H */ \ No newline at end of file diff --git a/wamr/include/runtime_timer.h b/wamr/include/runtime_timer.h new file mode 100644 index 0000000..32ae582 --- /dev/null +++ b/wamr/include/runtime_timer.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef LIB_BASE_RUNTIME_TIMER_H_ +#define LIB_BASE_RUNTIME_TIMER_H_ + +#include "bh_platform.h" // bh_platform.h + +#ifdef __cplusplus +extern "C" { +#endif + +uint64 +bh_get_tick_ms(void); +uint32 +bh_get_elpased_ms(uint32 *last_system_clock); + +struct _timer_ctx; +typedef struct _timer_ctx *timer_ctx_t; +typedef void (*timer_callback_f)(unsigned int id, unsigned int owner); +typedef void (*check_timer_expiry_f)(timer_ctx_t ctx); + +timer_ctx_t +create_timer_ctx(timer_callback_f timer_handler, check_timer_expiry_f, + int prealloc_num, unsigned int owner); +void destroy_timer_ctx(timer_ctx_t); +unsigned int +timer_ctx_get_owner(timer_ctx_t ctx); + +uint32 +sys_create_timer(timer_ctx_t ctx, int interval, bool is_period, + bool auto_start); +bool +sys_timer_destroy(timer_ctx_t ctx, uint32 timer_id); +bool +sys_timer_cancel(timer_ctx_t ctx, uint32 timer_id); +bool +sys_timer_restart(timer_ctx_t ctx, uint32 timer_id, int interval); +void +cleanup_app_timers(timer_ctx_t ctx); +uint32 +check_app_timers(timer_ctx_t ctx); +uint32 +get_expiry_ms(timer_ctx_t ctx); + +#ifdef __cplusplus +} +#endif +#endif /* LIB_BASE_RUNTIME_TIMER_H_ */ diff --git a/wamr/include/version.h b/wamr/include/version.h new file mode 100644 index 0000000..261c9b8 --- /dev/null +++ b/wamr/include/version.h @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* + * version.h.in is a template file. version.h is a generated file. + * Please do not edit both files directly. + * + * Any changes to the version should be done in build-scripts/version.cmake. + * + * Continue to maintain the version.h for certain embedded platforms. + */ + +#ifndef _WAMR_VERSION_H_ +#define _WAMR_VERSION_H_ + +/* clang-format off */ +#define WAMR_VERSION_MAJOR 2 +#define WAMR_VERSION_MINOR 4 +#define WAMR_VERSION_PATCH 1 +/* clang-format on */ + +#endif diff --git a/wamr/include/wasm.h b/wamr/include/wasm.h new file mode 100644 index 0000000..1225a60 --- /dev/null +++ b/wamr/include/wasm.h @@ -0,0 +1,1517 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_H_ +#define _WASM_H_ + +#include "bh_platform.h" // bh_platform.h +#include "bh_hashmap.h" // bh_hashmap.h +#include "bh_assert.h" // bh_assert.h + + +#ifdef __cplusplus +extern "C" { +#endif + +/* Value Type */ +#define VALUE_TYPE_I32 0x7F +#define VALUE_TYPE_I64 0X7E +#define VALUE_TYPE_F32 0x7D +#define VALUE_TYPE_F64 0x7C +#define VALUE_TYPE_V128 0x7B +#define VALUE_TYPE_FUNCREF 0x70 +#define VALUE_TYPE_EXTERNREF 0x6F +#define VALUE_TYPE_VOID 0x40 + +/* Packed Types */ +#define PACKED_TYPE_I8 0x78 +#define PACKED_TYPE_I16 0x77 + +/* Reference Types */ +#define REF_TYPE_NULLFUNCREF 0x73 +#define REF_TYPE_NULLEXTERNREF 0x72 +#define REF_TYPE_NULLREF 0x71 +#define REF_TYPE_FUNCREF VALUE_TYPE_FUNCREF /* 0x70 */ +#define REF_TYPE_EXTERNREF VALUE_TYPE_EXTERNREF /* 0x6F */ +#define REF_TYPE_ANYREF 0x6E +#define REF_TYPE_EQREF 0x6D +#define REF_TYPE_I31REF 0x6C +#define REF_TYPE_STRUCTREF 0x6B +#define REF_TYPE_ARRAYREF 0x6A +#define REF_TYPE_HT_NON_NULLABLE 0x64 +#define REF_TYPE_HT_NULLABLE 0x63 +#define REF_TYPE_STRINGREF VALUE_TYPE_STRINGREF /* 0x67 */ +#define REF_TYPE_STRINGVIEWWTF8 VALUE_TYPE_STRINGVIEWWTF8 /* 0x66 */ +#define REF_TYPE_STRINGVIEWWTF16 VALUE_TYPE_STRINGVIEWWTF16 /* 0x62 */ +#define REF_TYPE_STRINGVIEWITER VALUE_TYPE_STRINGVIEWITER /* 0x61 */ + +/* Heap Types */ +#define HEAP_TYPE_NOFUNC (-0x0D) +#define HEAP_TYPE_NOEXTERN (-0x0E) +#define HEAP_TYPE_NONE (-0x0F) +#define HEAP_TYPE_FUNC (-0x10) +#define HEAP_TYPE_EXTERN (-0x11) +#define HEAP_TYPE_ANY (-0x12) +#define HEAP_TYPE_EQ (-0x13) +#define HEAP_TYPE_I31 (-0x14) +#define HEAP_TYPE_STRUCT (-0x15) +#define HEAP_TYPE_ARRAY (-0x16) +#define HEAP_TYPE_STRINGREF (-0x19) +#define HEAP_TYPE_STRINGVIEWWTF8 (-0x1A) +#define HEAP_TYPE_STRINGVIEWWTF16 (-0x1E) +#define HEAP_TYPE_STRINGVIEWITER (-0x1F) + +/* Defined Types */ +#define DEFINED_TYPE_FUNC 0x60 +#define DEFINED_TYPE_STRUCT 0x5F +#define DEFINED_TYPE_ARRAY 0x5E +#define DEFINED_TYPE_SUB 0x50 +#define DEFINED_TYPE_SUB_FINAL 0x4F +#define DEFINED_TYPE_REC 0x4E + +/* Used by AOT */ +#define VALUE_TYPE_I1 0x41 +/** + * Used by loader to represent any type of i32/i64/f32/f64/v128 + * and ref types, including funcref, externref, anyref, eqref, + * (ref null $ht), (ref $ht), i31ref, structref, arrayref, + * nullfuncref, nullexternref, nullref and stringref + */ +#define VALUE_TYPE_ANY 0x42 +/** + * Used by wamr compiler to represent object ref types, + * including func object ref, externref object ref, + * internal object ref, eq object ref, i31 object ref, + * struct object ref, array object ref + */ +#define VALUE_TYPE_GC_REF 0x43 + +#define MAX_PAGE_COUNT_FLAG 0x01 +#define SHARED_MEMORY_FLAG 0x02 +#define MEMORY64_FLAG 0x04 +#define MAX_TABLE_SIZE_FLAG 0x01 +/* the shared flag for table is not actual used now */ +#define SHARED_TABLE_FLAG 0x02 +#define TABLE64_FLAG 0x04 + +/** + * In the multi-memory proposal, the memarg in loads and stores are + * reinterpreted as a bitfield, bit 6 serves as a flag indicating the presence + * of the optional memory index, if it is set, then an i32 memory index follows + * after the alignment bitfield + */ +#define OPT_MEMIDX_FLAG 0x40 + +#define DEFAULT_NUM_BYTES_PER_PAGE 65536 +#define DEFAULT_MAX_PAGES 65536 +#define DEFAULT_MEM64_MAX_PAGES UINT32_MAX + +/* Max size of linear memory */ +#define MAX_LINEAR_MEMORY_SIZE (4 * (uint64)BH_GB) +/* Roughly 274 TB */ +#define MAX_LINEAR_MEM64_MEMORY_SIZE \ + (DEFAULT_MEM64_MAX_PAGES * (uint64)64 * (uint64)BH_KB) +/* Macro to check memory flag and return appropriate memory size */ +#define GET_MAX_LINEAR_MEMORY_SIZE(is_memory64) \ + (is_memory64 ? MAX_LINEAR_MEM64_MEMORY_SIZE : MAX_LINEAR_MEMORY_SIZE) + +#if WASM_ENABLE_GC == 0 +typedef uintptr_t table_elem_type_t; +#define NULL_REF (0xFFFFFFFF) +#else +typedef void *table_elem_type_t; +#define NULL_REF (NULL) +#define REF_CELL_NUM ((uint32)sizeof(uintptr_t) / sizeof(uint32)) +#endif + +#define INIT_EXPR_NONE 0x00 +#define INIT_EXPR_TYPE_I32_CONST 0x41 +#define INIT_EXPR_TYPE_I64_CONST 0x42 +#define INIT_EXPR_TYPE_F32_CONST 0x43 +#define INIT_EXPR_TYPE_F64_CONST 0x44 +#define INIT_EXPR_TYPE_V128_CONST 0xFD +#define INIT_EXPR_TYPE_GET_GLOBAL 0x23 +#define INIT_EXPR_TYPE_I32_ADD 0x6A +#define INIT_EXPR_TYPE_I32_SUB 0x6B +#define INIT_EXPR_TYPE_I32_MUL 0x6C +#define INIT_EXPR_TYPE_I64_ADD 0x7C +#define INIT_EXPR_TYPE_I64_SUB 0x7D +#define INIT_EXPR_TYPE_I64_MUL 0x7E +#define INIT_EXPR_TYPE_REFNULL_CONST 0xD0 +#define INIT_EXPR_TYPE_FUNCREF_CONST 0xD2 +#define INIT_EXPR_TYPE_STRUCT_NEW 0xD3 +#define INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT 0xD4 +#define INIT_EXPR_TYPE_ARRAY_NEW 0xD5 +#define INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT 0xD6 +#define INIT_EXPR_TYPE_ARRAY_NEW_FIXED 0xD7 +#define INIT_EXPR_TYPE_I31_NEW 0xD8 +#define INIT_EXPR_TYPE_ANY_CONVERT_EXTERN 0xD9 +#define INIT_EXPR_TYPE_EXTERN_CONVERT_ANY 0xDA + +#define WASM_MAGIC_NUMBER 0x6d736100 +#define WASM_CURRENT_VERSION 1 + +#define SECTION_TYPE_USER 0 +#define SECTION_TYPE_TYPE 1 +#define SECTION_TYPE_IMPORT 2 +#define SECTION_TYPE_FUNC 3 +#define SECTION_TYPE_TABLE 4 +#define SECTION_TYPE_MEMORY 5 +#define SECTION_TYPE_GLOBAL 6 +#define SECTION_TYPE_EXPORT 7 +#define SECTION_TYPE_START 8 +#define SECTION_TYPE_ELEM 9 +#define SECTION_TYPE_CODE 10 +#define SECTION_TYPE_DATA 11 +#if WASM_ENABLE_BULK_MEMORY != 0 +#define SECTION_TYPE_DATACOUNT 12 +#endif +#if WASM_ENABLE_TAGS != 0 +#define SECTION_TYPE_TAG 13 +#endif +#if WASM_ENABLE_STRINGREF != 0 +#define SECTION_TYPE_STRINGREF 14 +#endif + +#define SUB_SECTION_TYPE_MODULE 0 +#define SUB_SECTION_TYPE_FUNC 1 +#define SUB_SECTION_TYPE_LOCAL 2 + +#define IMPORT_KIND_FUNC 0 +#define IMPORT_KIND_TABLE 1 +#define IMPORT_KIND_MEMORY 2 +#define IMPORT_KIND_GLOBAL 3 +#if WASM_ENABLE_TAGS != 0 +#define IMPORT_KIND_TAG 4 +#endif + +#define EXPORT_KIND_FUNC 0 +#define EXPORT_KIND_TABLE 1 +#define EXPORT_KIND_MEMORY 2 +#define EXPORT_KIND_GLOBAL 3 +#if WASM_ENABLE_TAGS != 0 +#define EXPORT_KIND_TAG 4 +#endif + +#define LABEL_TYPE_BLOCK 0 +#define LABEL_TYPE_LOOP 1 +#define LABEL_TYPE_IF 2 +#define LABEL_TYPE_FUNCTION 3 +#if WASM_ENABLE_EXCE_HANDLING != 0 +#define LABEL_TYPE_TRY 4 +#define LABEL_TYPE_CATCH 5 +#define LABEL_TYPE_CATCH_ALL 6 +#endif + +#define WASM_TYPE_FUNC 0 +#define WASM_TYPE_STRUCT 1 +#define WASM_TYPE_ARRAY 2 + +#if WASM_ENABLE_STRINGREF != 0 +#define WASM_TYPE_STRINGREF 3 +#define WASM_TYPE_STRINGVIEWWTF8 4 +#define WASM_TYPE_STRINGVIEWWTF16 5 +#define WASM_TYPE_STRINGVIEWITER 6 +#endif + +/* In WasmGC, a table can start with [0x40 0x00] to indicate it has an + * initializer */ +#define TABLE_INIT_EXPR_FLAG 0x40 + +typedef struct WASMModule WASMModule; +typedef struct WASMFunction WASMFunction; +typedef struct WASMGlobal WASMGlobal; +#if WASM_ENABLE_TAGS != 0 +typedef struct WASMTag WASMTag; +#endif + +#ifndef WASM_VALUE_DEFINED +#define WASM_VALUE_DEFINED + +typedef union V128 { + int8 i8x16[16]; + int16 i16x8[8]; + int32 i32x4[4]; + int64 i64x2[2]; + float32 f32x4[4]; + float64 f64x2[2]; +} V128; + +typedef union WASMValue { + int32 i32; + uint32 u32; + uint32 global_index; + uint32 ref_index; + int64 i64; + uint64 u64; + float32 f32; + float64 f64; + V128 v128; +#if WASM_ENABLE_GC != 0 + wasm_obj_t gc_obj; + uint32 type_index; + struct { + uint32 type_index; + uint32 length; + } array_new_default; + /* pointer to a memory space holding more data, current usage: + * struct.new init value: WASMStructNewInitValues * + * array.new init value: WASMArrayNewInitValues * + */ + void *data; +#endif +} WASMValue; +#endif /* end of WASM_VALUE_DEFINED */ + +typedef struct WASMStructNewInitValues { + uint32 type_idx; + uint32 count; + WASMValue fields[1]; +} WASMStructNewInitValues; + +typedef struct WASMArrayNewInitValues { + uint32 type_idx; + uint32 length; + WASMValue elem_data[1]; +} WASMArrayNewInitValues; + +typedef struct InitializerExpression { + /* type of INIT_EXPR_TYPE_XXX, which is an instruction of + constant expression */ + uint8 init_expr_type; + union { + struct { + WASMValue v; + } unary; + struct { + struct InitializerExpression *l_expr; + struct InitializerExpression *r_expr; + } binary; + } u; +} InitializerExpression; + +static inline bool +is_expr_binary_op(uint8 flag) +{ + return flag == INIT_EXPR_TYPE_I32_ADD || flag == INIT_EXPR_TYPE_I32_SUB + || flag == INIT_EXPR_TYPE_I32_MUL || flag == INIT_EXPR_TYPE_I64_ADD + || flag == INIT_EXPR_TYPE_I64_SUB || flag == INIT_EXPR_TYPE_I64_MUL; +} + +/* check if table or data offset is valid for i32 offset */ +static inline bool +is_valid_i32_offset(uint8 flag) +{ + return flag == INIT_EXPR_TYPE_I32_CONST || flag == INIT_EXPR_TYPE_I32_ADD + || flag == INIT_EXPR_TYPE_I32_SUB || flag == INIT_EXPR_TYPE_I32_MUL; +} + +/* check if table or data offset is valid for i64 offset */ +static inline bool +is_valid_i64_offset(uint8 flag) +{ + return flag == INIT_EXPR_TYPE_I64_CONST || flag == INIT_EXPR_TYPE_I64_ADD + || flag == INIT_EXPR_TYPE_I64_SUB || flag == INIT_EXPR_TYPE_I64_MUL; +} + +#if WASM_ENABLE_GC != 0 +/** + * Reference type of (ref null ht) or (ref ht), + * and heap type is defined type (type i), i >= 0 + */ +typedef struct RefHeapType_TypeIdx { + /* ref_type is REF_TYPE_HT_NULLABLE or + REF_TYPE_HT_NON_NULLABLE, (0x63 or 0x64) */ + uint8 ref_type; + /* true if ref_type is REF_TYPE_HT_NULLABLE */ + bool nullable; + /* heap type is defined type: type_index >= 0 */ + int32 type_idx; +} RefHeapType_TypeIdx; + +/** + * Reference type of (ref null ht) or (ref ht), + * and heap type is non-defined type + */ +typedef struct RefHeapType_Common { + /* ref_type is REF_TYPE_HT_NULLABLE or + REF_TYPE_HT_NON_NULLABLE (0x63 or 0x64) */ + uint8 ref_type; + /* true if ref_type is REF_TYPE_HT_NULLABLE */ + bool nullable; + /* Common heap type (not defined type): + -0x10 (func), -0x11 (extern), -0x12 (any), -0x13 (eq), + -0x16 (i31), -0x17 (nofunc), -0x18 (noextern), + -0x19 (struct), -0x20 (array), -0x21 (none) */ + int32 heap_type; +} RefHeapType_Common; + +/** + * Reference type + */ +typedef union WASMRefType { + uint8 ref_type; + RefHeapType_TypeIdx ref_ht_typeidx; + RefHeapType_Common ref_ht_common; +} WASMRefType; + +typedef struct WASMRefTypeMap { + /** + * The type index of a type array, which only stores + * the first byte of the type, e.g. WASMFuncType.types, + * WASMStructType.fields + */ + uint16 index; + /* The full type info if the type cannot be described + with one byte */ + WASMRefType *ref_type; +} WASMRefTypeMap; +#endif /* end of WASM_ENABLE_GC */ + +#if WASM_ENABLE_GC == 0 +typedef struct WASMFuncType WASMType; +typedef WASMType *WASMTypePtr; +#else +/** + * Common type, store the same fields of + * WASMFuncType, WASMStructType and WASMArrayType + */ +typedef struct WASMType { + /** + * type_flag must be WASM_TYPE_FUNC/STRUCT/ARRAY to + * denote that it is a WASMFuncType, WASMStructType or + * WASMArrayType + */ + uint16 type_flag; + + bool is_sub_final; + /* How many types are referring to this type */ + uint16 ref_count; + /* The inheritance depth */ + uint16 inherit_depth; + /* The root type */ + struct WASMType *root_type; + /* The parent type */ + struct WASMType *parent_type; + uint32 parent_type_idx; + + /* The number of internal types in the current rec group, and if + the type is not in a recursive group, rec_count is 1 since a + single type definition is reinterpreted as a short-hand for a + recursive group containing just one type */ + uint16 rec_count; + uint16 rec_idx; + /* The index of the begin type of this group */ + uint32 rec_begin_type_idx; +} WASMType, *WASMTypePtr; +#endif /* end of WASM_ENABLE_GC */ + +/* Function type */ +typedef struct WASMFuncType { +#if WASM_ENABLE_GC != 0 + WASMType base_type; +#endif + + uint16 param_count; + uint16 result_count; + uint16 param_cell_num; + uint16 ret_cell_num; + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + /* Code block to call llvm jit functions of this + kind of function type from fast jit jitted code */ + void *call_to_llvm_jit_from_fast_jit; +#endif + +#if WASM_ENABLE_GC != 0 + uint16 ref_type_map_count; + WASMRefTypeMap *ref_type_maps; + WASMRefTypeMap *result_ref_type_maps; +#else + uint16 ref_count; +#endif + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + /* Quick AOT/JIT entry of this func type */ + void *quick_aot_entry; +#endif + + /* types of params and results, only store the first byte + * of the type, if it cannot be described with one byte, + * then the full type info is stored in ref_type_maps */ + uint8 types[1]; +} WASMFuncType; + +#if WASM_ENABLE_GC != 0 +typedef struct WASMStructFieldType { + uint16 field_flags; + uint8 field_type; + uint8 field_size; + uint32 field_offset; +#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 + /* + * The field size and field offset of a wasm struct may vary + * in 32-bit target and 64-bit target, e.g., the size of a + * GC reference is 4 bytes in the former and 8 bytes in the + * latter, the AOT compiler needs to use the correct field + * offset according to the target info. + */ + uint8 field_size_64bit; + uint8 field_size_32bit; + uint32 field_offset_64bit; + uint32 field_offset_32bit; +#endif +} WASMStructFieldType; + +typedef struct WASMStructType { + WASMType base_type; + + /* total size of this struct object */ + uint32 total_size; + uint16 field_count; + + uint16 ref_type_map_count; + WASMRefTypeMap *ref_type_maps; + + /* Offsets of reference fields that need to be traced during GC. + The first element of the table is the number of such offsets. */ + uint16 *reference_table; + + /* Field info, note that fields[i]->field_type only stores + * the first byte of the field type, if it cannot be described + * with one byte, then the full field type info is stored in + * ref_type_maps */ + WASMStructFieldType fields[1]; +} WASMStructType; + +typedef struct WASMArrayType { + WASMType base_type; + + uint16 elem_flags; + uint8 elem_type; + /* The full elem type info if the elem type cannot be + described with one byte */ + WASMRefType *elem_ref_type; +} WASMArrayType; + +#if WASM_ENABLE_STRINGREF != 0 +/* stringref representation, we define it as a void * pointer here, the + * stringref implementation can use any structure */ +/* + WasmGC heap + +-----------------------+ + | | + | stringref | + | +----------+ | external string representation + | | host_ptr |--------o------+----->+------------+ + | +----------+ | | | | + | | | +------------+ + | stringview_wtf8/16 | | + | +----------+ | | + | | host_ptr |--------o------+ + | +----------+ | | + | | | + | stringview_iter | | + | +----------+ | | + | | host_ptr |--------o------+ + | +----------+ | + | | pos | | + | +----------+ | + | | + +-----------------------+ +*/ +typedef void *WASMString; + +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ +#endif /* end of WASM_ENABLE_GC != 0 */ + +typedef struct WASMTableType { + uint8 elem_type; + /** + * 0: no max size and not shared + * 1: has max size + * 2: shared + * 4: table64 + */ + uint8 flags; + bool possible_grow; + uint32 init_size; + /* specified if (flags & 1), else it is 0x10000 */ + uint32 max_size; +#if WASM_ENABLE_GC != 0 + WASMRefType *elem_ref_type; +#endif +} WASMTableType; + +typedef struct WASMTable { + WASMTableType table_type; +#if WASM_ENABLE_GC != 0 + /* init expr for the whole table */ + InitializerExpression init_expr; +#endif +} WASMTable; + +#if WASM_ENABLE_MEMORY64 != 0 +typedef uint64 mem_offset_t; +#define PR_MEM_OFFSET PRIu64 +#else +typedef uint32 mem_offset_t; +#define PR_MEM_OFFSET PRIu32 +#endif +typedef mem_offset_t tbl_elem_idx_t; + +typedef struct WASMMemory { + uint32 flags; + uint32 num_bytes_per_page; + uint32 init_page_count; + uint32 max_page_count; +} WASMMemory; +#ifndef WASM_MEMORY_T_DEFINED +#define WASM_MEMORY_T_DEFINED +typedef struct WASMMemory WASMMemoryType; +#endif + +typedef struct WASMTableImport { + char *module_name; + char *field_name; + WASMTableType table_type; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *import_module; + WASMTable *import_table_linked; +#endif +} WASMTableImport; + +typedef struct WASMMemoryImport { + char *module_name; + char *field_name; + WASMMemoryType mem_type; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *import_module; + WASMMemory *import_memory_linked; +#endif +} WASMMemoryImport; + +typedef struct WASMFunctionImport { + char *module_name; + char *field_name; + /* function type */ + WASMFuncType *func_type; + /* native function pointer after linked */ + void *func_ptr_linked; + /* signature from registered native symbols */ + const char *signature; + /* attachment */ + void *attachment; +#if WASM_ENABLE_GC != 0 + /* the type index of this function's func_type */ + uint32 type_idx; +#endif +#if WASM_ENABLE_INVOKE_NATIVE != 0 + bool call_conv_raw; +#endif + bool call_conv_wasm_c_api; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *import_module; + WASMFunction *import_func_linked; +#endif +} WASMFunctionImport; + +#if WASM_ENABLE_TAGS != 0 +typedef struct WASMTagImport { + char *module_name; + char *field_name; + uint8 attribute; /* the type of the tag (numerical) */ + uint32 type; /* the type of the catch function (numerical)*/ + WASMFuncType *tag_type; + void *tag_ptr_linked; + +#if WASM_ENABLE_MULTI_MODULE != 0 + /* imported tag pointer after linked */ + WASMModule *import_module; + WASMTag *import_tag_linked; + uint32 import_tag_index_linked; +#endif +} WASMTagImport; +#endif + +typedef struct WASMGlobalType { + uint8 val_type; + bool is_mutable; +} WASMGlobalType; + +typedef struct WASMGlobalImport { + char *module_name; + char *field_name; + WASMGlobalType type; + bool is_linked; + /* global data after linked */ + WASMValue global_data_linked; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type; +#endif +#if WASM_ENABLE_MULTI_MODULE != 0 + /* imported function pointer after linked */ + /* TODO: remove if not needed */ + WASMModule *import_module; + WASMGlobal *import_global_linked; +#endif +#if WASM_ENABLE_FAST_JIT != 0 + /* The data offset of current global in global data */ + uint32 data_offset; +#endif +} WASMGlobalImport; + +typedef struct WASMImport { + uint8 kind; + union { + WASMFunctionImport function; + WASMTableImport table; + WASMMemoryImport memory; +#if WASM_ENABLE_TAGS != 0 + WASMTagImport tag; +#endif + WASMGlobalImport global; + struct { + char *module_name; + char *field_name; + } names; + } u; +} WASMImport; + +struct WASMFunction { +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 + char *field_name; +#endif + /* the type of function */ + WASMFuncType *func_type; + uint32 local_count; + uint8 *local_types; +#if WASM_ENABLE_GC != 0 + uint16 local_ref_type_map_count; + WASMRefTypeMap *local_ref_type_maps; +#endif + + /* cell num of parameters */ + uint16 param_cell_num; + /* cell num of return type */ + uint16 ret_cell_num; + /* cell num of local variables */ + uint16 local_cell_num; + /* offset of each local, including function parameters + and local variables */ + uint16 *local_offsets; + + uint32 max_stack_cell_num; + uint32 max_block_num; + uint32 code_size; + uint8 *code; +#if WASM_ENABLE_FAST_INTERP != 0 + uint32 code_compiled_size; + uint8 *code_compiled; + uint8 *consts; + uint32 const_cell_num; +#endif + +#if WASM_ENABLE_GC != 0 + /* the type index of this function's func_type */ + uint32 type_idx; +#endif + +#if WASM_ENABLE_EXCE_HANDLING != 0 + uint32 exception_handler_count; +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 + /* Whether function has opcode memory.grow */ + bool has_op_memory_grow; + /* Whether function has opcode call or call_indirect */ + bool has_op_func_call; +#endif +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + /* Whether function has memory operation opcodes */ + bool has_memory_operations; + /* Whether function has opcode call_indirect */ + bool has_op_call_indirect; + /* Whether function has opcode set_global_aux_stack */ + bool has_op_set_global_aux_stack; +#endif + +#if WASM_ENABLE_FAST_JIT != 0 + /* The compiled fast jit jitted code block of this function */ + void *fast_jit_jitted_code; +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + /* The compiled llvm jit func ptr of this function */ + void *llvm_jit_func_ptr; + /* Code block to call fast jit jitted code of this function + from the llvm jit jitted code */ + void *call_to_fast_jit_from_llvm_jit; +#endif +#endif + +#if WASM_ENABLE_BRANCH_HINTS != 0 + uint8 *code_body_begin; +#endif +}; + +#if WASM_ENABLE_TAGS != 0 +struct WASMTag { + uint8 attribute; /* the attribute property of the tag (expected to be 0) */ + uint32 type; /* the type of the tag (expected valid inden in type table) */ + WASMFuncType *tag_type; +}; +#endif + +#if WASM_ENABLE_BRANCH_HINTS != 0 +enum WASMCompilationHintType { + DUMMY = 0, + WASM_COMPILATION_BRANCH_HINT = 0, +}; +struct WASMCompilationHint { + struct WASMCompilationHint *next; + enum WASMCompilationHintType type; +}; +struct WASMCompilationHintBranchHint { + struct WASMCompilationHint *next; + enum WASMCompilationHintType type; + uint32 offset; + bool is_likely; +}; +#endif + +struct WASMGlobal { + WASMGlobalType type; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type; +#endif + InitializerExpression init_expr; +#if WASM_ENABLE_FAST_JIT != 0 + /* The data offset of current global in global data */ + uint32 data_offset; +#endif +}; + +typedef struct WASMExport { + char *name; + uint8 kind; + uint32 index; +} WASMExport; + +typedef struct WASMTableSeg { + /* 0 to 7 */ + uint32 mode; + /* funcref or externref, elemkind will be considered as funcref */ + uint32 elem_type; +#if WASM_ENABLE_GC != 0 + WASMRefType *elem_ref_type; +#endif + /* optional, only for active */ + uint32 table_index; + InitializerExpression base_offset; + uint32 value_count; + InitializerExpression *init_values; +} WASMTableSeg; + +typedef struct WASMDataSeg { + uint32 memory_index; + InitializerExpression base_offset; + uint32 data_length; +#if WASM_ENABLE_BULK_MEMORY != 0 + bool is_passive; +#endif + uint8 *data; + bool is_data_cloned; +} WASMDataSeg; + +typedef struct BlockAddr { + const uint8 *start_addr; + uint8 *else_addr; + uint8 *end_addr; +} BlockAddr; + +#if WASM_ENABLE_LIBC_WASI != 0 +typedef struct WASIArguments { + const char **dir_list; + uint32 dir_count; + const char **map_dir_list; + uint32 map_dir_count; + const char **env; + uint32 env_count; + /* in CIDR notation */ + const char **addr_pool; + uint32 addr_count; + const char **ns_lookup_pool; + uint32 ns_lookup_count; + char **argv; + uint32 argc; + os_raw_file_handle stdio[3]; +} WASIArguments; +#endif + +typedef struct StringNode { + struct StringNode *next; + char *str; +} StringNode, *StringList; + +typedef struct BrTableCache { + struct BrTableCache *next; + /* Address of br_table opcode */ + uint8 *br_table_op_addr; + uint32 br_count; + uint32 br_depths[1]; +} BrTableCache; + +#if WASM_ENABLE_DEBUG_INTERP != 0 +typedef struct WASMFastOPCodeNode { + struct WASMFastOPCodeNode *next; + uint64 offset; + uint8 orig_op; +} WASMFastOPCodeNode; +#endif + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 +typedef struct WASMCustomSection { + struct WASMCustomSection *next; + /* Start address of the section name */ + char *name_addr; + /* Length of the section name decoded from leb */ + uint32 name_len; + /* Start address of the content (name len and name skipped) */ + uint8 *content_addr; + uint32 content_len; +} WASMCustomSection; +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 +struct AOTCompData; +struct AOTCompContext; + +/* Orc JIT thread arguments */ +typedef struct OrcJitThreadArg { +#if WASM_ENABLE_JIT != 0 + struct AOTCompContext *comp_ctx; +#endif + struct WASMModule *module; + uint32 group_idx; +} OrcJitThreadArg; +#endif + +struct WASMModuleInstance; + +struct WASMModule { + /* Module type, for module loaded from WASM bytecode binary, + this field is Wasm_Module_Bytecode; + for module loaded from AOT file, this field is + Wasm_Module_AoT, and this structure should be treated as + AOTModule structure. */ + uint32 module_type; + + /* the package version read from the WASM file */ + uint32 package_version; + + uint32 type_count; + uint32 import_count; + uint32 function_count; + uint32 table_count; + uint32 memory_count; +#if WASM_ENABLE_TAGS != 0 + uint32 tag_count; +#endif + uint32 global_count; + uint32 export_count; + uint32 table_seg_count; + /* data seg count read from data segment section */ + uint32 data_seg_count; +#if WASM_ENABLE_BULK_MEMORY != 0 + /* data count read from datacount section */ + uint32 data_seg_count1; +#endif +#if WASM_ENABLE_GC != 0 +#if WASM_ENABLE_STRINGREF != 0 + uint32 string_literal_count; + uint32 *string_literal_lengths; + const uint8 **string_literal_ptrs; +#endif +#endif + + uint32 import_function_count; + uint32 import_table_count; + uint32 import_memory_count; +#if WASM_ENABLE_TAGS != 0 + uint32 import_tag_count; +#endif + uint32 import_global_count; + + WASMImport *import_functions; + WASMImport *import_tables; + WASMImport *import_memories; +#if WASM_ENABLE_TAGS != 0 + WASMImport *import_tags; +#endif + WASMImport *import_globals; + + WASMType **types; + WASMImport *imports; + WASMFunction **functions; + WASMTable *tables; + WASMMemory *memories; +#if WASM_ENABLE_TAGS != 0 + WASMTag **tags; +#endif + WASMGlobal *globals; + WASMExport *exports; + WASMTableSeg *table_segments; + WASMDataSeg **data_segments; + uint32 start_function; + + /* total global variable size */ + uint32 global_data_size; + + /* the index of auxiliary __data_end global, + -1 means unexported */ + uint32 aux_data_end_global_index; + /* auxiliary __data_end exported by wasm app */ + uint64 aux_data_end; + + /* the index of auxiliary __heap_base global, + -1 means unexported */ + uint32 aux_heap_base_global_index; + /* auxiliary __heap_base exported by wasm app */ + uint64 aux_heap_base; + + /* the index of auxiliary stack top global, + -1 means unexported */ + uint32 aux_stack_top_global_index; + /* auxiliary stack bottom resolved */ + uint64 aux_stack_bottom; + /* auxiliary stack size resolved */ + uint32 aux_stack_size; + + /* the index of malloc/free function, + -1 means unexported */ + uint32 malloc_function; + uint32 free_function; + + /* the index of __retain function, + -1 means unexported */ + uint32 retain_function; + + /* Whether there is possible memory grow, e.g. memory.grow opcode */ + bool possible_memory_grow; + + StringList const_str_list; +#if WASM_ENABLE_FAST_INTERP == 0 + bh_list br_table_cache_list_head; + bh_list *br_table_cache_list; +#endif + +#if WASM_ENABLE_LIBC_WASI != 0 + WASIArguments wasi_args; + bool import_wasi_api; +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + /* TODO: add mutex for mutli-thread? */ + bh_list import_module_list_head; + bh_list *import_module_list; +#endif + +#if WASM_ENABLE_GC != 0 + /* Ref types hash set */ + HashMap *ref_type_set; + struct WASMRttType **rtt_types; + korp_mutex rtt_type_lock; +#if WASM_ENABLE_STRINGREF != 0 + /* special rtts for stringref types + - stringref + - stringview_wtf8 + - stringview_wtf16 + - stringview_iter + */ + struct WASMRttType *stringref_rtts[4]; +#endif +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 || WASM_ENABLE_DEBUG_AOT != 0 + bh_list fast_opcode_list; + uint8 *buf_code; + uint64 buf_code_size; +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 || WASM_ENABLE_FAST_JIT != 0 \ + || WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 + uint8 *load_addr; + uint64 load_size; +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 \ + || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0) + /** + * List of instances referred to this module. When source debugging + * feature is enabled, the debugger may modify the code section of + * the module, so we need to report a warning if user create several + * instances based on the same module. + * + * Also add the instance to the list for Fast JIT to LLVM JIT + * tier-up, since we need to lazily update the LLVM func pointers + * in the instance. + */ + struct WASMModuleInstance *instance_list; + korp_mutex instance_list_lock; +#endif + +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 + const uint8 *name_section_buf; + const uint8 *name_section_buf_end; +#endif + +#if WASM_ENABLE_BRANCH_HINTS != 0 + struct WASMCompilationHint **function_hints; +#endif + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 + WASMCustomSection *custom_section_list; +#endif + +#if WASM_ENABLE_FAST_JIT != 0 + /** + * func pointers of Fast JITed (un-imported) functions + * for non Multi-Tier JIT mode: + * (1) when lazy jit is disabled, each pointer is set to the compiled + * fast jit jitted code + * (2) when lazy jit is enabled, each pointer is firstly inited as + * jit_global->compile_fast_jit_and_then_call, and then set to the + * compiled fast jit jitted code when it is called (the stub will + * compile the jit function and then update itself) + * for Multi-Tier JIT mode: + * each pointer is firstly inited as compile_fast_jit_and_then_call, + * and then set to the compiled fast jit jitted code when it is called, + * and when the llvm jit func ptr of the same function is compiled, it + * will be set to call_to_llvm_jit_from_fast_jit of this function type + * (tier-up from fast-jit to llvm-jit) + */ + void **fast_jit_func_ptrs; + /* locks for Fast JIT lazy compilation */ + korp_mutex fast_jit_thread_locks[WASM_ORC_JIT_BACKEND_THREAD_NUM]; + bool fast_jit_thread_locks_inited[WASM_ORC_JIT_BACKEND_THREAD_NUM]; +#endif + +#if WASM_ENABLE_JIT != 0 + struct AOTCompData *comp_data; + struct AOTCompContext *comp_ctx; + /** + * func pointers of LLVM JITed (un-imported) functions + * for non Multi-Tier JIT mode: + * each pointer is set to the looked up llvm jit func ptr, note that it + * is a stub and will trigger the actual compilation when it is called + * for Multi-Tier JIT mode: + * each pointer is inited as call_to_fast_jit code block, when the llvm + * jit func ptr is actually compiled, it is set to the compiled llvm jit + * func ptr + */ + void **func_ptrs; + /* whether the func pointers are compiled */ + bool *func_ptrs_compiled; +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 + /* backend compilation threads */ + korp_tid orcjit_threads[WASM_ORC_JIT_BACKEND_THREAD_NUM]; + /* backend thread arguments */ + OrcJitThreadArg orcjit_thread_args[WASM_ORC_JIT_BACKEND_THREAD_NUM]; + /* whether to stop the compilation of backend threads */ + bool orcjit_stop_compiling; +#endif + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + /* wait lock/cond for the synchronization of + the llvm jit initialization */ + korp_mutex tierup_wait_lock; + korp_cond tierup_wait_cond; + bool tierup_wait_lock_inited; + korp_tid llvm_jit_init_thread; + /* whether the llvm jit is initialized */ + bool llvm_jit_inited; + /* Whether to enable llvm jit compilation: + it is set to true only when there is a module instance starts to + run with running mode Mode_LLVM_JIT or Mode_Multi_Tier_JIT, + since no need to enable llvm jit compilation for Mode_Interp and + Mode_Fast_JIT, so as to improve performance for them */ + bool enable_llvm_jit_compilation; + /* The count of groups which finish compiling the fast jit + functions in that group */ + uint32 fast_jit_ready_groups; +#endif + +#if WASM_ENABLE_WAMR_COMPILER != 0 + bool is_simd_used; + bool is_ref_types_used; + bool is_bulk_memory_used; +#endif + + /* user defined name */ + char *name; + + /* Whether the underlying wasm binary buffer can be freed */ + bool is_binary_freeable; +}; + +typedef struct BlockType { + /* Block type may be expressed in one of two forms: + * either by the type of the single return value or + * by a type index of module. + */ + union { + struct { + uint8 type; +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap ref_type_map; +#endif + } value_type; + WASMFuncType *type; + } u; + bool is_value_type; +} BlockType; + +typedef struct WASMBranchBlock { + uint8 *begin_addr; + uint8 *target_addr; + uint32 *frame_sp; + uint32 cell_num; +#if WASM_ENABLE_EXCE_HANDLING != 0 + /* in exception handling, label_type needs to be stored to lookup exception + * handlers */ + uint8 label_type; +#endif +} WASMBranchBlock; + +/** + * Align an unsigned value on a alignment boundary. + * + * @param v the value to be aligned + * @param b the alignment boundary (2, 4, 8, ...) + * + * @return the aligned value + */ +inline static unsigned +align_uint(unsigned v, unsigned b) +{ + unsigned m = b - 1; + return (v + m) & ~m; +} + +/** + * Align an 64 bit unsigned value on a alignment boundary. + * + * @param v the value to be aligned + * @param b the alignment boundary (2, 4, 8, ...) + * + * @return the aligned value + */ +inline static uint64 +align_uint64(uint64 v, uint64 b) +{ + uint64 m = b - 1; + return (v + m) & ~m; +} + +/** + * Check whether a piece of data is out of range + * + * @param offset the offset that the data starts + * @param len the length of the data + * @param max_size the maximum size of the data range + * + * @return true if out of range, false otherwise + */ +inline static bool +offset_len_out_of_bounds(uint32 offset, uint32 len, uint32 max_size) +{ + if (offset + len < offset /* integer overflow */ + || offset + len > max_size) + return true; + return false; +} + +/** + * Return the hash value of c string. + */ +inline static uint32 +wasm_string_hash(const char *str) +{ + unsigned h = (unsigned)strlen(str); + const uint8 *p = (uint8 *)str; + const uint8 *end = p + h; + + while (p != end) + h = ((h << 5) - h) + *p++; + return h; +} + +/** + * Whether two c strings are equal. + */ +inline static bool +wasm_string_equal(const char *s1, const char *s2) +{ + return strcmp(s1, s2) == 0 ? true : false; +} + +/** + * Return the byte size of value type with specific pointer size. + * + * Note: Please use wasm_value_type_size for interpreter, only aot compiler + * can use this API directly to calculate type size for different target + */ +inline static uint32 +wasm_value_type_size_internal(uint8 value_type, uint8 pointer_size) +{ + if (value_type == VALUE_TYPE_VOID) + return 0; + else if (value_type == VALUE_TYPE_I32 || value_type == VALUE_TYPE_F32 + || value_type == VALUE_TYPE_ANY) + return sizeof(int32); + else if (value_type == VALUE_TYPE_I64 || value_type == VALUE_TYPE_F64) + return sizeof(int64); +#if WASM_ENABLE_SIMD != 0 + else if (value_type == VALUE_TYPE_V128) + return sizeof(int64) * 2; +#endif +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + else if (value_type == VALUE_TYPE_FUNCREF + || value_type == VALUE_TYPE_EXTERNREF) + return sizeof(uint32); +#elif WASM_ENABLE_GC != 0 + else if ((value_type >= (uint8)REF_TYPE_ARRAYREF /* 0x6A */ + && value_type <= (uint8)REF_TYPE_NULLFUNCREF) /* 0x73 */ + || (value_type >= (uint8)REF_TYPE_HT_NULLABLE /* 0x63 */ + && value_type <= (uint8)REF_TYPE_HT_NON_NULLABLE) /* 0x64 */ +#if WASM_ENABLE_STRINGREF != 0 + || (value_type >= (uint8)REF_TYPE_STRINGVIEWWTF8 /* 0x66 */ + && value_type <= (uint8)REF_TYPE_STRINGREF) /* 0x67 */ + || (value_type >= (uint8)REF_TYPE_STRINGVIEWITER /* 0x61 */ + && value_type <= (uint8)REF_TYPE_STRINGVIEWWTF16) /* 0x62 */ +#endif + ) + return pointer_size; + else if (value_type == PACKED_TYPE_I8) + return sizeof(int8); + else if (value_type == PACKED_TYPE_I16) + return sizeof(int16); +#endif + else { + bh_assert(0 && "Unknown value type. It should be handled ahead."); + } +#if WASM_ENABLE_GC == 0 + (void)pointer_size; +#endif + return 0; +} + +/** + * Return the cell num of value type with specific pointer size. + * + * Note: Please use wasm_value_type_cell_num for interpreter, only aot compiler + * can use this API directly to calculate type cell num for different target + */ +inline static uint16 +wasm_value_type_cell_num_internal(uint8 value_type, uint8 pointer_size) +{ + return wasm_value_type_size_internal(value_type, pointer_size) / 4; +} + +/** + * Return the byte size of value type. + */ +inline static uint32 +wasm_value_type_size(uint8 value_type) +{ + return wasm_value_type_size_internal(value_type, sizeof(uintptr_t)); +} + +inline static uint16 +wasm_value_type_cell_num(uint8 value_type) +{ + return wasm_value_type_size(value_type) / 4; +} + +inline static uint32 +wasm_get_cell_num(const uint8 *types, uint32 type_count) +{ + uint32 cell_num = 0; + uint32 i; + for (i = 0; i < type_count; i++) + cell_num += wasm_value_type_cell_num(types[i]); + return cell_num; +} + +#if WASM_ENABLE_REF_TYPES != 0 +inline static uint16 +wasm_value_type_cell_num_outside(uint8 value_type) +{ + if (VALUE_TYPE_EXTERNREF == value_type) { + return sizeof(uintptr_t) / sizeof(uint32); + } + else { + return wasm_value_type_cell_num(value_type); + } +} +#endif + +#if WASM_ENABLE_GC == 0 +inline static bool +wasm_type_equal(const WASMType *type1, const WASMType *type2, + const WASMTypePtr *types, uint32 type_count) +{ + const WASMFuncType *func_type1 = (const WASMFuncType *)type1; + const WASMFuncType *func_type2 = (const WASMFuncType *)type2; + + if (type1 == type2) { + return true; + } + + return (func_type1->param_count == func_type2->param_count + && func_type1->result_count == func_type2->result_count + && memcmp( + func_type1->types, func_type2->types, + (uint32)(func_type1->param_count + func_type1->result_count)) + == 0) + ? true + : false; + (void)types; + (void)type_count; +} +#else +/* implemented in gc_type.c */ +bool +wasm_type_equal(const WASMType *type1, const WASMType *type2, + const WASMTypePtr *types, uint32 type_count); +#endif + +inline static uint32 +wasm_get_smallest_type_idx(const WASMTypePtr *types, uint32 type_count, + uint32 cur_type_idx) +{ + uint32 i; + + for (i = 0; i < cur_type_idx; i++) { + if (wasm_type_equal(types[cur_type_idx], types[i], types, type_count)) + return i; + } + return cur_type_idx; +} + +#if WASM_ENABLE_GC == 0 +static inline uint32 +block_type_get_param_types(BlockType *block_type, uint8 **p_param_types) +#else +static inline uint32 +block_type_get_param_types(BlockType *block_type, uint8 **p_param_types, + WASMRefTypeMap **p_param_reftype_maps, + uint32 *p_param_reftype_map_count) +#endif +{ + uint32 param_count = 0; + if (!block_type->is_value_type) { + WASMFuncType *func_type = block_type->u.type; + *p_param_types = func_type->types; + param_count = func_type->param_count; +#if WASM_ENABLE_GC != 0 + *p_param_reftype_maps = func_type->ref_type_maps; + *p_param_reftype_map_count = (uint32)(func_type->result_ref_type_maps + - func_type->ref_type_maps); +#endif + } + else { + *p_param_types = NULL; + param_count = 0; +#if WASM_ENABLE_GC != 0 + *p_param_reftype_maps = NULL; + *p_param_reftype_map_count = 0; +#endif + } + + return param_count; +} + +#if WASM_ENABLE_GC == 0 +static inline uint32 +block_type_get_result_types(BlockType *block_type, uint8 **p_result_types) +#else +static inline uint32 +block_type_get_result_types(BlockType *block_type, uint8 **p_result_types, + WASMRefTypeMap **p_result_reftype_maps, + uint32 *p_result_reftype_map_count) +#endif +{ + uint32 result_count = 0; + uint8 *result_types = NULL; +#if WASM_ENABLE_GC != 0 + uint8 type; + uint32 result_reftype_map_count = 0; + WASMRefTypeMap *result_reftype_maps = NULL; +#endif + + if (block_type->is_value_type) { + if (block_type->u.value_type.type != VALUE_TYPE_VOID) { + result_types = &block_type->u.value_type.type; + result_count = 1; +#if WASM_ENABLE_GC != 0 + type = block_type->u.value_type.type; + if (type == (uint8)REF_TYPE_HT_NULLABLE + || type == (uint8)REF_TYPE_HT_NON_NULLABLE) { + result_reftype_maps = &block_type->u.value_type.ref_type_map; + result_reftype_map_count = 1; + } +#endif + } + } + else { + WASMFuncType *func_type = block_type->u.type; + result_types = func_type->types + func_type->param_count; + result_count = func_type->result_count; +#if WASM_ENABLE_GC != 0 + result_reftype_maps = func_type->result_ref_type_maps; + result_reftype_map_count = (uint32)(func_type->ref_type_map_count + - (func_type->result_ref_type_maps + - func_type->ref_type_maps)); +#endif + } + *p_result_types = result_types; +#if WASM_ENABLE_GC != 0 + *p_result_reftype_maps = result_reftype_maps; + *p_result_reftype_map_count = result_reftype_map_count; +#endif + return result_count; +} + +static inline uint32 +block_type_get_arity(const BlockType *block_type, uint8 label_type) +{ + if (label_type == LABEL_TYPE_LOOP) { + if (block_type->is_value_type) + return 0; + else + return block_type->u.type->param_count; + } + else { + if (block_type->is_value_type) { + return block_type->u.value_type.type != VALUE_TYPE_VOID ? 1 : 0; + } + else + return block_type->u.type->result_count; + } + return 0; +} + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* end of _WASM_H_ */ diff --git a/wamr/include/wasm_c_api.h b/wamr/include/wasm_c_api.h new file mode 100644 index 0000000..241a0ee --- /dev/null +++ b/wamr/include/wasm_c_api.h @@ -0,0 +1,910 @@ +// WebAssembly C API + +/** + * @file wasm_c_api.h + * + * @brief This file defines the WebAssembly C APIs + */ + +#ifndef _WASM_C_API_H_ +#define _WASM_C_API_H_ + +#include +#include +#include +#include +#include + +#ifndef WASM_API_EXTERN +#if defined(_MSC_BUILD) +#if defined(COMPILING_WASM_RUNTIME_API) +#define WASM_API_EXTERN __declspec(dllexport) +#else +#define WASM_API_EXTERN __declspec(dllimport) +#endif +#else +#define WASM_API_EXTERN +#endif +#endif + +#if defined(__GNUC__) || defined(__clang__) +#define WASM_API_DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) +#define WASM_API_DEPRECATED __declspec(deprecated) +#else +#pragma message("WARNING: You need to implement DEPRECATED for this compiler") +#define WASM_API_DEPRECATED +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* clang-format off */ + +/////////////////////////////////////////////////////////////////////////////// +// Auxiliaries + +// Machine types +#if defined(__STDC_VERSION__) && (__STDC_VERSION__) > 199901L +inline void assertions(void) { + static_assert(sizeof(float) == sizeof(uint32_t), "incompatible float type"); + static_assert(sizeof(double) == sizeof(uint64_t), "incompatible double type"); + static_assert(sizeof(intptr_t) == sizeof(uint32_t) || + sizeof(intptr_t) == sizeof(uint64_t), + "incompatible pointer type"); +} +#endif + +typedef char byte_t; +typedef float float32_t; +typedef double float64_t; + + +// Ownership + +#define own + +// The qualifier `own` is used to indicate ownership of data in this API. +// It is intended to be interpreted similar to a `const` qualifier: +// +// - `own wasm_xxx_t*` owns the pointed-to data +// - `own wasm_xxx_t` distributes to all fields of a struct or union `xxx` +// - `own wasm_xxx_vec_t` owns the vector as well as its elements(!) +// - an `own` function parameter passes ownership from caller to callee +// - an `own` function result passes ownership from callee to caller +// - an exception are `own` pointer parameters named `out`, which are copy-back +// output parameters passing back ownership from callee to caller +// +// Own data is created by `wasm_xxx_new` functions and some others. +// It must be released with the corresponding `wasm_xxx_delete` function. +// +// Deleting a reference does not necessarily delete the underlying object, +// it merely indicates that this owner no longer uses it. +// +// For vectors, `const wasm_xxx_vec_t` is used informally to indicate that +// neither the vector nor its elements should be modified. +// TODO: introduce proper `wasm_xxx_const_vec_t`? + + +#define WASM_DECLARE_OWN(name) \ + typedef struct wasm_##name##_t wasm_##name##_t; \ + \ + WASM_API_EXTERN void wasm_##name##_delete(own wasm_##name##_t*); + + +// Vectors +// size: capacity +// num_elems: current number of elements +// size_of_elem: size of one element +#define WASM_DECLARE_VEC(name, ptr_or_none) \ + typedef struct wasm_##name##_vec_t { \ + size_t size; \ + wasm_##name##_t ptr_or_none* data; \ + size_t num_elems; \ + size_t size_of_elem; \ + void *lock; \ + } wasm_##name##_vec_t; \ + \ + WASM_API_EXTERN void wasm_##name##_vec_new_empty(own wasm_##name##_vec_t* out); \ + WASM_API_EXTERN void wasm_##name##_vec_new_uninitialized( \ + own wasm_##name##_vec_t* out, size_t); \ + WASM_API_EXTERN void wasm_##name##_vec_new( \ + own wasm_##name##_vec_t* out, \ + size_t, own wasm_##name##_t ptr_or_none const[]); \ + WASM_API_EXTERN void wasm_##name##_vec_copy( \ + own wasm_##name##_vec_t* out, const wasm_##name##_vec_t*); \ + WASM_API_EXTERN void wasm_##name##_vec_delete(own wasm_##name##_vec_t*); + + +// Byte vectors + +typedef byte_t wasm_byte_t; +WASM_DECLARE_VEC(byte, ) + +typedef wasm_byte_vec_t wasm_name_t; + +#define wasm_name wasm_byte_vec +#define wasm_name_new wasm_byte_vec_new +#define wasm_name_new_empty wasm_byte_vec_new_empty +#define wasm_name_new_new_uninitialized wasm_byte_vec_new_uninitialized +#define wasm_name_copy wasm_byte_vec_copy +#define wasm_name_delete wasm_byte_vec_delete + +static inline void wasm_name_new_from_string( + own wasm_name_t* out, const char* s +) { + wasm_name_new(out, strlen(s), s); +} + +static inline void wasm_name_new_from_string_nt( + own wasm_name_t* out, const char* s +) { + wasm_name_new(out, strlen(s) + 1, s); +} + + +/////////////////////////////////////////////////////////////////////////////// +// Runtime Environment + +// Configuration + +WASM_DECLARE_OWN(config) + +#ifndef MEM_ALLOC_OPTION_DEFINED +#define MEM_ALLOC_OPTION_DEFINED +/* same definition from wasm_export.h */ +/* Memory allocator type */ +typedef enum { + /* pool mode, allocate memory from user defined heap buffer */ + Alloc_With_Pool = 0, + /* user allocator mode, allocate memory from user defined + malloc function */ + Alloc_With_Allocator, + /* system allocator mode, allocate memory from system allocator, + or, platform's os_malloc function */ + Alloc_With_System_Allocator, +} mem_alloc_type_t; + +/* Memory allocator option */ +typedef union MemAllocOption { + struct { + void *heap_buf; + uint32_t heap_size; + } pool; + struct { + void *malloc_func; + void *realloc_func; + void *free_func; + /* allocator user data, only used when + WASM_MEM_ALLOC_WITH_USER_DATA is defined */ + void *user_data; + } allocator; +} MemAllocOption; +#endif /* MEM_ALLOC_OPTION_DEFINED */ + +/* Runtime configuration */ +struct wasm_config_t { + mem_alloc_type_t mem_alloc_type; + MemAllocOption mem_alloc_option; + uint32_t segue_flags; + bool enable_linux_perf; + /*TODO: wasi args*/ +}; + +#ifndef INSTANTIATION_ARGS_OPTION_DEFINED +#define INSTANTIATION_ARGS_OPTION_DEFINED +/* WASM module instantiation arguments */ +typedef struct InstantiationArgs { + uint32_t default_stack_size; + uint32_t host_managed_heap_size; + uint32_t max_memory_pages; +} InstantiationArgs; +#endif /* INSTANTIATION_ARGS_OPTION_DEFINED */ + +/* + * by default: + * - mem_alloc_type is Alloc_With_System_Allocator + * - mem_alloc_option is all 0 + * - enable_linux_perf is false + */ +WASM_API_EXTERN own wasm_config_t* wasm_config_new(void); + +// Embedders may provide custom functions for manipulating configs. +WASM_API_EXTERN own wasm_config_t* +wasm_config_set_mem_alloc_opt(wasm_config_t *, mem_alloc_type_t, MemAllocOption *); + +WASM_API_EXTERN own wasm_config_t* +wasm_config_set_linux_perf_opt(wasm_config_t *, bool); + +/** + * Enable using GS register as the base address of linear memory in linux x86_64, + * which may speedup the linear memory access for LLVM AOT/JIT: + * bit0 to bit4 denotes i32.load, i64.load, f32.load, f64.load, v128.load + * bit8 to bit12 denotes i32.store, i64.store, f32.store, f64.store, v128.store + * For example, 0x01 enables i32.load, 0x0100 enables i32.store. + * To enable all load/store operations, use 0x1F1F + */ +WASM_API_EXTERN wasm_config_t* +wasm_config_set_segue_flags(wasm_config_t *config, uint32_t segue_flags); + +// Engine + +WASM_DECLARE_OWN(engine) + +/** + * Create a new engine + * + * Note: for the engine new/delete operations, including this, + * wasm_engine_new_with_config, wasm_engine_new_with_args, and + * wasm_engine_delete, if the platform has mutex initializer, + * then they are thread-safe: we use a global lock to lock the + * operations of the engine. Otherwise they are not thread-safe: + * when there are engine new/delete operations happening + * simultaneously in multiple threads, developer must create + * the lock by himself, and add the lock when calling these + * functions. + */ +WASM_API_EXTERN own wasm_engine_t* wasm_engine_new(void); +WASM_API_EXTERN own wasm_engine_t* wasm_engine_new_with_config(wasm_config_t*); +WASM_API_DEPRECATED WASM_API_EXTERN own wasm_engine_t * +wasm_engine_new_with_args(mem_alloc_type_t type, const MemAllocOption *opts); + +// Store + +WASM_DECLARE_OWN(store) + +WASM_API_EXTERN own wasm_store_t* wasm_store_new(wasm_engine_t*); + + +/////////////////////////////////////////////////////////////////////////////// +// Type Representations + +// Type attributes + +typedef uint8_t wasm_mutability_t; +enum wasm_mutability_enum { + WASM_CONST, + WASM_VAR, +}; + +typedef struct wasm_limits_t { + uint32_t min; + uint32_t max; +} wasm_limits_t; + +static const uint32_t wasm_limits_max_default = 0xffffffff; + + +// Generic + +#define WASM_DECLARE_TYPE(name) \ + WASM_DECLARE_OWN(name) \ + WASM_DECLARE_VEC(name, *) \ + \ + WASM_API_EXTERN own wasm_##name##_t* wasm_##name##_copy(const wasm_##name##_t*); + + +// Value Types + +WASM_DECLARE_TYPE(valtype) + +#ifndef WASM_VALKIND_T_DEFINED +#define WASM_VALKIND_T_DEFINED +typedef uint8_t wasm_valkind_t; +enum wasm_valkind_enum { + WASM_I32, + WASM_I64, + WASM_F32, + WASM_F64, + WASM_V128, + WASM_EXTERNREF = 128, + WASM_FUNCREF, +}; +#endif + +WASM_API_EXTERN own wasm_valtype_t* wasm_valtype_new(wasm_valkind_t); + +WASM_API_EXTERN wasm_valkind_t wasm_valtype_kind(const wasm_valtype_t*); + +static inline bool wasm_valkind_is_num(wasm_valkind_t k) { + return k < WASM_EXTERNREF; +} +static inline bool wasm_valkind_is_ref(wasm_valkind_t k) { + return k >= WASM_EXTERNREF; +} + +static inline bool wasm_valtype_is_num(const wasm_valtype_t* t) { + return wasm_valkind_is_num(wasm_valtype_kind(t)); +} +static inline bool wasm_valtype_is_ref(const wasm_valtype_t* t) { + return wasm_valkind_is_ref(wasm_valtype_kind(t)); +} + + +// Function Types + +WASM_DECLARE_TYPE(functype) + +WASM_API_EXTERN own wasm_functype_t* wasm_functype_new( + own wasm_valtype_vec_t* params, own wasm_valtype_vec_t* results); + +WASM_API_EXTERN const wasm_valtype_vec_t* wasm_functype_params(const wasm_functype_t*); +WASM_API_EXTERN const wasm_valtype_vec_t* wasm_functype_results(const wasm_functype_t*); + + +// Global Types + +WASM_DECLARE_TYPE(globaltype) + +WASM_API_EXTERN own wasm_globaltype_t* wasm_globaltype_new( + own wasm_valtype_t*, wasm_mutability_t); + +WASM_API_EXTERN const wasm_valtype_t* wasm_globaltype_content(const wasm_globaltype_t*); +WASM_API_EXTERN wasm_mutability_t wasm_globaltype_mutability(const wasm_globaltype_t*); + + +// Table Types + +WASM_DECLARE_TYPE(tabletype) + +WASM_API_EXTERN own wasm_tabletype_t* wasm_tabletype_new( + own wasm_valtype_t*, const wasm_limits_t*); + +WASM_API_EXTERN const wasm_valtype_t* wasm_tabletype_element(const wasm_tabletype_t*); +WASM_API_EXTERN const wasm_limits_t* wasm_tabletype_limits(const wasm_tabletype_t*); + + +// Memory Types + +WASM_DECLARE_TYPE(memorytype) + +WASM_API_EXTERN own wasm_memorytype_t* wasm_memorytype_new(const wasm_limits_t*); + +WASM_API_EXTERN const wasm_limits_t* wasm_memorytype_limits(const wasm_memorytype_t*); + + +// Extern Types + +WASM_DECLARE_TYPE(externtype) + +typedef uint8_t wasm_externkind_t; +enum wasm_externkind_enum { + WASM_EXTERN_FUNC, + WASM_EXTERN_GLOBAL, + WASM_EXTERN_TABLE, + WASM_EXTERN_MEMORY, +}; + +WASM_API_EXTERN wasm_externkind_t wasm_externtype_kind(const wasm_externtype_t*); + +WASM_API_EXTERN wasm_externtype_t* wasm_functype_as_externtype(wasm_functype_t*); +WASM_API_EXTERN wasm_externtype_t* wasm_globaltype_as_externtype(wasm_globaltype_t*); +WASM_API_EXTERN wasm_externtype_t* wasm_tabletype_as_externtype(wasm_tabletype_t*); +WASM_API_EXTERN wasm_externtype_t* wasm_memorytype_as_externtype(wasm_memorytype_t*); + +WASM_API_EXTERN wasm_functype_t* wasm_externtype_as_functype(wasm_externtype_t*); +WASM_API_EXTERN wasm_globaltype_t* wasm_externtype_as_globaltype(wasm_externtype_t*); +WASM_API_EXTERN wasm_tabletype_t* wasm_externtype_as_tabletype(wasm_externtype_t*); +WASM_API_EXTERN wasm_memorytype_t* wasm_externtype_as_memorytype(wasm_externtype_t*); + +WASM_API_EXTERN const wasm_externtype_t* wasm_functype_as_externtype_const(const wasm_functype_t*); +WASM_API_EXTERN const wasm_externtype_t* wasm_globaltype_as_externtype_const(const wasm_globaltype_t*); +WASM_API_EXTERN const wasm_externtype_t* wasm_tabletype_as_externtype_const(const wasm_tabletype_t*); +WASM_API_EXTERN const wasm_externtype_t* wasm_memorytype_as_externtype_const(const wasm_memorytype_t*); + +WASM_API_EXTERN const wasm_functype_t* wasm_externtype_as_functype_const(const wasm_externtype_t*); +WASM_API_EXTERN const wasm_globaltype_t* wasm_externtype_as_globaltype_const(const wasm_externtype_t*); +WASM_API_EXTERN const wasm_tabletype_t* wasm_externtype_as_tabletype_const(const wasm_externtype_t*); +WASM_API_EXTERN const wasm_memorytype_t* wasm_externtype_as_memorytype_const(const wasm_externtype_t*); + + +// Import Types + +WASM_DECLARE_TYPE(importtype) + +WASM_API_EXTERN own wasm_importtype_t* wasm_importtype_new( + own wasm_name_t* module, own wasm_name_t* name, own wasm_externtype_t*); + +WASM_API_EXTERN const wasm_name_t* wasm_importtype_module(const wasm_importtype_t*); +WASM_API_EXTERN const wasm_name_t* wasm_importtype_name(const wasm_importtype_t*); +WASM_API_EXTERN const wasm_externtype_t* wasm_importtype_type(const wasm_importtype_t*); +WASM_API_EXTERN bool wasm_importtype_is_linked(const wasm_importtype_t*); + + +// Export Types + +WASM_DECLARE_TYPE(exporttype) + +WASM_API_EXTERN own wasm_exporttype_t* wasm_exporttype_new( + own wasm_name_t*, own wasm_externtype_t*); + +WASM_API_EXTERN const wasm_name_t* wasm_exporttype_name(const wasm_exporttype_t*); +WASM_API_EXTERN const wasm_externtype_t* wasm_exporttype_type(const wasm_exporttype_t*); + + +/////////////////////////////////////////////////////////////////////////////// +// Runtime Objects + +// Values + +#ifndef WASM_VAL_T_DEFINED +#define WASM_VAL_T_DEFINED +struct wasm_ref_t; + +typedef struct wasm_val_t { + wasm_valkind_t kind; + uint8_t _paddings[7]; + union { + int32_t i32; + int64_t i64; + float32_t f32; + float64_t f64; + struct wasm_ref_t* ref; + } of; +} wasm_val_t; +#endif + +WASM_API_EXTERN void wasm_val_delete(own wasm_val_t* v); +WASM_API_EXTERN void wasm_val_copy(own wasm_val_t* out, const wasm_val_t*); + +WASM_DECLARE_VEC(val, ) + + +// References + +#define WASM_DECLARE_REF_BASE(name) \ + WASM_DECLARE_OWN(name) \ + \ + WASM_API_EXTERN own wasm_##name##_t* wasm_##name##_copy(const wasm_##name##_t*); \ + WASM_API_EXTERN bool wasm_##name##_same(const wasm_##name##_t*, const wasm_##name##_t*); \ + \ + WASM_API_EXTERN void* wasm_##name##_get_host_info(const wasm_##name##_t*); \ + WASM_API_EXTERN void wasm_##name##_set_host_info(wasm_##name##_t*, void*); \ + WASM_API_EXTERN void wasm_##name##_set_host_info_with_finalizer( \ + wasm_##name##_t*, void*, void (*)(void*)); + +#define WASM_DECLARE_REF(name) \ + WASM_DECLARE_REF_BASE(name) \ + \ + WASM_API_EXTERN wasm_ref_t* wasm_##name##_as_ref(wasm_##name##_t*); \ + WASM_API_EXTERN wasm_##name##_t* wasm_ref_as_##name(wasm_ref_t*); \ + WASM_API_EXTERN const wasm_ref_t* wasm_##name##_as_ref_const(const wasm_##name##_t*); \ + WASM_API_EXTERN const wasm_##name##_t* wasm_ref_as_##name##_const(const wasm_ref_t*); + +#define WASM_DECLARE_SHARABLE_REF(name) \ + WASM_DECLARE_REF(name) \ + WASM_DECLARE_OWN(shared_##name) \ + \ + WASM_API_EXTERN own wasm_shared_##name##_t* wasm_##name##_share(const wasm_##name##_t*); \ + WASM_API_EXTERN own wasm_##name##_t* wasm_##name##_obtain(wasm_store_t*, const wasm_shared_##name##_t*); + + +WASM_DECLARE_REF_BASE(ref) + + +// Frames + +WASM_DECLARE_OWN(frame) +WASM_DECLARE_VEC(frame, *) +WASM_API_EXTERN own wasm_frame_t* wasm_frame_copy(const wasm_frame_t*); + +WASM_API_EXTERN struct wasm_instance_t* wasm_frame_instance(const wasm_frame_t*); +WASM_API_EXTERN uint32_t wasm_frame_func_index(const wasm_frame_t*); +WASM_API_EXTERN size_t wasm_frame_func_offset(const wasm_frame_t*); +WASM_API_EXTERN size_t wasm_frame_module_offset(const wasm_frame_t*); + + +// Traps + +typedef wasm_name_t wasm_message_t; // null terminated + +WASM_DECLARE_REF(trap) + +WASM_API_EXTERN own wasm_trap_t* wasm_trap_new(wasm_store_t* store, const wasm_message_t*); + +WASM_API_EXTERN void wasm_trap_message(const wasm_trap_t*, own wasm_message_t* out); +WASM_API_EXTERN own wasm_frame_t* wasm_trap_origin(const wasm_trap_t*); +WASM_API_EXTERN void wasm_trap_trace(const wasm_trap_t*, own wasm_frame_vec_t* out); + + +// Foreign Objects + +WASM_DECLARE_REF(foreign) + +WASM_API_EXTERN own wasm_foreign_t* wasm_foreign_new(wasm_store_t*); + + +// Modules +// WASM_DECLARE_SHARABLE_REF(module) + +#ifndef WASM_MODULE_T_DEFINED +#define WASM_MODULE_T_DEFINED +struct WASMModuleCommon; +typedef struct WASMModuleCommon *wasm_module_t; +#endif + +#ifndef LOAD_ARGS_OPTION_DEFINED +#define LOAD_ARGS_OPTION_DEFINED +typedef struct LoadArgs { + char *name; + /* True by default, used by wasm-c-api only. + If false, the wasm input buffer (wasm_byte_vec_t) is referenced by the + module instead of being cloned. Hence, it can be freed after module loading. */ + bool clone_wasm_binary; + /* This option is only used by the AOT/wasm loader (see wasm_export.h) */ + bool wasm_binary_freeable; + /* false by default, if true, don't resolve the symbols yet. The + wasm_runtime_load_ex has to be followed by a wasm_runtime_resolve_symbols + call */ + bool no_resolve; + /* TODO: more fields? */ +} LoadArgs; +#endif /* LOAD_ARGS_OPTION_DEFINED */ + +WASM_API_EXTERN own wasm_module_t* wasm_module_new( + wasm_store_t*, const wasm_byte_vec_t* binary); + +// please refer to wasm_runtime_load_ex(...) in core/iwasm/include/wasm_export.h +WASM_API_EXTERN own wasm_module_t* wasm_module_new_ex( + wasm_store_t*, wasm_byte_vec_t* binary, LoadArgs *args); + +WASM_API_EXTERN void wasm_module_delete(own wasm_module_t*); + +WASM_API_EXTERN bool wasm_module_validate(wasm_store_t*, const wasm_byte_vec_t* binary); + +WASM_API_EXTERN void wasm_module_imports(const wasm_module_t*, own wasm_importtype_vec_t* out); +WASM_API_EXTERN void wasm_module_exports(const wasm_module_t*, own wasm_exporttype_vec_t* out); + +WASM_API_EXTERN void wasm_module_serialize(wasm_module_t*, own wasm_byte_vec_t* out); +WASM_API_EXTERN own wasm_module_t* wasm_module_deserialize(wasm_store_t*, const wasm_byte_vec_t*); + +typedef wasm_module_t wasm_shared_module_t; +WASM_API_EXTERN own wasm_shared_module_t* wasm_module_share(wasm_module_t*); +WASM_API_EXTERN own wasm_module_t* wasm_module_obtain(wasm_store_t*, wasm_shared_module_t*); +WASM_API_EXTERN void wasm_shared_module_delete(own wasm_shared_module_t*); + +WASM_API_EXTERN bool wasm_module_set_name(wasm_module_t*, const char* name); +WASM_API_EXTERN const char *wasm_module_get_name(wasm_module_t*); + +WASM_API_EXTERN bool wasm_module_is_underlying_binary_freeable(const wasm_module_t *module); + + +// Function Instances + +WASM_DECLARE_REF(func) + +typedef own wasm_trap_t* (*wasm_func_callback_t)( + const wasm_val_vec_t* args, own wasm_val_vec_t *results); +typedef own wasm_trap_t* (*wasm_func_callback_with_env_t)( + void* env, const wasm_val_vec_t *args, wasm_val_vec_t *results); + +WASM_API_EXTERN own wasm_func_t* wasm_func_new( + wasm_store_t*, const wasm_functype_t*, wasm_func_callback_t); +WASM_API_EXTERN own wasm_func_t* wasm_func_new_with_env( + wasm_store_t*, const wasm_functype_t* type, wasm_func_callback_with_env_t, + void* env, void (*finalizer)(void*)); + +WASM_API_EXTERN own wasm_functype_t* wasm_func_type(const wasm_func_t*); +WASM_API_EXTERN size_t wasm_func_param_arity(const wasm_func_t*); +WASM_API_EXTERN size_t wasm_func_result_arity(const wasm_func_t*); + +WASM_API_EXTERN own wasm_trap_t* wasm_func_call( + const wasm_func_t*, const wasm_val_vec_t* args, wasm_val_vec_t* results); + + +// Global Instances + +WASM_DECLARE_REF(global) + +WASM_API_EXTERN own wasm_global_t* wasm_global_new( + wasm_store_t*, const wasm_globaltype_t*, const wasm_val_t*); + +WASM_API_EXTERN own wasm_globaltype_t* wasm_global_type(const wasm_global_t*); + +WASM_API_EXTERN void wasm_global_get(const wasm_global_t*, own wasm_val_t* out); +WASM_API_EXTERN void wasm_global_set(wasm_global_t*, const wasm_val_t*); + + +// Table Instances + +WASM_DECLARE_REF(table) + +typedef uint32_t wasm_table_size_t; + +WASM_API_EXTERN own wasm_table_t* wasm_table_new( + wasm_store_t*, const wasm_tabletype_t*, wasm_ref_t* init); + +WASM_API_EXTERN own wasm_tabletype_t* wasm_table_type(const wasm_table_t*); + +WASM_API_EXTERN own wasm_ref_t* wasm_table_get(const wasm_table_t*, wasm_table_size_t index); +WASM_API_EXTERN bool wasm_table_set(wasm_table_t*, wasm_table_size_t index, wasm_ref_t*); + +WASM_API_EXTERN wasm_table_size_t wasm_table_size(const wasm_table_t*); +WASM_API_EXTERN bool wasm_table_grow(wasm_table_t*, wasm_table_size_t delta, wasm_ref_t* init); + + +// Memory Instances + +WASM_DECLARE_REF(memory) + +typedef uint32_t wasm_memory_pages_t; + +static const size_t MEMORY_PAGE_SIZE = 0x10000; + +WASM_API_EXTERN own wasm_memory_t* wasm_memory_new(wasm_store_t*, const wasm_memorytype_t*); + +WASM_API_EXTERN own wasm_memorytype_t* wasm_memory_type(const wasm_memory_t*); + +WASM_API_EXTERN byte_t* wasm_memory_data(wasm_memory_t*); +WASM_API_EXTERN size_t wasm_memory_data_size(const wasm_memory_t*); + +WASM_API_EXTERN wasm_memory_pages_t wasm_memory_size(const wasm_memory_t*); +WASM_API_EXTERN bool wasm_memory_grow(wasm_memory_t*, wasm_memory_pages_t delta); + + +// Externals + +WASM_DECLARE_REF(extern) +WASM_DECLARE_VEC(extern, *) + +WASM_API_EXTERN wasm_externkind_t wasm_extern_kind(const wasm_extern_t*); +WASM_API_EXTERN own wasm_externtype_t* wasm_extern_type(const wasm_extern_t*); + +WASM_API_EXTERN wasm_extern_t* wasm_func_as_extern(wasm_func_t*); +WASM_API_EXTERN wasm_extern_t* wasm_global_as_extern(wasm_global_t*); +WASM_API_EXTERN wasm_extern_t* wasm_table_as_extern(wasm_table_t*); +WASM_API_EXTERN wasm_extern_t* wasm_memory_as_extern(wasm_memory_t*); + +WASM_API_EXTERN wasm_func_t* wasm_extern_as_func(wasm_extern_t*); +WASM_API_EXTERN wasm_global_t* wasm_extern_as_global(wasm_extern_t*); +WASM_API_EXTERN wasm_table_t* wasm_extern_as_table(wasm_extern_t*); +WASM_API_EXTERN wasm_memory_t* wasm_extern_as_memory(wasm_extern_t*); + +WASM_API_EXTERN const wasm_extern_t* wasm_func_as_extern_const(const wasm_func_t*); +WASM_API_EXTERN const wasm_extern_t* wasm_global_as_extern_const(const wasm_global_t*); +WASM_API_EXTERN const wasm_extern_t* wasm_table_as_extern_const(const wasm_table_t*); +WASM_API_EXTERN const wasm_extern_t* wasm_memory_as_extern_const(const wasm_memory_t*); + +WASM_API_EXTERN const wasm_func_t* wasm_extern_as_func_const(const wasm_extern_t*); +WASM_API_EXTERN const wasm_global_t* wasm_extern_as_global_const(const wasm_extern_t*); +WASM_API_EXTERN const wasm_table_t* wasm_extern_as_table_const(const wasm_extern_t*); +WASM_API_EXTERN const wasm_memory_t* wasm_extern_as_memory_const(const wasm_extern_t*); + + +// Module Instances + +WASM_DECLARE_REF(instance) + +WASM_API_EXTERN own wasm_instance_t* wasm_instance_new( + wasm_store_t*, const wasm_module_t*, const wasm_extern_vec_t *imports, + own wasm_trap_t** trap +); + +// please refer to wasm_runtime_instantiate(...) in core/iwasm/include/wasm_export.h +WASM_API_EXTERN own wasm_instance_t* wasm_instance_new_with_args( + wasm_store_t*, const wasm_module_t*, const wasm_extern_vec_t *imports, + own wasm_trap_t** trap, const uint32_t stack_size, const uint32_t heap_size +); + +// please refer to wasm_runtime_instantiate_ex(...) in core/iwasm/include/wasm_export.h +WASM_API_EXTERN own wasm_instance_t* wasm_instance_new_with_args_ex( + wasm_store_t*, const wasm_module_t*, const wasm_extern_vec_t *imports, + own wasm_trap_t** trap, const InstantiationArgs *inst_args +); + +WASM_API_EXTERN void wasm_instance_exports(const wasm_instance_t*, own wasm_extern_vec_t* out); + +// Return total wasm functions' execution time in ms +WASM_API_EXTERN double wasm_instance_sum_wasm_exec_time(const wasm_instance_t*); +// Return execution time in ms of a given wasm function with +// func_name. If the function is not found, return 0. +WASM_API_EXTERN double wasm_instance_get_wasm_func_exec_time(const wasm_instance_t*, const char *); + +/////////////////////////////////////////////////////////////////////////////// +// Convenience + +// Vectors + +#define WASM_EMPTY_VEC {0, NULL, 0, 0, NULL} +#define WASM_ARRAY_VEC(array) {sizeof(array)/sizeof(*(array)), array, sizeof(array)/sizeof(*(array)), sizeof(*(array)), NULL} + + +// Value Type construction short-hands + +static inline own wasm_valtype_t* wasm_valtype_new_i32(void) { + return wasm_valtype_new(WASM_I32); +} +static inline own wasm_valtype_t* wasm_valtype_new_i64(void) { + return wasm_valtype_new(WASM_I64); +} +static inline own wasm_valtype_t* wasm_valtype_new_f32(void) { + return wasm_valtype_new(WASM_F32); +} +static inline own wasm_valtype_t* wasm_valtype_new_f64(void) { + return wasm_valtype_new(WASM_F64); +} +static inline own wasm_valtype_t* wasm_valtype_new_v128(void) { + return wasm_valtype_new(WASM_V128); +} + +static inline own wasm_valtype_t* wasm_valtype_new_anyref(void) { + return wasm_valtype_new(WASM_EXTERNREF); +} +static inline own wasm_valtype_t* wasm_valtype_new_funcref(void) { + return wasm_valtype_new(WASM_FUNCREF); +} + + +// Function Types construction short-hands + +static inline own wasm_functype_t* wasm_functype_new_0_0(void) { + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new_empty(¶ms); + wasm_valtype_vec_new_empty(&results); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_1_0( + own wasm_valtype_t* p +) { + wasm_valtype_t* ps[1] = {p}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 1, ps); + wasm_valtype_vec_new_empty(&results); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_2_0( + own wasm_valtype_t* p1, own wasm_valtype_t* p2 +) { + wasm_valtype_t* ps[2] = {p1, p2}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 2, ps); + wasm_valtype_vec_new_empty(&results); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_3_0( + own wasm_valtype_t* p1, own wasm_valtype_t* p2, own wasm_valtype_t* p3 +) { + wasm_valtype_t* ps[3] = {p1, p2, p3}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 3, ps); + wasm_valtype_vec_new_empty(&results); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_0_1( + own wasm_valtype_t* r +) { + wasm_valtype_t* rs[1] = {r}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new_empty(¶ms); + wasm_valtype_vec_new(&results, 1, rs); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_1_1( + own wasm_valtype_t* p, own wasm_valtype_t* r +) { + wasm_valtype_t* ps[1] = {p}; + wasm_valtype_t* rs[1] = {r}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 1, ps); + wasm_valtype_vec_new(&results, 1, rs); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_2_1( + own wasm_valtype_t* p1, own wasm_valtype_t* p2, own wasm_valtype_t* r +) { + wasm_valtype_t* ps[2] = {p1, p2}; + wasm_valtype_t* rs[1] = {r}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 2, ps); + wasm_valtype_vec_new(&results, 1, rs); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_3_1( + own wasm_valtype_t* p1, own wasm_valtype_t* p2, own wasm_valtype_t* p3, + own wasm_valtype_t* r +) { + wasm_valtype_t* ps[3] = {p1, p2, p3}; + wasm_valtype_t* rs[1] = {r}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 3, ps); + wasm_valtype_vec_new(&results, 1, rs); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_0_2( + own wasm_valtype_t* r1, own wasm_valtype_t* r2 +) { + wasm_valtype_t* rs[2] = {r1, r2}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new_empty(¶ms); + wasm_valtype_vec_new(&results, 2, rs); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_1_2( + own wasm_valtype_t* p, own wasm_valtype_t* r1, own wasm_valtype_t* r2 +) { + wasm_valtype_t* ps[1] = {p}; + wasm_valtype_t* rs[2] = {r1, r2}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 1, ps); + wasm_valtype_vec_new(&results, 2, rs); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_2_2( + own wasm_valtype_t* p1, own wasm_valtype_t* p2, + own wasm_valtype_t* r1, own wasm_valtype_t* r2 +) { + wasm_valtype_t* ps[2] = {p1, p2}; + wasm_valtype_t* rs[2] = {r1, r2}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 2, ps); + wasm_valtype_vec_new(&results, 2, rs); + return wasm_functype_new(¶ms, &results); +} + +static inline own wasm_functype_t* wasm_functype_new_3_2( + own wasm_valtype_t* p1, own wasm_valtype_t* p2, own wasm_valtype_t* p3, + own wasm_valtype_t* r1, own wasm_valtype_t* r2 +) { + wasm_valtype_t* ps[3] = {p1, p2, p3}; + wasm_valtype_t* rs[2] = {r1, r2}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 3, ps); + wasm_valtype_vec_new(&results, 2, rs); + return wasm_functype_new(¶ms, &results); +} + + +// Value construction short-hands + +static inline void wasm_val_init_ptr(own wasm_val_t* out, void* p) { +#if UINTPTR_MAX == UINT32_MAX + out->kind = WASM_I32; + out->of.i32 = (intptr_t)p; +#elif UINTPTR_MAX == UINT64_MAX + out->kind = WASM_I64; + out->of.i64 = (intptr_t)p; +#endif +} + +static inline void* wasm_val_ptr(const wasm_val_t* val) { +#if UINTPTR_MAX == UINT32_MAX + return (void*)(intptr_t)val->of.i32; +#elif UINTPTR_MAX == UINT64_MAX + return (void*)(intptr_t)val->of.i64; +#endif +} + +#define WASM_I32_VAL(i) {.kind = WASM_I32, ._paddings = {0}, .of = {.i32 = i}} +#define WASM_I64_VAL(i) {.kind = WASM_I64, ._paddings = {0}, .of = {.i64 = i}} +#define WASM_F32_VAL(z) {.kind = WASM_F32, ._paddings = {0}, .of = {.f32 = z}} +#define WASM_F64_VAL(z) {.kind = WASM_F64, ._paddings = {0}, .of = {.f64 = z}} +#define WASM_REF_VAL(r) {.kind = WASM_EXTERNREF, ._paddings = {0}, .of = {.ref = r}} +#define WASM_INIT_VAL {.kind = WASM_EXTERNREF, ._paddings = {0}, .of = {.ref = NULL}} + +#define KILOBYTE(n) ((n) * 1024) + +// Create placeholders filled in `wasm_externvec_t* imports` for `wasm_instance_new()` +WASM_API_EXTERN wasm_extern_t *wasm_extern_new_empty(wasm_store_t *, wasm_externkind_t); + +/////////////////////////////////////////////////////////////////////////////// + +#undef own + +/* clang-format on */ + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // #ifdef _WASM_C_API_H_ diff --git a/wamr/include/wasm_c_api_internal.h b/wamr/include/wasm_c_api_internal.h new file mode 100644 index 0000000..cab3741 --- /dev/null +++ b/wamr/include/wasm_c_api_internal.h @@ -0,0 +1,246 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_C_API_INTERNAL_H +#define _WASM_C_API_INTERNAL_H + +#include "wasm_c_api.h" // ../include/wasm_c_api.h +#include "wasm_runtime_common.h" // wasm_runtime_common.h + +#ifndef own +#define own +#endif + +/* Vectors */ +/* we will malloc resource for the vector's data field */ +/* we will release resource of data */ +/* caller needs to take care resource for the vector itself */ +#define DEFAULT_VECTOR_INIT_LENGTH (64) + +WASM_DECLARE_VEC(instance, *) +WASM_DECLARE_VEC(module, *) +WASM_DECLARE_VEC(store, *) + +/* Runtime Environment */ +struct wasm_engine_t { + uint32 ref_count; + /* list of wasm_module_ex_t */ + Vector modules; + /* list of stores which are classified according to tids */ + Vector stores_by_tid; +}; + +struct wasm_store_t { + /* maybe should remove the list */ + wasm_module_vec_t *modules; + wasm_instance_vec_t *instances; + Vector *foreigns; +}; + +/* Type Representations */ +struct wasm_valtype_t { + wasm_valkind_t kind; +}; + +struct wasm_functype_t { + uint32 extern_kind; + /* gona to new and delete own */ + wasm_valtype_vec_t *params; + wasm_valtype_vec_t *results; +}; + +struct wasm_globaltype_t { + uint32 extern_kind; + /* gona to new and delete own */ + wasm_valtype_t *val_type; + wasm_mutability_t mutability; +}; + +struct wasm_tabletype_t { + uint32 extern_kind; + wasm_valtype_t *val_type; + wasm_limits_t limits; +}; + +struct wasm_memorytype_t { + uint32 extern_kind; + wasm_limits_t limits; +}; + +struct wasm_externtype_t { + uint32 extern_kind; + /* reserved space */ + uint8 data[1]; +}; + +struct wasm_importtype_t { + wasm_name_t *module_name; + wasm_name_t *name; + wasm_externtype_t *extern_type; +}; + +struct wasm_exporttype_t { + wasm_name_t *name; + wasm_externtype_t *extern_type; +}; + +/* Runtime Objects */ +enum wasm_reference_kind { + WASM_REF_foreign, + WASM_REF_func, + WASM_REF_global, + WASM_REF_memory, + WASM_REF_table, +}; + +struct wasm_host_info { + void *info; + void (*finalizer)(void *); +}; + +struct wasm_ref_t { + wasm_store_t *store; + enum wasm_reference_kind kind; + struct wasm_host_info host_info; + uint32 ref_idx_rt; + WASMModuleInstanceCommon *inst_comm_rt; +}; + +struct wasm_trap_t { + wasm_byte_vec_t *message; + Vector *frames; +}; + +struct wasm_foreign_t { + wasm_store_t *store; + enum wasm_reference_kind kind; + struct wasm_host_info host_info; + int32 ref_cnt; + uint32 foreign_idx_rt; + WASMModuleInstanceCommon *inst_comm_rt; +}; + +struct wasm_func_t { + wasm_store_t *store; + wasm_name_t *module_name; + wasm_name_t *name; + uint16 kind; + + struct wasm_host_info host_info; + wasm_functype_t *type; + uint16 param_count; + uint16 result_count; + + bool with_env; + union { + wasm_func_callback_t cb; + struct callback_ext { + void *env; + wasm_func_callback_with_env_t cb; + void (*finalizer)(void *); + } cb_env; + } u; + /* + * an index in both functions runtime instance lists + * of interpreter mode and aot mode + */ + uint16 func_idx_rt; + WASMModuleInstanceCommon *inst_comm_rt; + WASMFunctionInstanceCommon *func_comm_rt; +}; + +struct wasm_global_t { + wasm_store_t *store; + wasm_name_t *module_name; + wasm_name_t *name; + uint16 kind; + + struct wasm_host_info host_info; + wasm_globaltype_t *type; + wasm_val_t *init; + /* + * an index in both global runtime instance lists + * of interpreter mode and aot mode + */ + uint16 global_idx_rt; + WASMModuleInstanceCommon *inst_comm_rt; +}; + +struct wasm_memory_t { + wasm_store_t *store; + wasm_name_t *module_name; + wasm_name_t *name; + uint16 kind; + + struct wasm_host_info host_info; + wasm_memorytype_t *type; + /* + * an index in both memory runtime instance lists + * of interpreter mode and aot mode + */ + uint16 memory_idx_rt; + WASMModuleInstanceCommon *inst_comm_rt; +}; + +struct wasm_table_t { + wasm_store_t *store; + wasm_name_t *module_name; + wasm_name_t *name; + uint16 kind; + + struct wasm_host_info host_info; + wasm_tabletype_t *type; + /* + * an index in both table runtime instance lists + * of interpreter mode and aot mode + */ + uint16 table_idx_rt; + WASMModuleInstanceCommon *inst_comm_rt; +}; + +struct wasm_extern_t { + wasm_store_t *store; + wasm_name_t *module_name; + wasm_name_t *name; + wasm_externkind_t kind; + /* reserved space */ + uint8 data[1]; +}; + +struct wasm_instance_t { + wasm_store_t *store; + wasm_extern_vec_t *exports; + struct wasm_host_info host_info; + WASMModuleInstanceCommon *inst_comm_rt; +}; + +wasm_ref_t * +wasm_ref_new_internal(wasm_store_t *store, enum wasm_reference_kind kind, + uint32 obj_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt); + +wasm_foreign_t * +wasm_foreign_new_internal(wasm_store_t *store, uint32 foreign_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt); + +wasm_func_t * +wasm_func_new_internal(wasm_store_t *store, uint16 func_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt); + +wasm_global_t * +wasm_global_new_internal(wasm_store_t *store, uint16 global_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt); + +wasm_memory_t * +wasm_memory_new_internal(wasm_store_t *store, uint16 memory_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt); + +wasm_table_t * +wasm_table_new_internal(wasm_store_t *store, uint16 table_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt); + +void +wasm_frame_vec_clone_internal(Vector *src, Vector *out); +#endif /* _WASM_C_API_INTERNAL_H */ diff --git a/wamr/include/wasm_exec_env.h b/wamr/include/wasm_exec_env.h new file mode 100644 index 0000000..b49e409 --- /dev/null +++ b/wamr/include/wasm_exec_env.h @@ -0,0 +1,328 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_EXEC_ENV_H +#define _WASM_EXEC_ENV_H + +#include "bh_assert.h" // bh_assert.h +#include "wasm_suspend_flags.h" // wasm_suspend_flags.h +#if WASM_ENABLE_INTERP != 0 +#include "wasm.h" // ../interpreter/wasm.h +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +struct WASMModuleInstanceCommon; +struct WASMInterpFrame; + +#if WASM_ENABLE_THREAD_MGR != 0 +typedef struct WASMCluster WASMCluster; +#if WASM_ENABLE_DEBUG_INTERP != 0 +typedef struct WASMCurrentEnvStatus WASMCurrentEnvStatus; +#endif +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK +typedef struct WASMJmpBuf { + struct WASMJmpBuf *prev; + korp_jmpbuf jmpbuf; +} WASMJmpBuf; +#endif + +/* Execution environment */ +typedef struct WASMExecEnv { + /* Next thread's exec env of a WASM module instance. */ + struct WASMExecEnv *next; + + /* Current interpreter/AOT frame of current thread */ + struct WASMInterpFrame *cur_frame; + + /* Note: field module_inst, argv_buf, native_stack_boundary, + suspend_flags, aux_stack_boundary, aux_stack_bottom, and + native_symbol are used by AOTed code, don't change the + places of them */ + + /* The WASM module instance of current thread */ + struct WASMModuleInstanceCommon *module_inst; + +#if WASM_ENABLE_AOT != 0 + uint32 *argv_buf; +#endif + + /* The boundary of native stack. When runtime detects that native + frame may overrun this boundary, it throws stack overflow + exception. */ + uint8 *native_stack_boundary; + + /* Used to terminate or suspend current thread */ + WASMSuspendFlags suspend_flags; + + /* Auxiliary stack boundary */ + uintptr_t aux_stack_boundary; + + /* Auxiliary stack bottom */ + uintptr_t aux_stack_bottom; + +#if WASM_ENABLE_AOT != 0 + /* Native symbol list, reserved */ + void **native_symbol; +#endif + + /* + * The lowest stack pointer value observed. + * Assumption: native stack grows to the lower address. + */ + uint8 *native_stack_top_min; + + struct { + /* The top boundary of the stack. */ + uint8 *top_boundary; + /* The top to of the wasm stack which is free. */ + uint8 *top; + /* The bottom of the wasm stack. */ + uint8 *bottom; + } wasm_stack; + +#if WASM_ENABLE_INSTRUCTION_METERING != 0 + /* instructions to execute */ + int instructions_to_execute; +#endif + +#if WASM_ENABLE_FAST_JIT != 0 + /** + * Cache for + * - jit native operations in 32-bit target which hasn't 64-bit + * int/float registers, mainly for the operations of double and int64, + * such as F64TOI64, F32TOI64, I64 MUL/REM, and so on. + * - SSE instructions. + **/ + uint64 jit_cache[2]; +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 + /* thread return value */ + void *thread_ret_value; + + /* Must be provided by thread library */ + void *(*thread_start_routine)(void *); + void *thread_arg; + + /* pointer to the cluster */ + WASMCluster *cluster; + + /* used to support debugger */ + korp_mutex wait_lock; + korp_cond wait_cond; + /* the count of threads which are joining current thread */ + uint32 wait_count; + + /* whether current thread is detached */ + bool thread_is_detached; + + /* whether the aux stack is allocated */ + bool is_aux_stack_allocated; +#endif + +#if WASM_ENABLE_GC != 0 + /* Current local object reference variable */ + struct WASMLocalObjectRef *cur_local_object_ref; +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 + WASMCurrentEnvStatus *current_status; +#endif + + /* attachment for native function */ + void *attachment; + + void *user_data; + + /* The boundary of native stack set by host embedder. It is used + if it is not NULL when calling wasm functions. */ + uint8 *user_native_stack_boundary; + + /* The native thread handle of current thread */ + korp_tid handle; + +#if WASM_ENABLE_INTERP != 0 && WASM_ENABLE_FAST_INTERP == 0 + BlockAddr block_addr_cache[BLOCK_ADDR_CACHE_SIZE][BLOCK_ADDR_CONFLICT_SIZE]; +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK + WASMJmpBuf *jmpbuf_stack_top; + /* One guard page for the exception check */ + uint8 *exce_check_guard_page; +#endif + +#if WASM_ENABLE_MEMORY_PROFILING != 0 + uint32 max_wasm_stack_used; +#endif + + /* The WASM stack size */ + uint32 wasm_stack_size; + + /* The WASM stack of current thread */ + union { + uint64 __make_it_8_byte_aligned_; + /* The WASM stack. */ + uint8 bottom[1]; + } wasm_stack_u; +} WASMExecEnv; + +#if WASM_ENABLE_MEMORY_PROFILING != 0 +#define RECORD_STACK_USAGE(e, p) \ + do { \ + if ((e)->native_stack_top_min > (p)) { \ + (e)->native_stack_top_min = (p); \ + } \ + } while (0) +#else +#define RECORD_STACK_USAGE(e, p) (void)0 +#endif + +WASMExecEnv * +wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst, + uint32 stack_size); + +void +wasm_exec_env_destroy_internal(WASMExecEnv *exec_env); + +WASMExecEnv * +wasm_exec_env_create(struct WASMModuleInstanceCommon *module_inst, + uint32 stack_size); + +void +wasm_exec_env_destroy(WASMExecEnv *exec_env); + +static inline bool +wasm_exec_env_is_aux_stack_managed_by_runtime(WASMExecEnv *exec_env) +{ + return exec_env->aux_stack_boundary != 0 || exec_env->aux_stack_bottom != 0; +} + +/** + * Allocate a WASM frame from the WASM stack. + * + * @param exec_env the current execution environment + * @param size size of the WASM frame, it must be a multiple of 4 + * + * @return the WASM frame if there is enough space in the stack area + * with a protection area, NULL otherwise + */ +static inline void * +wasm_exec_env_alloc_wasm_frame(WASMExecEnv *exec_env, unsigned size) +{ + uint8 *addr = exec_env->wasm_stack.top; + + bh_assert(!(size & 3)); + + /* For classic interpreter, the outs area doesn't contain the const cells, + its size cannot be larger than the frame size, so here checking stack + overflow with multiplying by 2 is enough. For fast interpreter, since + the outs area contains const cells, its size may be larger than current + frame size, we should check again before putting the function arguments + into the outs area. */ + if (size * 2 + > (uint32)(uintptr_t)(exec_env->wasm_stack.top_boundary - addr)) { + /* WASM stack overflow. */ + return NULL; + } + + exec_env->wasm_stack.top += size; + +#if WASM_ENABLE_MEMORY_PROFILING != 0 + { + uint32 wasm_stack_used = + exec_env->wasm_stack.top - exec_env->wasm_stack.bottom; + if (wasm_stack_used > exec_env->max_wasm_stack_used) + exec_env->max_wasm_stack_used = wasm_stack_used; + } +#endif + return addr; +} + +static inline void +wasm_exec_env_free_wasm_frame(WASMExecEnv *exec_env, void *prev_top) +{ + bh_assert((uint8 *)prev_top >= exec_env->wasm_stack.bottom); + exec_env->wasm_stack.top = (uint8 *)prev_top; +} + +/** + * Get the current WASM stack top pointer. + * + * @param exec_env the current execution environment + * + * @return the current WASM stack top pointer + */ +static inline void * +wasm_exec_env_wasm_stack_top(WASMExecEnv *exec_env) +{ + return exec_env->wasm_stack.top; +} + +/** + * Set the current frame pointer. + * + * @param exec_env the current execution environment + * @param frame the WASM frame to be set for the current exec env + */ +static inline void +wasm_exec_env_set_cur_frame(WASMExecEnv *exec_env, + struct WASMInterpFrame *frame) +{ + exec_env->cur_frame = frame; +} + +/** + * Get the current frame pointer. + * + * @param exec_env the current execution environment + * + * @return the current frame pointer + */ +static inline struct WASMInterpFrame * +wasm_exec_env_get_cur_frame(WASMExecEnv *exec_env) +{ + return exec_env->cur_frame; +} + +struct WASMModuleInstanceCommon * +wasm_exec_env_get_module_inst(WASMExecEnv *exec_env); + +void +wasm_exec_env_set_module_inst( + WASMExecEnv *exec_env, struct WASMModuleInstanceCommon *const module_inst); + +void +wasm_exec_env_restore_module_inst( + WASMExecEnv *exec_env, struct WASMModuleInstanceCommon *const module_inst); + +void +wasm_exec_env_set_thread_info(WASMExecEnv *exec_env); + +#if WASM_ENABLE_THREAD_MGR != 0 +void * +wasm_exec_env_get_thread_arg(WASMExecEnv *exec_env); + +void +wasm_exec_env_set_thread_arg(WASMExecEnv *exec_env, void *thread_arg); +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK +void +wasm_exec_env_push_jmpbuf(WASMExecEnv *exec_env, WASMJmpBuf *jmpbuf); + +WASMJmpBuf * +wasm_exec_env_pop_jmpbuf(WASMExecEnv *exec_env); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_EXEC_ENV_H */ diff --git a/wamr/include/wasm_export.h b/wamr/include/wasm_export.h new file mode 100644 index 0000000..43d1bda --- /dev/null +++ b/wamr/include/wasm_export.h @@ -0,0 +1,2440 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/** + * @file wasm_export.h + * + * @brief This file defines the exported common runtime APIs + */ + +#ifndef _WASM_EXPORT_H +#define _WASM_EXPORT_H + +#include +#include +#include "lib_export.h" // lib_export.h + +#ifndef WASM_RUNTIME_API_EXTERN +#if defined(_MSC_BUILD) +#if defined(COMPILING_WASM_RUNTIME_API) +#define WASM_RUNTIME_API_EXTERN __declspec(dllexport) +#else +#define WASM_RUNTIME_API_EXTERN __declspec(dllimport) +#endif +#elif defined(__GNUC__) || defined(__clang__) +#define WASM_RUNTIME_API_EXTERN __attribute__((visibility("default"))) +#else +#define WASM_RUNTIME_API_EXTERN +#endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define get_module_inst(exec_env) wasm_runtime_get_module_inst(exec_env) + +#define validate_app_addr(offset, size) \ + wasm_runtime_validate_app_addr(module_inst, offset, size) + +#define validate_app_str_addr(offset) \ + wasm_runtime_validate_app_str_addr(module_inst, offset) + +#define addr_app_to_native(offset) \ + wasm_runtime_addr_app_to_native(module_inst, offset) + +#define addr_native_to_app(ptr) \ + wasm_runtime_addr_native_to_app(module_inst, ptr) + +#define module_malloc(size, p_native_addr) \ + wasm_runtime_module_malloc(module_inst, size, p_native_addr) + +#define module_free(offset) wasm_runtime_module_free(module_inst, offset) + +#define native_raw_return_type(type, args) type *raw_ret = (type *)(args) + +#define native_raw_get_arg(type, name, args) type name = *((type *)(args++)) + +#define native_raw_set_return(val) *raw_ret = (val) + +#ifndef WASM_MODULE_T_DEFINED +#define WASM_MODULE_T_DEFINED +/* Uninstantiated WASM module loaded from WASM binary file + or AoT binary file*/ +struct WASMModuleCommon; +typedef struct WASMModuleCommon *wasm_module_t; +#endif + +typedef enum { + WASM_IMPORT_EXPORT_KIND_FUNC, + WASM_IMPORT_EXPORT_KIND_TABLE, + WASM_IMPORT_EXPORT_KIND_MEMORY, + WASM_IMPORT_EXPORT_KIND_GLOBAL +} wasm_import_export_kind_t; + +struct WASMFuncType; +typedef struct WASMFuncType *wasm_func_type_t; + +struct WASMTableType; +typedef struct WASMTableType *wasm_table_type_t; + +struct WASMGlobalType; +typedef struct WASMGlobalType *wasm_global_type_t; + +#ifndef WASM_MEMORY_T_DEFINED +#define WASM_MEMORY_T_DEFINED +struct WASMMemory; +typedef struct WASMMemory WASMMemoryType; +#endif +typedef WASMMemoryType *wasm_memory_type_t; + +typedef struct wasm_import_t { + const char *module_name; + const char *name; + wasm_import_export_kind_t kind; + bool linked; + union { + wasm_func_type_t func_type; + wasm_table_type_t table_type; + wasm_global_type_t global_type; + wasm_memory_type_t memory_type; + } u; +} wasm_import_t; + +typedef struct wasm_export_t { + const char *name; + wasm_import_export_kind_t kind; + union { + wasm_func_type_t func_type; + wasm_table_type_t table_type; + wasm_global_type_t global_type; + wasm_memory_type_t memory_type; + } u; +} wasm_export_t; + +/* Instantiated WASM module */ +struct WASMModuleInstanceCommon; +typedef struct WASMModuleInstanceCommon *wasm_module_inst_t; + +/* Function instance */ +typedef void WASMFunctionInstanceCommon; +typedef WASMFunctionInstanceCommon *wasm_function_inst_t; + +/* Memory instance */ +struct WASMMemoryInstance; +typedef struct WASMMemoryInstance *wasm_memory_inst_t; + +typedef struct wasm_frame_t { + /* wasm_instance_t */ + void *instance; + uint32_t module_offset; + uint32_t func_index; + uint32_t func_offset; + const char *func_name_wp; + + uint32_t *sp; + uint8_t *frame_ref; + uint32_t *lp; +} WASMCApiFrame; + +/* WASM section */ +typedef struct wasm_section_t { + struct wasm_section_t *next; + /* section type */ + int section_type; + /* section body, not include type and size */ + uint8_t *section_body; + /* section body size */ + uint32_t section_body_size; +} wasm_section_t, aot_section_t, *wasm_section_list_t, *aot_section_list_t; + +/* Execution environment, e.g. stack info */ +struct WASMExecEnv; +typedef struct WASMExecEnv *wasm_exec_env_t; + +struct WASMSharedHeap; +typedef struct WASMSharedHeap *wasm_shared_heap_t; + +/* Package Type */ +typedef enum { + Wasm_Module_Bytecode = 0, + Wasm_Module_AoT, + Package_Type_Unknown = 0xFFFF +} package_type_t; + +#ifndef MEM_ALLOC_OPTION_DEFINED +#define MEM_ALLOC_OPTION_DEFINED +/* Memory allocator type */ +typedef enum { + /* pool mode, allocate memory from user defined heap buffer */ + Alloc_With_Pool = 0, + /* user allocator mode, allocate memory from user defined + malloc function */ + Alloc_With_Allocator, + /* system allocator mode, allocate memory from system allocator, + or, platform's os_malloc function */ + Alloc_With_System_Allocator, +} mem_alloc_type_t; + +typedef enum { Alloc_For_Runtime, Alloc_For_LinearMemory } mem_alloc_usage_t; + +/* Memory allocator option */ +typedef union MemAllocOption { + struct { + void *heap_buf; + uint32_t heap_size; + } pool; + struct { + /* the function signature is varied when + WASM_MEM_ALLOC_WITH_USER_DATA and + WASM_MEM_ALLOC_WITH_USAGE are defined */ + void *malloc_func; + void *realloc_func; + void *free_func; + /* allocator user data, only used when + WASM_MEM_ALLOC_WITH_USER_DATA is defined */ + void *user_data; + } allocator; +} MemAllocOption; +#endif + +/* Memory pool info */ +typedef struct mem_alloc_info_t { + uint32_t total_size; + uint32_t total_free_size; + uint32_t highmark_size; +} mem_alloc_info_t; + +/* Running mode of runtime and module instance*/ +typedef enum RunningMode { + Mode_Interp = 1, + Mode_Fast_JIT, + Mode_LLVM_JIT, + Mode_Multi_Tier_JIT, +} RunningMode; + +/* WASM runtime initialize arguments */ +typedef struct RuntimeInitArgs { + mem_alloc_type_t mem_alloc_type; + MemAllocOption mem_alloc_option; + + const char *native_module_name; + NativeSymbol *native_symbols; + uint32_t n_native_symbols; + + /* maximum thread number, only used when + WASM_ENABLE_THREAD_MGR is defined */ + uint32_t max_thread_num; + + /* Debug settings, only used when + WASM_ENABLE_DEBUG_INTERP != 0 */ + char ip_addr[128]; + int unused; /* was platform_port */ + int instance_port; + + /* Fast JIT code cache size */ + uint32_t fast_jit_code_cache_size; + + /* Default GC heap size */ + uint32_t gc_heap_size; + + /* Default running mode of the runtime */ + RunningMode running_mode; + + /* LLVM JIT opt and size level */ + uint32_t llvm_jit_opt_level; + uint32_t llvm_jit_size_level; + /* Segue optimization flags for LLVM JIT */ + uint32_t segue_flags; + /** + * If enabled + * - llvm-jit will output a jitdump file for `perf inject` + * - aot will output a perf-${pid}.map for `perf record` + * - fast-jit. TBD + * - multi-tier-jit. TBD + * - interpreter. TBD + */ + bool enable_linux_perf; +} RuntimeInitArgs; + +#ifndef LOAD_ARGS_OPTION_DEFINED +#define LOAD_ARGS_OPTION_DEFINED +typedef struct LoadArgs { + char *name; + /* This option is only used by the Wasm C API (see wasm_c_api.h) */ + bool clone_wasm_binary; + /* False by default, used by AOT/wasm loader only. + If true, the AOT/wasm loader creates a copy of some module fields (e.g. + const strings), making it possible to free the wasm binary buffer after + loading. */ + bool wasm_binary_freeable; + + /* false by default, if true, don't resolve the symbols yet. The + wasm_runtime_load_ex has to be followed by a wasm_runtime_resolve_symbols + call */ + bool no_resolve; + /* TODO: more fields? */ +} LoadArgs; +#endif /* LOAD_ARGS_OPTION_DEFINED */ + +#ifndef INSTANTIATION_ARGS_OPTION_DEFINED +#define INSTANTIATION_ARGS_OPTION_DEFINED +/* WASM module instantiation arguments */ +typedef struct InstantiationArgs { + uint32_t default_stack_size; + uint32_t host_managed_heap_size; + uint32_t max_memory_pages; +} InstantiationArgs; +#endif /* INSTANTIATION_ARGS_OPTION_DEFINED */ + +struct InstantiationArgs2; + +#ifndef WASM_VALKIND_T_DEFINED +#define WASM_VALKIND_T_DEFINED +typedef uint8_t wasm_valkind_t; +enum wasm_valkind_enum { + WASM_I32, + WASM_I64, + WASM_F32, + WASM_F64, + WASM_V128, + WASM_EXTERNREF = 128, + WASM_FUNCREF, +}; +#endif + +#ifndef WASM_VAL_T_DEFINED +#define WASM_VAL_T_DEFINED +struct wasm_ref_t; + +typedef struct wasm_val_t { + wasm_valkind_t kind; + uint8_t _paddings[7]; + union { + /* also represent a function index */ + int32_t i32; + int64_t i64; + float f32; + double f64; + /* represent a foreign object, aka externref in .wat */ + uintptr_t foreign; + struct wasm_ref_t *ref; + } of; +} wasm_val_t; +#endif + +/* Global instance*/ +typedef struct wasm_global_inst_t { + wasm_valkind_t kind; + bool is_mutable; + void *global_data; +} wasm_global_inst_t; + +/* Table instance*/ +typedef struct wasm_table_inst_t { + wasm_valkind_t elem_kind; + uint32_t cur_size; + uint32_t max_size; + /* represents the elements of the table, for internal use only */ + void *elems; +} wasm_table_inst_t; + +typedef enum { + WASM_LOG_LEVEL_FATAL = 0, + WASM_LOG_LEVEL_ERROR = 1, + WASM_LOG_LEVEL_WARNING = 2, + WASM_LOG_LEVEL_DEBUG = 3, + WASM_LOG_LEVEL_VERBOSE = 4 +} log_level_t; + +typedef struct SharedHeapInitArgs { + uint32_t size; + void *pre_allocated_addr; +} SharedHeapInitArgs; + +/** + * Initialize the WASM runtime environment, and also initialize + * the memory allocator with system allocator, which calls os_malloc + * to allocate memory + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_init(void); + +/** + * Initialize the WASM runtime environment, WASM running mode, + * and also initialize the memory allocator and register native symbols, + * which are specified with init arguments + * + * @param init_args specifies the init arguments + * + * @return return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_full_init(RuntimeInitArgs *init_args); + +/** + * Set the log level. To be called after the runtime is initialized. + * + * @param level the log level to set + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_log_level(log_level_t level); + +/** + * Query whether a certain running mode is supported for the runtime + * + * @param running_mode the running mode to query + * + * @return true if this running mode is supported, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_running_mode_supported(RunningMode running_mode); + +/** + * Set the default running mode for the runtime. It is inherited + * to set the running mode of a module instance when it is instantiated, + * and can be changed by calling wasm_runtime_set_running_mode + * + * @param running_mode the running mode to set + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_set_default_running_mode(RunningMode running_mode); + +/** + * Destroy the WASM runtime environment. + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_destroy(void); + +/** + * Allocate memory from runtime memory environment. + * + * @param size bytes need to allocate + * + * @return the pointer to memory allocated + */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_malloc(unsigned int size); + +/** + * Reallocate memory from runtime memory environment + * + * @param ptr the original memory + * @param size bytes need to reallocate + * + * @return the pointer to memory reallocated + */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_realloc(void *ptr, unsigned int size); + +/* + * Free memory to runtime memory environment. + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_free(void *ptr); + +/* + * Get memory info, only pool mode is supported now. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_get_mem_alloc_info(mem_alloc_info_t *mem_alloc_info); + +/** + * Get the package type of a buffer. + * + * @param buf the package buffer + * @param size the package buffer size + * + * @return the package type, return Package_Type_Unknown if the type is unknown + */ +WASM_RUNTIME_API_EXTERN package_type_t +get_package_type(const uint8_t *buf, uint32_t size); + +/** + * Get the package type of a buffer (same as get_package_type). + * + * @param buf the package buffer + * @param size the package buffer size + * + * @return the package type, return Package_Type_Unknown if the type is unknown + */ +WASM_RUNTIME_API_EXTERN package_type_t +wasm_runtime_get_file_package_type(const uint8_t *buf, uint32_t size); + +/** + * Get the package type of a module. + * + * @param module the module + * + * @return the package type, return Package_Type_Unknown if the type is + * unknown + */ +WASM_RUNTIME_API_EXTERN package_type_t +wasm_runtime_get_module_package_type(const wasm_module_t module); + +/** + * Get the package version of a buffer. + * + * @param buf the package buffer + * @param size the package buffer size + * + * @return the package version, return zero if the version is unknown + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_get_file_package_version(const uint8_t *buf, uint32_t size); + +/** + * Get the package version of a module + * + * @param module the module + * + * @return the package version, or zero if version is unknown + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_get_module_package_version(const wasm_module_t module); + +/** + * Get the currently supported version of the package type + * + * @param package_type the package type + * + * @return the currently supported version, or zero if package type is unknown + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_get_current_package_version(package_type_t package_type); + +/** + * Check whether a file is an AOT XIP (Execution In Place) file + * + * @param buf the package buffer + * @param size the package buffer size + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_xip_file(const uint8_t *buf, uint32_t size); + +/** + * Callback to load a module file into a buffer in multi-module feature + */ +typedef bool (*module_reader)(package_type_t module_type, + const char *module_name, uint8_t **p_buffer, + uint32_t *p_size); + +/** + * Callback to release the buffer loaded by module_reader callback + */ +typedef void (*module_destroyer)(uint8_t *buffer, uint32_t size); + +/** + * Setup callbacks for reading and releasing a buffer about a module file + * + * @param reader a callback to read a module file into a buffer + * @param destroyer a callback to release above buffer + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_module_reader(const module_reader reader, + const module_destroyer destroyer); +/** + * Give the "module" a name "module_name". + * Can not assign a new name to a module if it already has a name + * + * @param module_name indicate a name + * @param module the target module + * @param error_buf output of the exception info + * @param error_buf_size the size of the exception string + * + * @return true means success, false means failed + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_register_module(const char *module_name, wasm_module_t module, + char *error_buf, uint32_t error_buf_size); + +/** + * Check if there is already a loaded module named module_name in the + * runtime. Repeatedly loading a module with the same name is not allowed. + * + * @param module_name indicate a name + * + * @return return WASM module loaded, NULL if failed + */ +WASM_RUNTIME_API_EXTERN wasm_module_t +wasm_runtime_find_module_registered(const char *module_name); + +/** + * Load a WASM module from a specified byte buffer. The byte buffer can be + * WASM binary data when interpreter or JIT is enabled, or AOT binary data + * when AOT is enabled. If it is AOT binary data, it must be 4-byte aligned. + * + * Note: In case of AOT XIP modules, the runtime doesn't make modifications + * to the buffer. (Except the "Known issues" mentioned in doc/xip.md.) + * Otherwise, the runtime can make modifications to the buffer for its + * internal purposes. Thus, in general, it isn't safe to create multiple + * modules from a single buffer. + * + * @param buf the byte buffer which contains the WASM/AOT binary data, + * note that the byte buffer must be writable since runtime may + * change its content for footprint and performance purpose, and + * it must be referenceable until wasm_runtime_unload is called + * @param size the size of the buffer + * @param error_buf output of the exception info + * @param error_buf_size the size of the exception string + * + * @return return WASM module loaded, NULL if failed + */ +WASM_RUNTIME_API_EXTERN wasm_module_t +wasm_runtime_load(uint8_t *buf, uint32_t size, char *error_buf, + uint32_t error_buf_size); + +/** + * Load a WASM module with specified load argument. + */ +WASM_RUNTIME_API_EXTERN wasm_module_t +wasm_runtime_load_ex(uint8_t *buf, uint32_t size, const LoadArgs *args, + char *error_buf, uint32_t error_buf_size); + +/** + * Resolve symbols for a previously loaded WASM module. Only useful when the + * module was loaded with LoadArgs::no_resolve set to true + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_resolve_symbols(wasm_module_t module); +/** + * Load a WASM module from a specified WASM or AOT section list. + * + * @param section_list the section list which contains each section data + * @param is_aot whether the section list is AOT section list + * @param error_buf output of the exception info + * @param error_buf_size the size of the exception string + * + * @return return WASM module loaded, NULL if failed + */ +WASM_RUNTIME_API_EXTERN wasm_module_t +wasm_runtime_load_from_sections(wasm_section_list_t section_list, bool is_aot, + char *error_buf, uint32_t error_buf_size); + +/** + * Unload a WASM module. + * + * @param module the module to be unloaded + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_unload(wasm_module_t module); + +/** + * Get the module hash of a WASM module, currently only available on + * linux-sgx platform when the remote attestation feature is enabled + * + * @param module the WASM module to retrieve + * + * @return the module hash of the WASM module + */ +char * +wasm_runtime_get_module_hash(wasm_module_t module); + +/** + * Set WASI parameters. + * + * While this API operates on a module, these parameters will be used + * only when the module is instantiated. That is, you can consider these + * as extra parameters for wasm_runtime_instantiate(). + * + * @param module The module to set WASI parameters. + * @param dir_list The list of directories to preopen. (real path) + * @param dir_count The number of elements in dir_list. + * @param map_dir_list The list of directories to preopen. (mapped path) + * Format for each map entry: :: + * @param map_dir_count The number of elements in map_dir_list. + * If map_dir_count is smaller than dir_count, + * mapped path is assumed to be same as the + * corresponding real path for the rest of entries. + * @param env The list of environment variables. + * @param env_count The number of elements in env. + * @param argv The list of command line arguments. + * @param argc The number of elements in argv. + * @param stdin_handle The raw host handle to back WASI STDIN_FILENO. + * If an invalid handle is specified (e.g. -1 on POSIX, + * INVALID_HANDLE_VALUE on Windows), the platform default + * for STDIN is used. + * @param stdoutfd The raw host handle to back WASI STDOUT_FILENO. + * If an invalid handle is specified (e.g. -1 on POSIX, + * INVALID_HANDLE_VALUE on Windows), the platform default + * for STDOUT is used. + * @param stderrfd The raw host handle to back WASI STDERR_FILENO. + * If an invalid handle is specified (e.g. -1 on POSIX, + * INVALID_HANDLE_VALUE on Windows), the platform default + * for STDERR is used. + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_wasi_args_ex(wasm_module_t module, const char *dir_list[], + uint32_t dir_count, const char *map_dir_list[], + uint32_t map_dir_count, const char *env[], + uint32_t env_count, char *argv[], int argc, + int64_t stdinfd, int64_t stdoutfd, + int64_t stderrfd); + +/** + * Set WASI parameters. + * + * Same as wasm_runtime_set_wasi_args_ex but with default stdio handles + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_wasi_args(wasm_module_t module, const char *dir_list[], + uint32_t dir_count, const char *map_dir_list[], + uint32_t map_dir_count, const char *env[], + uint32_t env_count, char *argv[], int argc); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_wasi_addr_pool(wasm_module_t module, const char *addr_pool[], + uint32_t addr_pool_size); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_wasi_ns_lookup_pool(wasm_module_t module, + const char *ns_lookup_pool[], + uint32_t ns_lookup_pool_size); + +/** + * Instantiate a WASM module. + * + * @param module the WASM module to instantiate + * @param default_stack_size the default stack size of the module instance when + * the exec env's operation stack isn't created by user, e.g. API + * wasm_application_execute_main() and wasm_application_execute_func() + * create the operation stack internally with the stack size specified + * here. And API wasm_runtime_create_exec_env() creates the operation + * stack with stack size specified by its parameter, the stack size + * specified here is ignored. + * @param host_managed_heap_size the default heap size of the module instance, + * a heap will be created besides the app memory space. Both wasm app + * and native function can allocate memory from the heap. + * @param error_buf buffer to output the error info if failed + * @param error_buf_size the size of the error buffer + * + * @return return the instantiated WASM module instance, NULL if failed + */ +WASM_RUNTIME_API_EXTERN wasm_module_inst_t +wasm_runtime_instantiate(const wasm_module_t module, + uint32_t default_stack_size, + uint32_t host_managed_heap_size, char *error_buf, + uint32_t error_buf_size); + +/** + * Instantiate a WASM module, with specified instantiation arguments + * + * Same as wasm_runtime_instantiate, but it also allows overwriting maximum + * memory + */ +WASM_RUNTIME_API_EXTERN wasm_module_inst_t +wasm_runtime_instantiate_ex(const wasm_module_t module, + const InstantiationArgs *args, char *error_buf, + uint32_t error_buf_size); + +/** + * Create an InstantiationArgs2 object with default parameters. + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_instantiation_args_create(struct InstantiationArgs2 **p); + +/** + * Dispose an InstantiationArgs2 object. + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_instantiation_args_destroy(struct InstantiationArgs2 *p); + +/** + * Setter functions for the InstantiationArgs2 object. + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_instantiation_args_set_default_stack_size( + struct InstantiationArgs2 *p, uint32_t v); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_instantiation_args_set_host_managed_heap_size( + struct InstantiationArgs2 *p, uint32_t v); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_instantiation_args_set_max_memory_pages( + struct InstantiationArgs2 *p, uint32_t v); + +/** + * Instantiate a WASM module, with specified instantiation arguments + * + * Same as wasm_runtime_instantiate_ex, but this version takes + * InstantiationArgs2, which can be extended without breaking the ABI. + */ +WASM_RUNTIME_API_EXTERN wasm_module_inst_t +wasm_runtime_instantiate_ex2(const wasm_module_t module, + const struct InstantiationArgs2 *args, + char *error_buf, uint32_t error_buf_size); + +/** + * Set the running mode of a WASM module instance, override the + * default running mode of the runtime. Note that it only makes sense when + * the input is a wasm bytecode file: for the AOT file, runtime always runs + * it with AOT engine, and this function always returns true. + * + * @param module_inst the WASM module instance to set running mode + * @param running_mode the running mode to set + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_set_running_mode(wasm_module_inst_t module_inst, + RunningMode running_mode); + +/** + * Get the running mode of a WASM module instance, if no running mode + * is explicitly set the default running mode of runtime will + * be used and returned. Note that it only makes sense when the input is a + * wasm bytecode file: for the AOT file, this function always returns 0. + * + * @param module_inst the WASM module instance to query for running mode + * + * @return the running mode this module instance currently use + */ +WASM_RUNTIME_API_EXTERN RunningMode +wasm_runtime_get_running_mode(wasm_module_inst_t module_inst); + +/** + * Deinstantiate a WASM module instance, destroy the resources. + * + * @param module_inst the WASM module instance to destroy + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_deinstantiate(wasm_module_inst_t module_inst); + +/** + * Get WASM module from WASM module instance + * + * @param module_inst the WASM module instance to retrieve + * + * @return the WASM module + */ +WASM_RUNTIME_API_EXTERN wasm_module_t +wasm_runtime_get_module(wasm_module_inst_t module_inst); + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_wasi_mode(wasm_module_inst_t module_inst); + +WASM_RUNTIME_API_EXTERN wasm_function_inst_t +wasm_runtime_lookup_wasi_start_function(wasm_module_inst_t module_inst); + +/** + * Get WASI exit code. + * + * After a WASI command completed its execution, an embedder can + * call this function to get its exit code. (that is, the value given + * to proc_exit.) + * + * @param module_inst the module instance + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_get_wasi_exit_code(wasm_module_inst_t module_inst); + +/** + * Lookup an exported function in the WASM module instance. + * + * @param module_inst the module instance + * @param name the name of the function + * + * @return the function instance found, NULL if not found + */ +WASM_RUNTIME_API_EXTERN wasm_function_inst_t +wasm_runtime_lookup_function(const wasm_module_inst_t module_inst, + const char *name); + +/** + * Get parameter count of the function instance + * + * @param func_inst the function instance + * @param module_inst the module instance the function instance belongs to + * + * @return the parameter count of the function instance + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_func_get_param_count(const wasm_function_inst_t func_inst, + const wasm_module_inst_t module_inst); + +/** + * Get result count of the function instance + * + * @param func_inst the function instance + * @param module_inst the module instance the function instance belongs to + * + * @return the result count of the function instance + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_func_get_result_count(const wasm_function_inst_t func_inst, + const wasm_module_inst_t module_inst); + +/** + * Get parameter types of the function instance + * + * @param func_inst the function instance + * @param module_inst the module instance the function instance belongs to + * @param param_types the parameter types returned + */ +WASM_RUNTIME_API_EXTERN void +wasm_func_get_param_types(const wasm_function_inst_t func_inst, + const wasm_module_inst_t module_inst, + wasm_valkind_t *param_types); + +/** + * Get result types of the function instance + * + * @param func_inst the function instance + * @param module_inst the module instance the function instance belongs to + * @param result_types the result types returned + */ +WASM_RUNTIME_API_EXTERN void +wasm_func_get_result_types(const wasm_function_inst_t func_inst, + const wasm_module_inst_t module_inst, + wasm_valkind_t *result_types); + +/** + * Create execution environment for a WASM module instance. + * + * @param module_inst the module instance + * @param stack_size the stack size to execute a WASM function + * + * @return the execution environment, NULL if failed, e.g. invalid + * stack size is passed + */ +WASM_RUNTIME_API_EXTERN wasm_exec_env_t +wasm_runtime_create_exec_env(wasm_module_inst_t module_inst, + uint32_t stack_size); + +/** + * Destroy the execution environment. + * + * @param exec_env the execution environment to destroy + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_destroy_exec_env(wasm_exec_env_t exec_env); + +/** + * @brief Copy callstack frames. + * + * Caution: This is not a thread-safe function. Ensure the exec_env + * is suspended before calling it from another thread. + * + * Usage: In the callback to read frames fields use APIs + * for wasm_frame_t from wasm_c_api.h + * + * Note: The function is async-signal-safe if called with verified arguments. + * Meaning it's safe to call it from a signal handler even on a signal + * interruption from another thread if next variables hold valid pointers + * - exec_env + * - exec_env->module_inst + * - exec_env->module_inst->module + * + * @param exec_env the execution environment that containes frames + * @param buffer the buffer of size equal length * sizeof(wasm_frame_t) to copy + * frames to + * @param length the number of frames to copy + * @param skip_n the number of frames to skip from the top of the stack + * + * @return number of copied frames + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_copy_callstack(const wasm_exec_env_t exec_env, WASMCApiFrame *buffer, + const uint32_t length, const uint32_t skip_n, + char *error_buf, uint32_t error_buf_size); + +/** + * Get the singleton execution environment for the instance. + * + * Note: The singleton execution environment is the execution + * environment used internally by the runtime for the API functions + * like wasm_application_execute_main, which don't take explicit + * execution environment. It's associated to the corresponding + * module instance and managed by the runtime. The API user should + * not destroy it with wasm_runtime_destroy_exec_env. + * + * @param module_inst the module instance + * + * @return exec_env the execution environment to destroy + */ +WASM_RUNTIME_API_EXTERN wasm_exec_env_t +wasm_runtime_get_exec_env_singleton(wasm_module_inst_t module_inst); + +/** + * Start debug instance based on given execution environment. + * Note: + * The debug instance will be destroyed during destroying the + * execution environment, developers don't need to destroy it + * manually. + * If the cluster of this execution environment has already + * been bound to a debug instance, this function will return true + * directly. + * If developer spawns some exec_env by wasm_runtime_spawn_exec_env, + * don't need to call this function for every spawned exec_env as + * they are sharing the same cluster with the main exec_env. + * + * @param exec_env the execution environment to start debug instance + * @param port the port for the debug server to listen on. + * 0 means automatic assignment. + * -1 means to use the global setting in RuntimeInitArgs. + * + * @return debug port if success, 0 otherwise. + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_start_debug_instance_with_port(wasm_exec_env_t exec_env, + int32_t port); + +/** + * Same as wasm_runtime_start_debug_instance_with_port(env, -1). + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_start_debug_instance(wasm_exec_env_t exec_env); + +/** + * Initialize the thread environment. + * Note: + * If developer creates a child thread by himself to call the + * the wasm function in that thread, he should call this API + * firstly before calling the wasm function and then call + * wasm_runtime_destroy_thread_env() after calling the wasm + * function. If the thread is created from the runtime API, + * it is unnecessary to call these two APIs. + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_init_thread_env(void); + +/** + * Destroy the thread environment + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_destroy_thread_env(void); + +/** + * Whether the thread environment is initialized + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_thread_env_inited(void); + +/** + * Get WASM module instance from execution environment + * + * @param exec_env the execution environment to retrieve + * + * @return the WASM module instance + */ +WASM_RUNTIME_API_EXTERN wasm_module_inst_t +wasm_runtime_get_module_inst(wasm_exec_env_t exec_env); + +/** + * Set WASM module instance of execution environment + * Caution: + * normally the module instance is bound with the execution + * environment one by one, if multiple module instances want + * to share to the same execution environment, developer should + * be responsible for the backup and restore of module instance + * + * @param exec_env the execution environment + * @param module_inst the WASM module instance to set + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_module_inst(wasm_exec_env_t exec_env, + const wasm_module_inst_t module_inst); + +/** + * @brief Lookup a memory instance by name + * + * @param module_inst The module instance + * @param name The name of the memory instance + * + * @return The memory instance if found, NULL otherwise + */ +WASM_RUNTIME_API_EXTERN wasm_memory_inst_t +wasm_runtime_lookup_memory(const wasm_module_inst_t module_inst, + const char *name); + +/** + * @brief Get the default memory instance + * + * @param module_inst The module instance + * + * @return The memory instance if found, NULL otherwise + */ +WASM_RUNTIME_API_EXTERN wasm_memory_inst_t +wasm_runtime_get_default_memory(const wasm_module_inst_t module_inst); + +/** + * @brief Get a memory instance by index + * + * @param module_inst The module instance + * @param index The index of the memory instance + * + * @return The memory instance if found, NULL otherwise + */ +WASM_RUNTIME_API_EXTERN wasm_memory_inst_t +wasm_runtime_get_memory(const wasm_module_inst_t module_inst, uint32_t index); + +/** + * @brief Get the current number of pages for a memory instance + * + * @param memory_inst The memory instance + * + * @return The current number of pages + */ +WASM_RUNTIME_API_EXTERN uint64_t +wasm_memory_get_cur_page_count(const wasm_memory_inst_t memory_inst); + +/** + * @brief Get the maximum number of pages for a memory instance + * + * @param memory_inst The memory instance + * + * @return The maximum number of pages + */ +WASM_RUNTIME_API_EXTERN uint64_t +wasm_memory_get_max_page_count(const wasm_memory_inst_t memory_inst); + +/** + * @brief Get the number of bytes per page for a memory instance + * + * @param memory_inst The memory instance + * + * @return The number of bytes per page + */ +WASM_RUNTIME_API_EXTERN uint64_t +wasm_memory_get_bytes_per_page(const wasm_memory_inst_t memory_inst); + +/** + * @brief Get the shared status for a memory instance + * + * @param memory_inst The memory instance + * + * @return True if shared, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_memory_get_shared(const wasm_memory_inst_t memory_inst); + +/** + * @brief Get the base address for a memory instance + * + * @param memory_inst The memory instance + * + * @return The base address on success, false otherwise + */ +WASM_RUNTIME_API_EXTERN void * +wasm_memory_get_base_address(const wasm_memory_inst_t memory_inst); + +/** + * @brief Enlarge a memory instance by a number of pages + * + * @param memory_inst The memory instance + * @param inc_page_count The number of pages to add + * + * @return True if successful, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_memory_enlarge(wasm_memory_inst_t memory_inst, uint64_t inc_page_count); + +/** + * Call the given WASM function of a WASM module instance with + * arguments (bytecode and AoT). + * + * @param exec_env the execution environment to call the function, + * which must be created from wasm_create_exec_env() + * @param function the function to call + * @param argc total cell number that the function parameters occupy, + * a cell is a slot of the uint32 array argv[], e.g. i32/f32 argument + * occupies one cell, i64/f64 argument occupies two cells, note that + * it might be different from the parameter number of the function + * @param argv the arguments. If the function has return value, + * the first (or first two in case 64-bit return value) element of + * argv stores the return value of the called WASM function after this + * function returns. + * + * @return true if success, false otherwise and exception will be thrown, + * the caller can call wasm_runtime_get_exception to get the exception + * info. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_wasm(wasm_exec_env_t exec_env, wasm_function_inst_t function, + uint32_t argc, uint32_t argv[]); + +/** + * Call the given WASM function of a WASM module instance with + * provided results space and arguments (bytecode and AoT). + * + * @param exec_env the execution environment to call the function, + * which must be created from wasm_create_exec_env() + * @param function the function to call + * @param num_results the number of results + * @param results the pre-alloced pointer to get the results + * @param num_args the number of arguments + * @param args the arguments + * + * @return true if success, false otherwise and exception will be thrown, + * the caller can call wasm_runtime_get_exception to get the exception + * info. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_wasm_a(wasm_exec_env_t exec_env, + wasm_function_inst_t function, uint32_t num_results, + wasm_val_t results[], uint32_t num_args, + wasm_val_t *args); + +/** + * Call the given WASM function of a WASM module instance with + * provided results space and variant arguments (bytecode and AoT). + * + * @param exec_env the execution environment to call the function, + * which must be created from wasm_create_exec_env() + * @param function the function to call + * @param num_results the number of results + * @param results the pre-alloced pointer to get the results + * @param num_args the number of arguments + * @param ... the variant arguments + * + * @return true if success, false otherwise and exception will be thrown, + * the caller can call wasm_runtime_get_exception to get the exception + * info. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_wasm_v(wasm_exec_env_t exec_env, + wasm_function_inst_t function, uint32_t num_results, + wasm_val_t results[], uint32_t num_args, ...); + +/** + * Call a function reference of a given WASM runtime instance with + * arguments. + * + * Note: this can be used to call a function which is not exported + * by the module explicitly. You might consider it as an abstraction + * violation. + * + * @param exec_env the execution environment to call the function + * which must be created from wasm_create_exec_env() + * @param element_index the function reference index, usually + * provided by the caller of a registered native function + * @param argc the number of arguments + * @param argv the arguments. If the function method has return value, + * the first (or first two in case 64-bit return value) element of + * argv stores the return value of the called WASM function after this + * function returns. + * + * @return true if success, false otherwise and exception will be thrown, + * the caller can call wasm_runtime_get_exception to get exception info. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_indirect(wasm_exec_env_t exec_env, uint32_t element_index, + uint32_t argc, uint32_t argv[]); + +/** + * Find the unique main function from a WASM module instance + * and execute that function. + * + * @param module_inst the WASM module instance + * @param argc the number of arguments + * @param argv the arguments array, if the main function has return value, + * *(int*)argv stores the return value of the called main function after + * this function returns. + * + * @return true if the main function is called, false otherwise and exception + * will be thrown, the caller can call wasm_runtime_get_exception to get + * the exception info. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_application_execute_main(wasm_module_inst_t module_inst, int32_t argc, + char *argv[]); + +/** + * Find the specified function from a WASM module instance and execute + * that function. + * + * @param module_inst the WASM module instance + * @param name the name of the function to execute. + * to indicate the module name via: $module_name$function_name + * or just a function name: function_name + * @param argc the number of arguments + * @param argv the arguments array + * + * @return true if the specified function is called, false otherwise and + * exception will be thrown, the caller can call wasm_runtime_get_exception + * to get the exception info. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_application_execute_func(wasm_module_inst_t module_inst, const char *name, + int32_t argc, char *argv[]); + +/** + * Get exception info of the WASM module instance. + * + * @param module_inst the WASM module instance + * + * @return the exception string + */ +WASM_RUNTIME_API_EXTERN const char * +wasm_runtime_get_exception(wasm_module_inst_t module_inst); + +/** + * Set exception info of the WASM module instance. + * + * @param module_inst the WASM module instance + * + * @param exception the exception string + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_exception(wasm_module_inst_t module_inst, + const char *exception); + +/** + * Clear exception info of the WASM module instance. + * + * @param module_inst the WASM module instance + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_clear_exception(wasm_module_inst_t module_inst); + +/** + * Terminate the WASM module instance. + * + * This function causes the module instance fail as if it raised a trap. + * + * This is intended to be used in situations like: + * + * - A thread is executing the WASM module instance + * (eg. it's in the middle of `wasm_application_execute_main`) + * + * - Another thread has a copy of `wasm_module_inst_t` of + * the module instance and wants to terminate it asynchronously. + * + * @param module_inst the WASM module instance + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_terminate(wasm_module_inst_t module_inst); + +/** + * Set custom data to WASM module instance. + * Note: + * If WAMR_BUILD_LIB_PTHREAD is enabled, this API + * will spread the custom data to all threads + * + * @param module_inst the WASM module instance + * @param custom_data the custom data to be set + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_custom_data(wasm_module_inst_t module_inst, void *custom_data); + +/** + * Get the custom data within a WASM module instance. + * + * @param module_inst the WASM module instance + * + * @return the custom data (NULL if not set yet) + */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_get_custom_data(wasm_module_inst_t module_inst); + +/** + * Set the memory bounds checks flag of a WASM module instance. + * + * @param module_inst the WASM module instance + * @param enable the flag to enable/disable the memory bounds checks + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_bounds_checks(wasm_module_inst_t module_inst, bool enable); + +/** + * Check if the memory bounds checks flag is enabled for a WASM module instance. + * + * @param module_inst the WASM module instance + * @return true if the memory bounds checks flag is enabled, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_bounds_checks_enabled(wasm_module_inst_t module_inst); + +/** + * Allocate memory from the heap of WASM module instance + * + * Note: wasm_runtime_module_malloc can call heap functions inside + * the module instance and thus cause a memory growth. + * This API needs to be used very carefully when you have a native + * pointers to the module instance memory obtained with + * wasm_runtime_addr_app_to_native or similar APIs. + * + * @param module_inst the WASM module instance which contains heap + * @param size the size bytes to allocate + * @param p_native_addr return native address of the allocated memory + * if it is not NULL, and return NULL if memory malloc failed + * + * @return the allocated memory address, which is a relative offset to the + * base address of the module instance's memory space. Note that + * it is not an absolute address. + * Return non-zero if success, zero if failed. + */ +WASM_RUNTIME_API_EXTERN uint64_t +wasm_runtime_module_malloc(wasm_module_inst_t module_inst, uint64_t size, + void **p_native_addr); + +/** + * Free memory to the heap of WASM module instance + * + * @param module_inst the WASM module instance which contains heap + * @param ptr the pointer to free + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_module_free(wasm_module_inst_t module_inst, uint64_t ptr); + +/** + * Allocate memory from the heap of WASM module instance and initialize + * the memory with src + * + * @param module_inst the WASM module instance which contains heap + * @param src the source data to copy + * @param size the size of the source data + * + * @return the allocated memory address, which is a relative offset to the + * base address of the module instance's memory space. Note that + * it is not an absolute address. + * Return non-zero if success, zero if failed. + */ +WASM_RUNTIME_API_EXTERN uint64_t +wasm_runtime_module_dup_data(wasm_module_inst_t module_inst, const char *src, + uint64_t size); + +/** + * Validate the app address, check whether it belongs to WASM module + * instance's address space, or in its heap space or memory space. + * + * @param module_inst the WASM module instance + * @param app_offset the app address to validate, which is a relative address + * @param size the size bytes of the app address + * + * @return true if success, false otherwise. If failed, an exception will + * be thrown. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_validate_app_addr(wasm_module_inst_t module_inst, + uint64_t app_offset, uint64_t size); + +/** + * Similar to wasm_runtime_validate_app_addr(), except that the size parameter + * is not provided. This function validates the app string address, check + * whether it belongs to WASM module instance's address space, or in its heap + * space or memory space. Moreover, it checks whether it is the offset of a + * string that is end with '\0'. + * + * Note: The validation result, especially the NUL termination check, + * is not reliable for a module instance with multiple threads because + * other threads can modify the heap behind us. + * + * @param module_inst the WASM module instance + * @param app_str_offset the app address of the string to validate, which is a + * relative address + * + * @return true if success, false otherwise. If failed, an exception will + * be thrown. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_validate_app_str_addr(wasm_module_inst_t module_inst, + uint64_t app_str_offset); + +/** + * Validate the native address, check whether it belongs to WASM module + * instance's address space, or in its heap space or memory space. + * + * @param module_inst the WASM module instance + * @param native_ptr the native address to validate, which is an absolute + * address + * @param size the size bytes of the app address + * + * @return true if success, false otherwise. If failed, an exception will + * be thrown. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_validate_native_addr(wasm_module_inst_t module_inst, + void *native_ptr, uint64_t size); + +/** + * Convert app address (relative address) to native address (absolute address) + * + * Note that native addresses to module instance memory can be invalidated + * on a memory growth. (Except shared memory, whose native addresses are + * stable.) + * + * @param module_inst the WASM module instance + * @param app_offset the app address + * + * @return the native address converted + */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_addr_app_to_native(wasm_module_inst_t module_inst, + uint64_t app_offset); + +/** + * Convert native address (absolute address) to app address (relative address) + * + * @param module_inst the WASM module instance + * @param native_ptr the native address + * + * @return the app address converted + */ +WASM_RUNTIME_API_EXTERN uint64_t +wasm_runtime_addr_native_to_app(wasm_module_inst_t module_inst, + void *native_ptr); + +/** + * Get the app address range (relative address) that a app address belongs to + * + * @param module_inst the WASM module instance + * @param app_offset the app address to retrieve + * @param p_app_start_offset buffer to output the app start offset if not NULL + * @param p_app_end_offset buffer to output the app end offset if not NULL + * + * @return true if success, false otherwise. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_get_app_addr_range(wasm_module_inst_t module_inst, + uint64_t app_offset, + uint64_t *p_app_start_offset, + uint64_t *p_app_end_offset); + +/** + * Get the native address range (absolute address) that a native address + * belongs to + * + * @param module_inst the WASM module instance + * @param native_ptr the native address to retrieve + * @param p_native_start_addr buffer to output the native start address + * if not NULL + * @param p_native_end_addr buffer to output the native end address + * if not NULL + * + * @return true if success, false otherwise. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_get_native_addr_range(wasm_module_inst_t module_inst, + uint8_t *native_ptr, + uint8_t **p_native_start_addr, + uint8_t **p_native_end_addr); + +/** + * Get the number of import items for a WASM module + * + * @param module the WASM module + * + * @return the number of imports (zero for none), or -1 for failure + */ +WASM_RUNTIME_API_EXTERN int32_t +wasm_runtime_get_import_count(const wasm_module_t module); + +/** + * Get information about a specific WASM module import + * + * @param module the WASM module + * @param import_index the desired import index + * @param import_type the location to store information about the import + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_get_import_type(const wasm_module_t module, int32_t import_index, + wasm_import_t *import_type); + +/** + * Get the number of export items for a WASM module + * + * @param module the WASM module + * + * @return the number of exports (zero for none), or -1 for failure + */ +WASM_RUNTIME_API_EXTERN int32_t +wasm_runtime_get_export_count(const wasm_module_t module); + +/** + * Get information about a specific WASM module export + * + * @param module the WASM module + * @param export_index the desired export index + * @param export_type the location to store information about the export + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_get_export_type(const wasm_module_t module, int32_t export_index, + wasm_export_t *export_type); + +/** + * Get the number of parameters for a function type + * + * @param func_type the function type + * + * @return the number of parameters for the function type + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_func_type_get_param_count(const wasm_func_type_t func_type); + +/** + * Get the kind of a parameter for a function type + * + * @param func_type the function type + * @param param_index the index of the parameter to get + * + * @return the kind of the parameter if successful, -1 otherwise + */ +WASM_RUNTIME_API_EXTERN wasm_valkind_t +wasm_func_type_get_param_valkind(const wasm_func_type_t func_type, + uint32_t param_index); + +/** + * Get the number of results for a function type + * + * @param func_type the function type + * + * @return the number of results for the function type + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_func_type_get_result_count(const wasm_func_type_t func_type); + +/** + * Get the kind of a result for a function type + * + * @param func_type the function type + * @param result_index the index of the result to get + * + * @return the kind of the result if successful, -1 otherwise + */ +WASM_RUNTIME_API_EXTERN wasm_valkind_t +wasm_func_type_get_result_valkind(const wasm_func_type_t func_type, + uint32_t result_index); + +/** + * Get the kind for a global type + * + * @param global_type the global type + * + * @return the kind of the global + */ +WASM_RUNTIME_API_EXTERN wasm_valkind_t +wasm_global_type_get_valkind(const wasm_global_type_t global_type); + +/** + * Get the mutability for a global type + * + * @param global_type the global type + * + * @return true if mutable, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_global_type_get_mutable(const wasm_global_type_t global_type); + +/** + * Get the shared setting for a memory type + * + * @param memory_type the memory type + * + * @return true if shared, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_memory_type_get_shared(const wasm_memory_type_t memory_type); + +/** + * Get the initial page count for a memory type + * + * @param memory_type the memory type + * + * @return the initial memory page count + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_memory_type_get_init_page_count(const wasm_memory_type_t memory_type); + +/** + * Get the maximum page count for a memory type + * + * @param memory_type the memory type + * + * @return the maximum memory page count + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_memory_type_get_max_page_count(const wasm_memory_type_t memory_type); + +/** + * Get the element kind for a table type + * + * @param table_type the table type + * + * @return the element kind + */ +WASM_RUNTIME_API_EXTERN wasm_valkind_t +wasm_table_type_get_elem_kind(const wasm_table_type_t table_type); + +/** + * Get the sharing setting for a table type + * + * @param table_type the table type + * + * @return true if shared, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_table_type_get_shared(const wasm_table_type_t table_type); + +/** + * Get the initial size for a table type + * + * @param table_type the table type + * + * @return the initial table size + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_table_type_get_init_size(const wasm_table_type_t table_type); + +/** + * Get the maximum size for a table type + * + * @param table_type the table type + * + * @return the maximum table size + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_table_type_get_max_size(const wasm_table_type_t table_type); + +/** + * Register native functions with same module name + * + * Note: The array `native_symbols` should not be read-only because the + * library can modify it in-place. + * + * Note: After successful call of this function, the array `native_symbols` + * is owned by the library. + * + * @param module_name the module name of the native functions + * @param native_symbols specifies an array of NativeSymbol structures which + * contain the names, function pointers and signatures + * Note: WASM runtime will not allocate memory to clone the data, so + * user must ensure the array can be used forever + * Meanings of letters in function signature: + * 'i': the parameter is i32 type + * 'I': the parameter is i64 type + * 'f': the parameter is f32 type + * 'F': the parameter is f64 type + * 'r': the parameter is externref type, it should be a uintptr_t + * in host + * '*': the parameter is a pointer (i32 in WASM), and runtime will + * auto check its boundary before calling the native function. + * If it is followed by '~', the checked length of the pointer + * is gotten from the following parameter, if not, the checked + * length of the pointer is 1. + * '~': the parameter is the pointer's length with i32 type, and must + * follow after '*' + * '$': the parameter is a string (i32 in WASM), and runtime will + * auto check its boundary before calling the native function + * @param n_native_symbols specifies the number of native symbols in the array + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_register_natives(const char *module_name, + NativeSymbol *native_symbols, + uint32_t n_native_symbols); + +/** + * Register native functions with same module name, similar to + * wasm_runtime_register_natives, the difference is that runtime passes raw + * arguments to native API, which means that the native API should be defined as + * void foo(wasm_exec_env_t exec_env, uint64 *args); + * and native API should extract arguments one by one from args array with macro + * native_raw_get_arg + * and write the return value back to args[0] with macro + * native_raw_return_type and native_raw_set_return + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_register_natives_raw(const char *module_name, + NativeSymbol *native_symbols, + uint32_t n_native_symbols); + +/** + * Undo wasm_runtime_register_natives or wasm_runtime_register_natives_raw + * + * @param module_name Should be the same as the corresponding + * wasm_runtime_register_natives. + * (Same in term of strcmp.) + * + * @param native_symbols Should be the same as the corresponding + * wasm_runtime_register_natives. + * (Same in term of pointer comparison.) + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_unregister_natives(const char *module_name, + NativeSymbol *native_symbols); + +/** + * Get an export global instance + * + * @param module_inst the module instance + * @param name the export global name + * @param global_inst location to store the global instance + * + * @return true if success, false otherwise + * + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_get_export_global_inst(const wasm_module_inst_t module_inst, + const char *name, + wasm_global_inst_t *global_inst); + +/** + * Get an export table instance + * + * @param module_inst the module instance + * @param name the export table name + * @param table_inst location to store the table instance + * + * @return true if success, false otherwise + * + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_get_export_table_inst(const wasm_module_inst_t module_inst, + const char *name, + wasm_table_inst_t *table_inst); + +/** + * Get a function instance from a table. + * + * @param module_inst the module instance + * @param table_inst the table instance + * @param idx the index in the table + * + * @return the function instance if successful, NULL otherwise + */ +WASM_RUNTIME_API_EXTERN wasm_function_inst_t +wasm_table_get_func_inst(const wasm_module_inst_t module_inst, + const wasm_table_inst_t *table_inst, uint32_t idx); + +/** + * Get attachment of native function from execution environment + * + * @param exec_env the execution environment to retrieve + * + * @return the attachment of native function + */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_get_function_attachment(wasm_exec_env_t exec_env); + +/** + * Set user data to execution environment. + * + * @param exec_env the execution environment + * @param user_data the user data to be set + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_user_data(wasm_exec_env_t exec_env, void *user_data); + +/** + * Get the user data within execution environment. + * + * @param exec_env the execution environment + * + * @return the user data (NULL if not set yet) + */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_get_user_data(wasm_exec_env_t exec_env); + +/** + * Set native stack boundary to execution environment, if it is set, + * it will be used instead of getting the boundary with the platform + * layer API when calling wasm functions. This is useful for some + * fiber cases. + * + * Note: unlike setting the boundary by runtime, this API doesn't add + * the WASM_STACK_GUARD_SIZE(see comments in core/config.h) to the + * exec_env's native_stack_boundary to reserve bytes to the native + * thread stack boundary, which is used to throw native stack overflow + * exception if the guard boundary is reached. Developer should ensure + * that enough guard bytes are kept. + * + * @param exec_env the execution environment + * @param native_stack_boundary the user data to be set + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_native_stack_boundary(wasm_exec_env_t exec_env, + uint8_t *native_stack_boundary); + +/** + * Set the instruction count limit to the execution environment. + * By default the instruction count limit is -1, which means no limit. + * However, if the instruction count limit is set to a positive value, + * the execution will be terminated when the instruction count reaches + * the limit. + * + * @param exec_env the execution environment + * @param instruction_count the instruction count limit + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_instruction_count_limit(wasm_exec_env_t exec_env, + int instruction_count); + +/** + * Dump runtime memory consumption, including: + * Exec env memory consumption + * WASM module memory consumption + * WASM module instance memory consumption + * stack and app heap used info + * + * @param exec_env the execution environment + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_dump_mem_consumption(wasm_exec_env_t exec_env); + +/** + * Dump runtime performance profiler data of each function + * + * @param module_inst the WASM module instance to profile + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_dump_perf_profiling(wasm_module_inst_t module_inst); + +/** + * Return total wasm functions' execution time in ms + * + * @param module_inst the WASM module instance to profile + */ +WASM_RUNTIME_API_EXTERN double +wasm_runtime_sum_wasm_exec_time(wasm_module_inst_t module_inst); + +/** + * Return execution time in ms of a given wasm function with + * func_name. If the function is not found, return 0. + * + * @param module_inst the WASM module instance to profile + * @param func_name could be an export name or a name in the + * name section + */ +WASM_RUNTIME_API_EXTERN double +wasm_runtime_get_wasm_func_exec_time(wasm_module_inst_t inst, + const char *func_name); + +/* wasm thread callback function type */ +typedef void *(*wasm_thread_callback_t)(wasm_exec_env_t, void *); +/* wasm thread type */ +typedef uintptr_t wasm_thread_t; + +/** + * Set the max thread num per cluster. + * + * @param num maximum thread num + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_max_thread_num(uint32_t num); + +/** + * Spawn a new exec_env, the spawned exec_env + * can be used in other threads + * + * @param num the original exec_env + * + * @return the spawned exec_env if success, NULL otherwise + */ +WASM_RUNTIME_API_EXTERN wasm_exec_env_t +wasm_runtime_spawn_exec_env(wasm_exec_env_t exec_env); + +/** + * Destroy the spawned exec_env + * + * @param exec_env the spawned exec_env + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_destroy_spawned_exec_env(wasm_exec_env_t exec_env); + +/** + * Spawn a thread from the given exec_env + * + * @param exec_env the original exec_env + * @param tid thread id to be returned to the caller + * @param callback the callback function provided by the user + * @param arg the arguments passed to the callback + * + * @return 0 if success, -1 otherwise + */ +WASM_RUNTIME_API_EXTERN int32_t +wasm_runtime_spawn_thread(wasm_exec_env_t exec_env, wasm_thread_t *tid, + wasm_thread_callback_t callback, void *arg); + +/** + * Wait a spawned thread to terminate + * + * @param tid thread id + * @param retval if not NULL, output the return value of the thread + * + * @return 0 if success, error number otherwise + */ +WASM_RUNTIME_API_EXTERN int32_t +wasm_runtime_join_thread(wasm_thread_t tid, void **retval); + +/** + * Map external object to an internal externref index: if the index + * has been created, return it, otherwise create the index. + * + * @param module_inst the WASM module instance that the extern object + * belongs to + * @param extern_obj the external object to be mapped + * @param p_externref_idx return externref index of the external object + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_obj2ref(wasm_module_inst_t module_inst, void *extern_obj, + uint32_t *p_externref_idx); + +/** + * Delete external object registered by `wasm_externref_obj2ref`. + * + * @param module_inst the WASM module instance that the extern object + * belongs to + * @param extern_obj the external object to be deleted + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_objdel(wasm_module_inst_t module_inst, void *extern_obj); + +/** + * Set cleanup callback to release external object. + * + * @param module_inst the WASM module instance that the extern object + * belongs to + * @param extern_obj the external object to which to set the + * `extern_obj_cleanup` cleanup callback. + * @param extern_obj_cleanup a callback to release `extern_obj` + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_set_cleanup(wasm_module_inst_t module_inst, void *extern_obj, + void (*extern_obj_cleanup)(void *)); + +/** + * Retrieve the external object from an internal externref index + * + * @param externref_idx the externref index to retrieve + * @param p_extern_obj return the mapped external object of + * the externref index + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_ref2obj(uint32_t externref_idx, void **p_extern_obj); + +/** + * Retain an extern object which is mapped to the internal externref + * so that the object won't be cleaned during extern object reclaim + * if it isn't used. + * + * @param externref_idx the externref index of an external object + * to retain + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_retain(uint32_t externref_idx); + +/** + * Dump the call stack to stdout + * + * @param exec_env the execution environment + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_dump_call_stack(wasm_exec_env_t exec_env); + +/** + * Get the size required to store the call stack contents, including + * the space for terminating null byte ('\0') + * + * @param exec_env the execution environment + * + * @return size required to store the contents, 0 means error + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_get_call_stack_buf_size(wasm_exec_env_t exec_env); + +/** + * Dump the call stack to buffer. + * + * @note this function is not thread-safe, please only use this API + * when the exec_env is not executing + * + * @param exec_env the execution environment + * @param buf buffer to store the dumped content + * @param len length of the buffer + * + * @return bytes dumped to the buffer, including the terminating null + * byte ('\0'), 0 means error and data in buf may be invalid + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_dump_call_stack_to_buf(wasm_exec_env_t exec_env, char *buf, + uint32_t len); + +/** + * Get the size required to store the LLVM PGO profile data + * + * @param module_inst the WASM module instance + * + * @return size required to store the contents, 0 means error + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_get_pgo_prof_data_size(wasm_module_inst_t module_inst); + +/** + * Dump the LLVM PGO profile data to buffer + * + * @param module_inst the WASM module instance + * @param buf buffer to store the dumped content + * @param len length of the buffer + * + * @return bytes dumped to the buffer, 0 means error and data in buf + * may be invalid + */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_dump_pgo_prof_data_to_buf(wasm_module_inst_t module_inst, + char *buf, uint32_t len); + +/** + * Get a custom section by name + * + * @param module_comm the module to find + * @param name name of the custom section + * @param len return the length of the content if found + * + * @return Custom section content (not including the name length + * and name string) if found, NULL otherwise + */ +WASM_RUNTIME_API_EXTERN const uint8_t * +wasm_runtime_get_custom_section(const wasm_module_t module_comm, + const char *name, uint32_t *len); + +/** + * Get WAMR semantic version + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_get_version(uint32_t *major, uint32_t *minor, uint32_t *patch); + +/** + * Check whether an import func `(import (func ...))` + * is linked or not with runtime registered native functions + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_import_func_linked(const char *module_name, + const char *func_name); + +/** + * Check whether an import global `(import + * (global ...))` is linked or not with runtime registered native globals + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_import_global_linked(const char *module_name, + const char *global_name); + +/** + * Enlarge the memory region for a module instance + * + * @param module_inst the module instance + * @param inc_page_count the number of pages to add + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_enlarge_memory(wasm_module_inst_t module_inst, + uint64_t inc_page_count); + +typedef enum { + INTERNAL_ERROR, + MAX_SIZE_REACHED, +} enlarge_memory_error_reason_t; + +typedef void (*enlarge_memory_error_callback_t)( + uint32_t inc_page_count, uint64_t current_memory_size, + uint32_t memory_index, enlarge_memory_error_reason_t failure_reason, + wasm_module_inst_t instance, wasm_exec_env_t exec_env, void *user_data); + +/** + * Setup callback invoked when memory.grow fails + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_enlarge_mem_error_callback( + const enlarge_memory_error_callback_t callback, void *user_data); + +/* + * module instance context APIs + * wasm_runtime_create_context_key + * wasm_runtime_destroy_context_key + * wasm_runtime_set_context + * wasm_runtime_set_context_spread + * wasm_runtime_get_context + * + * This set of APIs is intended to be used by an embedder which provides + * extra sets of native functions, which need per module instance state + * and are maintained outside of the WAMR tree. + * + * It's modelled after the pthread specific API. + * + * wasm_runtime_set_context_spread is similar to + * wasm_runtime_set_context, except that + * wasm_runtime_set_context_spread applies the change + * to all threads in the cluster. + * It's an undefined behavior if multiple threads in a cluster call + * wasm_runtime_set_context_spread on the same key + * simultaneously. It's a caller's responsibility to perform necessary + * serialization if necessary. For example: + * + * if (wasm_runtime_get_context(inst, key) == NULL) { + * newctx = alloc_and_init(...); + * lock(some_lock); + * if (wasm_runtime_get_context(inst, key) == NULL) { + * // this thread won the race + * wasm_runtime_set_context_spread(inst, key, newctx); + * newctx = NULL; + * } + * unlock(some_lock); + * if (newctx != NULL) { + * // this thread lost the race, free it + * cleanup_and_free(newctx); + * } + * } + * + * Note: dynamic key create/destroy while instances are live is not + * implemented as of writing this. + * it's caller's responsibility to ensure destroying all module instances + * before calling wasm_runtime_create_context_key or + * wasm_runtime_destroy_context_key. + * otherwise, it's an undefined behavior. + * + * Note about threads: + * - When spawning a thread, the contexts (the pointers given to + * wasm_runtime_set_context) are copied from the parent + * instance. + * - The destructor is called only on the main instance. + */ + +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_create_context_key(void (*dtor)(wasm_module_inst_t inst, + void *ctx)); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_destroy_context_key(void *key); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_context(wasm_module_inst_t inst, void *key, void *ctx); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_context_spread(wasm_module_inst_t inst, void *key, void *ctx); + +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_get_context(wasm_module_inst_t inst, void *key); + +/* + * wasm_runtime_begin_blocking_op/wasm_runtime_end_blocking_op + * + * These APIs are intended to be used by the implementations of + * host functions. It wraps an operation which possibly blocks for long + * to prepare for async termination. + * + * For simplicity, we recommend to wrap only the very minimum piece of + * the code with this. Ideally, just a single system call. + * + * eg. + * + * if (!wasm_runtime_begin_blocking_op(exec_env)) { + * return EINTR; + * } + * ret = possibly_blocking_op(); + * wasm_runtime_end_blocking_op(exec_env); + * return ret; + * + * If threading support (WASM_ENABLE_THREAD_MGR) is not enabled, + * these functions are no-op. + * + * If the underlying platform support (OS_ENABLE_WAKEUP_BLOCKING_OP) is + * not available, these functions are no-op. In that case, the runtime + * might not terminate a blocking thread in a timely manner. + * + * If the underlying platform support is available, it's used to wake up + * the thread for async termination. The expectation here is that a + * `os_wakeup_blocking_op` call makes the blocking operation + * (`possibly_blocking_op` in the above example) return in a timely manner. + * + * The actual wake up mechanism used by `os_wakeup_blocking_op` is + * platform-dependent. It might impose some platform-dependent restrictions + * on the implementation of the blocking operation. + * + * For example, on POSIX-like platforms, a signal (by default SIGUSR1) is + * used. The signal delivery configurations (eg. signal handler, signal mask, + * etc) for the signal are set up by the runtime. You can change the signal + * to use for this purpose by calling os_set_signal_number_for_blocking_op + * before the runtime initialization. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_begin_blocking_op(wasm_exec_env_t exec_env); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_end_blocking_op(wasm_exec_env_t exec_env); + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_set_module_name(wasm_module_t module, const char *name, + char *error_buf, uint32_t error_buf_size); + +/* return the most recently set module name or "" if never set before */ +WASM_RUNTIME_API_EXTERN const char * +wasm_runtime_get_module_name(wasm_module_t module); + +/* + * wasm_runtime_detect_native_stack_overflow + * + * Detect native stack shortage. + * Ensure that the calling thread still has a reasonable amount of + * native stack (WASM_STACK_GUARD_SIZE bytes) available. + * + * If enough stack is left, this function returns true. + * Otherwise, this function raises a "native stack overflow" trap and + * returns false. + * + * Note: please do not expect a very strict detection. it's a good idea + * to give some margins. wasm_runtime_detect_native_stack_overflow itself + * requires a small amount of stack to run. + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_detect_native_stack_overflow(wasm_exec_env_t exec_env); + +/* + * wasm_runtime_detect_native_stack_overflow_size + * + * Similar to wasm_runtime_detect_native_stack_overflow, + * but use the caller-specified size instead of WASM_STACK_GUARD_SIZE. + * + * An expected usage: + * ```c + * __attribute__((noinline)) // inlining can break the stack check + * void stack_hog(void) + * { + * // consume a lot of stack here + * } + * + * void + * stack_hog_wrapper(exec_env) { + * // the amount of stack stack_hog would consume, + * // plus a small margin + * uint32_t size = 10000000; + * + * if (!wasm_runtime_detect_native_stack_overflow_size(exec_env, size)) { + * // wasm_runtime_detect_native_stack_overflow_size has raised + * // a trap. + * return; + * } + * stack_hog(); + * } + * ``` + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_detect_native_stack_overflow_size(wasm_exec_env_t exec_env, + uint32_t required_size); + +/** + * Query whether the wasm binary buffer used to create the module can be freed + * + * @param module the target module + * @return true if the wasm binary buffer can be freed + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_underlying_binary_freeable(const wasm_module_t module); + +/** + * Create a shared heap + * + * @param init_args the initialization arguments + * @return the shared heap created + */ +WASM_RUNTIME_API_EXTERN wasm_shared_heap_t +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args); + +/** + * This function links two shared heap(lists), `head` and `body` in to a single + * shared heap list, where `head` becomes the new shared heap list head. The + * shared heap list remains one continuous shared heap in wasm app's point of + * view. At most one shared heap in shared heap list can be dynamically + * allocated, the rest have to be the pre-allocated shared heap. * + * + * @param head The head of the shared heap chain. + * @param body The body of the shared heap chain to be appended. + * @return The new head of the shared heap chain. NULL if failed. + */ +WASM_RUNTIME_API_EXTERN wasm_shared_heap_t +wasm_runtime_chain_shared_heaps(wasm_shared_heap_t head, + wasm_shared_heap_t body); + +/** + * This function unchains the shared heaps from the given head. If + * `entire_chain` is true, it will unchain the entire chain of shared heaps. + * Otherwise, it will unchain only the first shared heap in the chain. + * + * @param head The head of the shared heap chain. + * @param entire_chain A boolean flag indicating whether to unchain the entire + * chain. + * @return The new head of the shared heap chain. Or the last shared heap in the + * chain if `entire_chain` is true. + */ +wasm_shared_heap_t +wasm_runtime_unchain_shared_heaps(wasm_shared_heap_t head, bool entire_chain); + +/** + * Attach a shared heap, it can be the head of shared heap chain, in that case, + * attach the shared heap chain, to a module instance + * + * @param module_inst the module instance + * @param shared_heap the shared heap + * @return true if success, false if failed + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_attach_shared_heap(wasm_module_inst_t module_inst, + wasm_shared_heap_t shared_heap); + +/** + * Detach a shared heap from a module instance + * + * @param module_inst the module instance + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_detach_shared_heap(wasm_module_inst_t module_inst); + +/** + * Allocate memory from a shared heap, or the non-preallocated shared heap from + * the shared heap chain + * + * @param module_inst the module instance + * @param size required memory size + * @param p_native_addr native address of allocated memory + * + * @return return the allocated memory address, which reuses part of the wasm + * address space and is in the range of [UINT32 - shared_heap_size + 1, UINT32] + * (when the wasm memory is 32-bit) or [UINT64 - shared_heap_size + 1, UINT64] + * (when the wasm memory is 64-bit). Note that it is not an absolute address. + * Return non-zero if success, zero if failed. + */ +WASM_RUNTIME_API_EXTERN uint64_t +wasm_runtime_shared_heap_malloc(wasm_module_inst_t module_inst, uint64_t size, + void **p_native_addr); + +/** + * Free the memory allocated from shared heap, or the non-preallocated shared + * heap from the shared heap chain + * + * @param module_inst the module instance + * @param ptr the offset in wasm app + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64_t ptr); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_EXPORT_H */ diff --git a/wamr/include/wasm_interp.h b/wamr/include/wasm_interp.h new file mode 100644 index 0000000..dd32c8b --- /dev/null +++ b/wamr/include/wasm_interp.h @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_INTERP_H +#define _WASM_INTERP_H + +#include "wasm.h" // wasm.h + +#ifdef __cplusplus +extern "C" { +#endif + +struct WASMModuleInstance; +struct WASMFunctionInstance; +struct WASMExecEnv; + +typedef struct WASMInterpFrame { + /* The frame of the caller that are calling the current function. */ + struct WASMInterpFrame *prev_frame; + + /* The current WASM function. */ + struct WASMFunctionInstance *function; + + /* Instruction pointer of the bytecode array. */ + uint8 *ip; + +#if WASM_ENABLE_FAST_JIT != 0 + uint8 *jitted_return_addr; +#endif + +#if WASM_ENABLE_PERF_PROFILING != 0 + uint64 time_started; +#endif + +#if WASM_ENABLE_EXCE_HANDLING != 0 + /* set to true if the callee returns an exception rather than + * result values on the stack + */ + bool exception_raised; + uint32 tag_index; +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + /* Return offset of the first return value of current frame, + the callee will put return values here continuously */ + uint32 ret_offset; + uint32 *lp; +#if WASM_ENABLE_GC != 0 + uint8 *frame_ref; +#endif + uint32 operand[1]; +#else /* else of WASM_ENABLE_FAST_INTERP != 0 */ + /* Operand stack top pointer of the current frame. The bottom of + the stack is the next cell after the last local variable. */ + uint32 *sp_bottom; + uint32 *sp_boundary; + uint32 *sp; + + WASMBranchBlock *csp_bottom; + WASMBranchBlock *csp_boundary; + WASMBranchBlock *csp; + + /** + * Frame data, the layout is: + * lp: parameters and local variables + * sp_bottom to sp_boundary: wasm operand stack + * csp_bottom to csp_boundary: wasm label stack + * frame ref flags: only available for GC + * whether each cell in local and stack area is a GC obj + * jit spill cache: only available for fast jit + */ + uint32 lp[1]; +#endif /* end of WASM_ENABLE_FAST_INTERP != 0 */ +} WASMInterpFrame; + +/** + * Calculate the size of interpreter area of frame of a function. + * + * @param all_cell_num number of all cells including local variables + * and the working stack slots + * + * @return the size of interpreter area of the frame + */ +static inline unsigned +wasm_interp_interp_frame_size(unsigned all_cell_num) +{ + unsigned frame_size; + +#if WASM_ENABLE_FAST_INTERP == 0 +#if WASM_ENABLE_GC == 0 + frame_size = (uint32)offsetof(WASMInterpFrame, lp) + all_cell_num * 4; +#else + frame_size = + (uint32)offsetof(WASMInterpFrame, lp) + align_uint(all_cell_num * 5, 4); +#endif +#else + frame_size = (uint32)offsetof(WASMInterpFrame, operand) + all_cell_num * 4; +#endif + return align_uint(frame_size, 4); +} + +void +wasm_interp_call_wasm(struct WASMModuleInstance *module_inst, + struct WASMExecEnv *exec_env, + struct WASMFunctionInstance *function, uint32 argc, + uint32 argv[]); + +#if WASM_ENABLE_GC != 0 +bool +wasm_interp_traverse_gc_rootset(struct WASMExecEnv *exec_env, void *heap); + +uint8 * +wasm_interp_get_frame_ref(WASMInterpFrame *frame); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_INTERP_H */ diff --git a/wamr/include/wasm_loader.h b/wamr/include/wasm_loader.h new file mode 100644 index 0000000..248108e --- /dev/null +++ b/wamr/include/wasm_loader.h @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#ifndef _WASM_LOADER_H +#define _WASM_LOADER_H + +#include "wasm.h" // wasm.h +#include "bh_hashmap.h" // bh_hashmap.h +#include "wasm_runtime_common.h" // ../common/wasm_runtime_common.h + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Load a WASM module from a specified byte buffer. + * + * @param buf the byte buffer which contains the WASM binary data + * @param size the size of the buffer + * @param error_buf output of the exception info + * @param error_buf_size the size of the exception string + * + * @return return module loaded, NULL if failed + */ +WASMModule * +wasm_loader_load(uint8 *buf, uint32 size, +#if WASM_ENABLE_MULTI_MODULE != 0 + bool main_module, +#endif + const LoadArgs *args, char *error_buf, uint32 error_buf_size); + +/** + * Load a WASM module from a specified WASM section list. + * + * @param section_list the section list which contains each section data + * @param error_buf output of the exception info + * @param error_buf_size the size of the exception string + * + * @return return WASM module loaded, NULL if failed + */ +WASMModule * +wasm_loader_load_from_sections(WASMSection *section_list, char *error_buf, + uint32 error_buf_size); + +/** + * Unload a WASM module. + * + * @param module the module to be unloaded + */ +void +wasm_loader_unload(WASMModule *module); + +/** + * Find address of related else opcode and end opcode of opcode block/loop/if + * according to the start address of opcode. + * + * @param module the module to find + * @param start_addr the next address of opcode block/loop/if + * @param code_end_addr the end address of function code block + * @param block_type the type of block, 0/1/2 denotes block/loop/if + * @param p_else_addr returns the else addr if found + * @param p_end_addr returns the end addr if found + * @param error_buf returns the error log for this function + * @param error_buf_size returns the error log string length + * + * @return true if success, false otherwise + */ + +bool +wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, + const uint8 *start_addr, const uint8 *code_end_addr, + uint8 block_type, uint8 **p_else_addr, + uint8 **p_end_addr); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_LOADER_H */ diff --git a/wamr/include/wasm_loader_common.h b/wamr/include/wasm_loader_common.h new file mode 100644 index 0000000..3997622 --- /dev/null +++ b/wamr/include/wasm_loader_common.h @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2024 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_LOADER_COMMON_H +#define _WASM_LOADER_COMMON_H + +#include "platform_common.h" // platform_common.h +#include "wasm.h" // ../interpreter/wasm.h + +#ifdef __cplusplus +extern "C" { +#endif + +#if WASM_ENABLE_MEMORY64 != 0 +/* check consistency of memory64 flags across all memories, + * they must be either all wasm64 or all wasm32 */ +bool +check_memory64_flags_consistency(WASMModule *module, char *error_buf, + uint32 error_buf_size, bool is_aot); +#endif + +bool +wasm_memory_check_flags(const uint8 mem_flag, char *error_buf, + uint32 error_buf_size, bool is_aot); + +bool +wasm_table_check_flags(const uint8 table_flag, char *error_buf, + uint32 error_buf_size, bool is_aot); + +bool +is_valid_value_type(uint8 value_tpye); + +bool +is_valid_value_type_for_interpreter(uint8 value_tpye); + +bool +is_valid_func_type(const WASMFuncType *func_type); + +bool +is_valid_packed_type(uint8 packed_type); + +bool +is_valid_field_type(uint8 field_type); + +bool +is_indices_overflow(uint32 import, uint32 other, char *error_buf, + uint32 error_buf_size); + +bool +read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign, + uint64 *p_result, char *error_buf, uint32 error_buf_size); + +void +wasm_loader_set_error_buf(char *error_buf, uint32 error_buf_size, + const char *string, bool is_aot); + +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 +void +destroy_init_expr_recursive(InitializerExpression *expr); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_LOADER_COMMON_H */ diff --git a/wamr/include/wasm_memory.h b/wamr/include/wasm_memory.h new file mode 100644 index 0000000..54f768c --- /dev/null +++ b/wamr/include/wasm_memory.h @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_MEMORY_H +#define _WASM_MEMORY_H + +#include "bh_common.h" // bh_common.h +#include "wasm_export.h" // ../include/wasm_export.h +#include "wasm_runtime.h" // ../interpreter/wasm_runtime.h +#include "wasm_shared_memory.h" // ../common/wasm_shared_memory.h + +#ifdef __cplusplus +extern "C" { +#endif + +#if WASM_ENABLE_SHARED_MEMORY != 0 && BH_ATOMIC_64_IS_ATOMIC != 0 +#define GET_LINEAR_MEMORY_SIZE(memory) \ + BH_ATOMIC_64_LOAD(memory->memory_data_size) +#define SET_LINEAR_MEMORY_SIZE(memory, size) \ + BH_ATOMIC_64_STORE(memory->memory_data_size, size) +#elif WASM_ENABLE_SHARED_MEMORY != 0 +static inline uint64 +GET_LINEAR_MEMORY_SIZE(const WASMMemoryInstance *memory) +{ + SHARED_MEMORY_LOCK(memory); + uint64 memory_data_size = BH_ATOMIC_64_LOAD(memory->memory_data_size); + SHARED_MEMORY_UNLOCK(memory); + return memory_data_size; +} +static inline void +SET_LINEAR_MEMORY_SIZE(WASMMemoryInstance *memory, uint64 size) +{ + SHARED_MEMORY_LOCK(memory); + BH_ATOMIC_64_STORE(memory->memory_data_size, size); + SHARED_MEMORY_UNLOCK(memory); +} +#else +#define GET_LINEAR_MEMORY_SIZE(memory) memory->memory_data_size +#define SET_LINEAR_MEMORY_SIZE(memory, size) memory->memory_data_size = size +#endif + +#if WASM_ENABLE_INTERP != 0 +#if WASM_ENABLE_SHARED_HEAP != 0 + +#if WASM_ENABLE_MULTI_MEMORY != 0 +/* Only enable shared heap for the default memory */ +#define is_default_memory (memidx == 0) +#else +#define is_default_memory true +#endif + +#if UINTPTR_MAX == UINT64_MAX +#define get_shared_heap_end_off() module->e->shared_heap_end_off.u64 +#else +#define get_shared_heap_end_off() \ + (uint64)(module->e->shared_heap_end_off.u32[0]) +#endif + +#if WASM_ENABLE_MEMORY64 != 0 +#define shared_heap_is_memory64 is_memory64 +#else +#define shared_heap_is_memory64 false +#endif + +#define app_addr_in_shared_heap(app_addr, bytes) \ + (is_default_memory \ + && is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module, \ + shared_heap_is_memory64, (uint64)app_addr, \ + bytes)) +#define shared_heap_addr_app_to_native(app_addr, native_addr) \ + native_addr = module->e->shared_heap_base_addr_adj + app_addr +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ + if (app_addr_in_shared_heap(app_addr, bytes)) \ + shared_heap_addr_app_to_native(app_addr, native_addr); \ + else + +#else /* else of WASM_ENABLE_SHARED_HEAP != 0 */ +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ +#endif /* end of WASM_ENABLE_INTERP != 0 */ + +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, + bool is_memory64, uint64 app_offset, uint32 bytes); + +WASMSharedHeap * +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args); + +WASMSharedHeap * +wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body); + +WASMSharedHeap * +wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain); + +bool +wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap); +bool +wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap); + +void +wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst); + +void +wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst); + +WASMSharedHeap * +wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm); + +uint64 +wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, + uint64 size, void **p_native_addr); + +void +wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, + uint64 ptr); +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ + +bool +wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, + const MemAllocOption *alloc_option); + +void +wasm_runtime_memory_destroy(void); + +unsigned +wasm_runtime_memory_pool_size(void); + +void +wasm_runtime_set_mem_bound_check_bytes(WASMMemoryInstance *memory, + uint64 memory_data_size); + +void +wasm_runtime_set_enlarge_mem_error_callback( + const enlarge_memory_error_callback_t callback, void *user_data); + +void +wasm_deallocate_linear_memory(WASMMemoryInstance *memory_inst); + +int +wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory, + bool is_memory64, uint64 num_bytes_per_page, + uint64 init_page_count, uint64 max_page_count, + uint64 *memory_data_size); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_MEMORY_H */ diff --git a/wamr/include/wasm_native.h b/wamr/include/wasm_native.h new file mode 100644 index 0000000..5c65eef --- /dev/null +++ b/wamr/include/wasm_native.h @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_NATIVE_H +#define _WASM_NATIVE_H + +#include "bh_common.h" // bh_common.h +#include "wasm_export.h" // ../include/wasm_export.h +#include "wasm.h" // ../interpreter/wasm.h + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct NativeSymbolsNode { + struct NativeSymbolsNode *next; + const char *module_name; + NativeSymbol *native_symbols; + uint32 n_native_symbols; + bool call_conv_raw; +} NativeSymbolsNode, *NativeSymbolsList; + +/** + * Lookup global variable of a given import global + * from libc builtin globals + * + * @param module_name the module name of the import global + * @param global_name the global name of the import global + * @param global return the global data + * + * @param true if success, false otherwise + */ +bool +wasm_native_lookup_libc_builtin_global(const char *module_name, + const char *global_name, + WASMGlobalImport *global); + +/** + * Resolve native symbol in all libraries, including libc-builtin, libc-wasi, + * base lib and extension lib, and user registered natives + * function, which can be auto checked by vm before calling native function + * + * @param module_name the module name of the import function + * @param func_name the function name of the import function + * @param func_type the function prototype of the import function + * @param p_signature output the signature if resolve success + * + * @return the native function pointer if success, NULL otherwise + */ +void * +wasm_native_resolve_symbol(const char *module_name, const char *field_name, + const WASMFuncType *func_type, + const char **p_signature, void **p_attachment, + bool *p_call_conv_raw); + +bool +wasm_native_register_natives(const char *module_name, + NativeSymbol *native_symbols, + uint32 n_native_symbols); + +bool +wasm_native_register_natives_raw(const char *module_name, + NativeSymbol *native_symbols, + uint32 n_native_symbols); + +bool +wasm_native_unregister_natives(const char *module_name, + NativeSymbol *native_symbols); + +#if WASM_ENABLE_MODULE_INST_CONTEXT != 0 +struct WASMModuleInstanceCommon; + +void * +wasm_native_create_context_key( + void (*dtor)(struct WASMModuleInstanceCommon *inst, void *ctx)); + +void +wasm_native_destroy_context_key(void *key); + +void +wasm_native_set_context(struct WASMModuleInstanceCommon *inst, void *key, + void *ctx); +void +wasm_native_set_context_spread(struct WASMModuleInstanceCommon *inst, void *key, + void *ctx); +void * +wasm_native_get_context(struct WASMModuleInstanceCommon *inst, void *key); + +void +wasm_native_call_context_dtors(struct WASMModuleInstanceCommon *inst); + +void +wasm_native_inherit_contexts(struct WASMModuleInstanceCommon *child, + struct WASMModuleInstanceCommon *parent); +#else /* WASM_ENABLE_MODULE_INST_CONTEXT */ +#define wasm_native_call_context_dtors(inst) (void)(inst) +#define wasm_native_inherit_contexts(child, parent) (void)(parent) +#endif /* WASM_ENABLE_MODULE_INST_CONTEXT */ + +bool +wasm_native_init(void); + +void +wasm_native_destroy(void); + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 +void * +wasm_native_lookup_quick_aot_entry(const WASMFuncType *func_type); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_NATIVE_H */ diff --git a/wamr/include/wasm_opcode.h b/wamr/include/wasm_opcode.h new file mode 100644 index 0000000..63f531e --- /dev/null +++ b/wamr/include/wasm_opcode.h @@ -0,0 +1,1050 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_OPCODE_H +#define _WASM_OPCODE_H + +#include "wasm.h" // wasm.h + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum WASMOpcode { + /* control instructions */ + WASM_OP_UNREACHABLE = 0x00, /* unreachable */ + WASM_OP_NOP = 0x01, /* nop */ + WASM_OP_BLOCK = 0x02, /* block */ + WASM_OP_LOOP = 0x03, /* loop */ + WASM_OP_IF = 0x04, /* if */ + WASM_OP_ELSE = 0x05, /* else */ + WASM_OP_TRY = 0x06, /* try */ + WASM_OP_CATCH = 0x07, /* catch* */ + WASM_OP_THROW = 0x08, /* throw of a try catch */ + WASM_OP_RETHROW = 0x09, /* rethrow of a try catch */ + WASM_OP_UNUSED_0x0a = 0x0a, + + WASM_OP_END = 0x0b, /* end */ + WASM_OP_BR = 0x0c, /* br */ + WASM_OP_BR_IF = 0x0d, /* br if */ + WASM_OP_BR_TABLE = 0x0e, /* br table */ + WASM_OP_RETURN = 0x0f, /* return */ + WASM_OP_CALL = 0x10, /* call */ + WASM_OP_CALL_INDIRECT = 0x11, /* call_indirect */ + WASM_OP_RETURN_CALL = 0x12, /* return_call */ + WASM_OP_RETURN_CALL_INDIRECT = 0x13, /* return_call_indirect */ + WASM_OP_CALL_REF = 0x14, /* call_ref */ + WASM_OP_RETURN_CALL_REF = 0x15, /* return_call_ref */ + + WASM_OP_UNUSED_0x16 = 0x16, + WASM_OP_UNUSED_0x17 = 0x17, + + WASM_OP_DELEGATE = 0x18, /* delegate block of the try catch*/ + WASM_OP_CATCH_ALL = 0x19, /* a catch_all handler in a try block */ + + /* parametric instructions */ + WASM_OP_DROP = 0x1a, /* drop */ + WASM_OP_SELECT = 0x1b, /* select */ + WASM_OP_SELECT_T = 0x1c, /* select t */ + + WASM_OP_GET_GLOBAL_64 = 0x1d, + WASM_OP_SET_GLOBAL_64 = 0x1e, + WASM_OP_SET_GLOBAL_AUX_STACK = 0x1f, + + /* variable instructions */ + WASM_OP_GET_LOCAL = 0x20, /* get_local */ + WASM_OP_SET_LOCAL = 0x21, /* set_local */ + WASM_OP_TEE_LOCAL = 0x22, /* tee_local */ + WASM_OP_GET_GLOBAL = 0x23, /* get_global */ + WASM_OP_SET_GLOBAL = 0x24, /* set_global */ + + WASM_OP_TABLE_GET = 0x25, /* table.get */ + WASM_OP_TABLE_SET = 0x26, /* table.set */ + WASM_OP_UNUSED_0x27 = 0x27, + + /* memory instructions */ + WASM_OP_I32_LOAD = 0x28, /* i32.load */ + WASM_OP_I64_LOAD = 0x29, /* i64.load */ + WASM_OP_F32_LOAD = 0x2a, /* f32.load */ + WASM_OP_F64_LOAD = 0x2b, /* f64.load */ + WASM_OP_I32_LOAD8_S = 0x2c, /* i32.load8_s */ + WASM_OP_I32_LOAD8_U = 0x2d, /* i32.load8_u */ + WASM_OP_I32_LOAD16_S = 0x2e, /* i32.load16_s */ + WASM_OP_I32_LOAD16_U = 0x2f, /* i32.load16_u */ + WASM_OP_I64_LOAD8_S = 0x30, /* i64.load8_s */ + WASM_OP_I64_LOAD8_U = 0x31, /* i64.load8_u */ + WASM_OP_I64_LOAD16_S = 0x32, /* i64.load16_s */ + WASM_OP_I64_LOAD16_U = 0x33, /* i64.load16_u */ + WASM_OP_I64_LOAD32_S = 0x34, /* i32.load32_s */ + WASM_OP_I64_LOAD32_U = 0x35, /* i32.load32_u */ + WASM_OP_I32_STORE = 0x36, /* i32.store */ + WASM_OP_I64_STORE = 0x37, /* i64.store */ + WASM_OP_F32_STORE = 0x38, /* f32.store */ + WASM_OP_F64_STORE = 0x39, /* f64.store */ + WASM_OP_I32_STORE8 = 0x3a, /* i32.store8 */ + WASM_OP_I32_STORE16 = 0x3b, /* i32.store16 */ + WASM_OP_I64_STORE8 = 0x3c, /* i64.store8 */ + WASM_OP_I64_STORE16 = 0x3d, /* i64.store16 */ + WASM_OP_I64_STORE32 = 0x3e, /* i64.store32 */ + WASM_OP_MEMORY_SIZE = 0x3f, /* memory.size */ + WASM_OP_MEMORY_GROW = 0x40, /* memory.grow */ + + /* constant instructions */ + WASM_OP_I32_CONST = 0x41, /* i32.const */ + WASM_OP_I64_CONST = 0x42, /* i64.const */ + WASM_OP_F32_CONST = 0x43, /* f32.const */ + WASM_OP_F64_CONST = 0x44, /* f64.const */ + + /* comparison instructions */ + WASM_OP_I32_EQZ = 0x45, /* i32.eqz */ + WASM_OP_I32_EQ = 0x46, /* i32.eq */ + WASM_OP_I32_NE = 0x47, /* i32.ne */ + WASM_OP_I32_LT_S = 0x48, /* i32.lt_s */ + WASM_OP_I32_LT_U = 0x49, /* i32.lt_u */ + WASM_OP_I32_GT_S = 0x4a, /* i32.gt_s */ + WASM_OP_I32_GT_U = 0x4b, /* i32.gt_u */ + WASM_OP_I32_LE_S = 0x4c, /* i32.le_s */ + WASM_OP_I32_LE_U = 0x4d, /* i32.le_u */ + WASM_OP_I32_GE_S = 0x4e, /* i32.ge_s */ + WASM_OP_I32_GE_U = 0x4f, /* i32.ge_u */ + + WASM_OP_I64_EQZ = 0x50, /* i64.eqz */ + WASM_OP_I64_EQ = 0x51, /* i64.eq */ + WASM_OP_I64_NE = 0x52, /* i64.ne */ + WASM_OP_I64_LT_S = 0x53, /* i64.lt_s */ + WASM_OP_I64_LT_U = 0x54, /* i64.lt_u */ + WASM_OP_I64_GT_S = 0x55, /* i64.gt_s */ + WASM_OP_I64_GT_U = 0x56, /* i64.gt_u */ + WASM_OP_I64_LE_S = 0x57, /* i64.le_s */ + WASM_OP_I64_LE_U = 0x58, /* i64.le_u */ + WASM_OP_I64_GE_S = 0x59, /* i64.ge_s */ + WASM_OP_I64_GE_U = 0x5a, /* i64.ge_u */ + + WASM_OP_F32_EQ = 0x5b, /* f32.eq */ + WASM_OP_F32_NE = 0x5c, /* f32.ne */ + WASM_OP_F32_LT = 0x5d, /* f32.lt */ + WASM_OP_F32_GT = 0x5e, /* f32.gt */ + WASM_OP_F32_LE = 0x5f, /* f32.le */ + WASM_OP_F32_GE = 0x60, /* f32.ge */ + + WASM_OP_F64_EQ = 0x61, /* f64.eq */ + WASM_OP_F64_NE = 0x62, /* f64.ne */ + WASM_OP_F64_LT = 0x63, /* f64.lt */ + WASM_OP_F64_GT = 0x64, /* f64.gt */ + WASM_OP_F64_LE = 0x65, /* f64.le */ + WASM_OP_F64_GE = 0x66, /* f64.ge */ + + /* numeric operators */ + WASM_OP_I32_CLZ = 0x67, /* i32.clz */ + WASM_OP_I32_CTZ = 0x68, /* i32.ctz */ + WASM_OP_I32_POPCNT = 0x69, /* i32.popcnt */ + WASM_OP_I32_ADD = 0x6a, /* i32.add */ + WASM_OP_I32_SUB = 0x6b, /* i32.sub */ + WASM_OP_I32_MUL = 0x6c, /* i32.mul */ + WASM_OP_I32_DIV_S = 0x6d, /* i32.div_s */ + WASM_OP_I32_DIV_U = 0x6e, /* i32.div_u */ + WASM_OP_I32_REM_S = 0x6f, /* i32.rem_s */ + WASM_OP_I32_REM_U = 0x70, /* i32.rem_u */ + WASM_OP_I32_AND = 0x71, /* i32.and */ + WASM_OP_I32_OR = 0x72, /* i32.or */ + WASM_OP_I32_XOR = 0x73, /* i32.xor */ + WASM_OP_I32_SHL = 0x74, /* i32.shl */ + WASM_OP_I32_SHR_S = 0x75, /* i32.shr_s */ + WASM_OP_I32_SHR_U = 0x76, /* i32.shr_u */ + WASM_OP_I32_ROTL = 0x77, /* i32.rotl */ + WASM_OP_I32_ROTR = 0x78, /* i32.rotr */ + + WASM_OP_I64_CLZ = 0x79, /* i64.clz */ + WASM_OP_I64_CTZ = 0x7a, /* i64.ctz */ + WASM_OP_I64_POPCNT = 0x7b, /* i64.popcnt */ + WASM_OP_I64_ADD = 0x7c, /* i64.add */ + WASM_OP_I64_SUB = 0x7d, /* i64.sub */ + WASM_OP_I64_MUL = 0x7e, /* i64.mul */ + WASM_OP_I64_DIV_S = 0x7f, /* i64.div_s */ + WASM_OP_I64_DIV_U = 0x80, /* i64.div_u */ + WASM_OP_I64_REM_S = 0x81, /* i64.rem_s */ + WASM_OP_I64_REM_U = 0x82, /* i64.rem_u */ + WASM_OP_I64_AND = 0x83, /* i64.and */ + WASM_OP_I64_OR = 0x84, /* i64.or */ + WASM_OP_I64_XOR = 0x85, /* i64.xor */ + WASM_OP_I64_SHL = 0x86, /* i64.shl */ + WASM_OP_I64_SHR_S = 0x87, /* i64.shr_s */ + WASM_OP_I64_SHR_U = 0x88, /* i64.shr_u */ + WASM_OP_I64_ROTL = 0x89, /* i64.rotl */ + WASM_OP_I64_ROTR = 0x8a, /* i64.rotr */ + + WASM_OP_F32_ABS = 0x8b, /* f32.abs */ + WASM_OP_F32_NEG = 0x8c, /* f32.neg */ + WASM_OP_F32_CEIL = 0x8d, /* f32.ceil */ + WASM_OP_F32_FLOOR = 0x8e, /* f32.floor */ + WASM_OP_F32_TRUNC = 0x8f, /* f32.trunc */ + WASM_OP_F32_NEAREST = 0x90, /* f32.nearest */ + WASM_OP_F32_SQRT = 0x91, /* f32.sqrt */ + WASM_OP_F32_ADD = 0x92, /* f32.add */ + WASM_OP_F32_SUB = 0x93, /* f32.sub */ + WASM_OP_F32_MUL = 0x94, /* f32.mul */ + WASM_OP_F32_DIV = 0x95, /* f32.div */ + WASM_OP_F32_MIN = 0x96, /* f32.min */ + WASM_OP_F32_MAX = 0x97, /* f32.max */ + WASM_OP_F32_COPYSIGN = 0x98, /* f32.copysign */ + + WASM_OP_F64_ABS = 0x99, /* f64.abs */ + WASM_OP_F64_NEG = 0x9a, /* f64.neg */ + WASM_OP_F64_CEIL = 0x9b, /* f64.ceil */ + WASM_OP_F64_FLOOR = 0x9c, /* f64.floor */ + WASM_OP_F64_TRUNC = 0x9d, /* f64.trunc */ + WASM_OP_F64_NEAREST = 0x9e, /* f64.nearest */ + WASM_OP_F64_SQRT = 0x9f, /* f64.sqrt */ + WASM_OP_F64_ADD = 0xa0, /* f64.add */ + WASM_OP_F64_SUB = 0xa1, /* f64.sub */ + WASM_OP_F64_MUL = 0xa2, /* f64.mul */ + WASM_OP_F64_DIV = 0xa3, /* f64.div */ + WASM_OP_F64_MIN = 0xa4, /* f64.min */ + WASM_OP_F64_MAX = 0xa5, /* f64.max */ + WASM_OP_F64_COPYSIGN = 0xa6, /* f64.copysign */ + + /* conversions */ + WASM_OP_I32_WRAP_I64 = 0xa7, /* i32.wrap/i64 */ + WASM_OP_I32_TRUNC_S_F32 = 0xa8, /* i32.trunc_s/f32 */ + WASM_OP_I32_TRUNC_U_F32 = 0xa9, /* i32.trunc_u/f32 */ + WASM_OP_I32_TRUNC_S_F64 = 0xaa, /* i32.trunc_s/f64 */ + WASM_OP_I32_TRUNC_U_F64 = 0xab, /* i32.trunc_u/f64 */ + + WASM_OP_I64_EXTEND_S_I32 = 0xac, /* i64.extend_s/i32 */ + WASM_OP_I64_EXTEND_U_I32 = 0xad, /* i64.extend_u/i32 */ + WASM_OP_I64_TRUNC_S_F32 = 0xae, /* i64.trunc_s/f32 */ + WASM_OP_I64_TRUNC_U_F32 = 0xaf, /* i64.trunc_u/f32 */ + WASM_OP_I64_TRUNC_S_F64 = 0xb0, /* i64.trunc_s/f64 */ + WASM_OP_I64_TRUNC_U_F64 = 0xb1, /* i64.trunc_u/f64 */ + + WASM_OP_F32_CONVERT_S_I32 = 0xb2, /* f32.convert_s/i32 */ + WASM_OP_F32_CONVERT_U_I32 = 0xb3, /* f32.convert_u/i32 */ + WASM_OP_F32_CONVERT_S_I64 = 0xb4, /* f32.convert_s/i64 */ + WASM_OP_F32_CONVERT_U_I64 = 0xb5, /* f32.convert_u/i64 */ + WASM_OP_F32_DEMOTE_F64 = 0xb6, /* f32.demote/f64 */ + + WASM_OP_F64_CONVERT_S_I32 = 0xb7, /* f64.convert_s/i32 */ + WASM_OP_F64_CONVERT_U_I32 = 0xb8, /* f64.convert_u/i32 */ + WASM_OP_F64_CONVERT_S_I64 = 0xb9, /* f64.convert_s/i64 */ + WASM_OP_F64_CONVERT_U_I64 = 0xba, /* f64.convert_u/i64 */ + WASM_OP_F64_PROMOTE_F32 = 0xbb, /* f64.promote/f32 */ + + /* reinterpretations */ + WASM_OP_I32_REINTERPRET_F32 = 0xbc, /* i32.reinterpret/f32 */ + WASM_OP_I64_REINTERPRET_F64 = 0xbd, /* i64.reinterpret/f64 */ + WASM_OP_F32_REINTERPRET_I32 = 0xbe, /* f32.reinterpret/i32 */ + WASM_OP_F64_REINTERPRET_I64 = 0xbf, /* f64.reinterpret/i64 */ + + WASM_OP_I32_EXTEND8_S = 0xc0, /* i32.extend8_s */ + WASM_OP_I32_EXTEND16_S = 0xc1, /* i32.extend16_s */ + WASM_OP_I64_EXTEND8_S = 0xc2, /* i64.extend8_s */ + WASM_OP_I64_EXTEND16_S = 0xc3, /* i64.extend16_s */ + WASM_OP_I64_EXTEND32_S = 0xc4, /* i64.extend32_s */ + + /* drop/select specified types*/ + WASM_OP_DROP_64 = 0xc5, + WASM_OP_SELECT_64 = 0xc6, + + /* extend op code */ + EXT_OP_GET_LOCAL_FAST = 0xc7, + EXT_OP_SET_LOCAL_FAST_I64 = 0xc8, + EXT_OP_SET_LOCAL_FAST = 0xc9, + EXT_OP_TEE_LOCAL_FAST = 0xca, + EXT_OP_TEE_LOCAL_FAST_I64 = 0xcb, + EXT_OP_COPY_STACK_TOP = 0xcc, + EXT_OP_COPY_STACK_TOP_I64 = 0xcd, + EXT_OP_COPY_STACK_VALUES = 0xce, + + WASM_OP_IMPDEP = 0xcf, + + WASM_OP_REF_NULL = 0xd0, /* ref.null */ + WASM_OP_REF_IS_NULL = 0xd1, /* ref.is_null */ + WASM_OP_REF_FUNC = 0xd2, /* ref.func */ + WASM_OP_REF_EQ = 0xd3, /* ref.eq */ + WASM_OP_REF_AS_NON_NULL = 0xd4, /* ref.as_non_null */ + WASM_OP_BR_ON_NULL = 0xd5, /* br_on_null */ + WASM_OP_BR_ON_NON_NULL = 0xd6, /* br_on_non_null */ + + EXT_OP_BLOCK = 0xd7, /* block with blocktype */ + EXT_OP_LOOP = 0xd8, /* loop with blocktype */ + EXT_OP_IF = 0xd9, /* if with blocktype */ + EXT_OP_BR_TABLE_CACHE = 0xda, /* br_table from cache */ + + EXT_OP_TRY = 0xdb, /* try block with blocktype */ + +#if WASM_ENABLE_DEBUG_INTERP != 0 + DEBUG_OP_BREAK = 0xdc, /* debug break point */ +#endif + +#if WASM_ENABLE_SIMDE != 0 + EXT_OP_SET_LOCAL_FAST_V128 = 0xdd, + EXT_OP_TEE_LOCAL_FAST_V128 = 0xde, + EXT_OP_COPY_STACK_TOP_V128 = 0xdf, + WASM_OP_GET_GLOBAL_V128 = 0xe0, + WASM_OP_SET_GLOBAL_V128 = 0xe1, + WASM_OP_SELECT_128 = 0xe2, +#endif + + /* Post-MVP extend op prefix */ + WASM_OP_GC_PREFIX = 0xfb, + WASM_OP_MISC_PREFIX = 0xfc, + WASM_OP_SIMD_PREFIX = 0xfd, + WASM_OP_ATOMIC_PREFIX = 0xfe, +} WASMOpcode; + +typedef enum WASMGCEXTOpcode { + WASM_OP_STRUCT_NEW = 0x00, /* struct.new */ + WASM_OP_STRUCT_NEW_DEFAULT = 0x01, /* struct.new_default */ + WASM_OP_STRUCT_GET = 0x02, /* struct.get */ + WASM_OP_STRUCT_GET_S = 0x03, /* struct.get_s */ + WASM_OP_STRUCT_GET_U = 0x04, /* struct.get_u */ + WASM_OP_STRUCT_SET = 0x05, /* struct.set */ + + WASM_OP_ARRAY_NEW = 0x06, /* array.new */ + WASM_OP_ARRAY_NEW_DEFAULT = 0x07, /* array.new_default */ + WASM_OP_ARRAY_NEW_FIXED = 0x08, /* array.new_fixed */ + WASM_OP_ARRAY_NEW_DATA = 0x09, /* array.new_data */ + WASM_OP_ARRAY_NEW_ELEM = 0x0A, /* array.new_elem */ + WASM_OP_ARRAY_GET = 0x0B, /* array.get */ + WASM_OP_ARRAY_GET_S = 0x0C, /* array.get_s */ + WASM_OP_ARRAY_GET_U = 0x0D, /* array.get_u */ + WASM_OP_ARRAY_SET = 0x0E, /* array.set */ + WASM_OP_ARRAY_LEN = 0x0F, /* array.len */ + WASM_OP_ARRAY_FILL = 0x10, /* array.fill */ + WASM_OP_ARRAY_COPY = 0x11, /* array.copy */ + WASM_OP_ARRAY_INIT_DATA = 0x12, + /* array.init_data */ /* TODO */ + WASM_OP_ARRAY_INIT_ELEM = 0x13, + /* array.init_elem */ /* TODO */ + + WASM_OP_REF_TEST = 0x14, /* ref.test */ + WASM_OP_REF_TEST_NULLABLE = 0x15, /* ref.test_nullable */ + WASM_OP_REF_CAST = 0x16, /* ref.cast */ + WASM_OP_REF_CAST_NULLABLE = 0x17, /* ref.cast_nullable */ + + WASM_OP_BR_ON_CAST = 0x18, /* br_on_cast */ + WASM_OP_BR_ON_CAST_FAIL = 0x19, /* br_on_cast_fail */ + + WASM_OP_ANY_CONVERT_EXTERN = 0x1A, /* any.convert_extern */ + WASM_OP_EXTERN_CONVERT_ANY = 0x1B, /* extern.covert_any */ + + WASM_OP_REF_I31 = 0x1C, /* ref.i31 */ + WASM_OP_I31_GET_S = 0x1D, /* i31.get_s */ + WASM_OP_I31_GET_U = 0x1E, /* i31.get_u */ + + /* stringref related opcodes */ + WASM_OP_STRING_NEW_UTF8 = 0x80, /* string.new_utf8 */ + WASM_OP_STRING_NEW_WTF16 = 0x81, /* string.new_wtf16 */ + WASM_OP_STRING_CONST = 0x82, /* string.const */ + WASM_OP_STRING_MEASURE_UTF8 = 0x83, /* string.measure_utf8 */ + WASM_OP_STRING_MEASURE_WTF8 = 0x84, /* string.measure_wtf8 */ + WASM_OP_STRING_MEASURE_WTF16 = 0x85, /* string.measure_wtf16 */ + WASM_OP_STRING_ENCODE_UTF8 = 0x86, /* string.encode_utf8 */ + WASM_OP_STRING_ENCODE_WTF16 = 0x87, /* string.encode_wtf16 */ + WASM_OP_STRING_CONCAT = 0x88, /* string.concat */ + WASM_OP_STRING_EQ = 0x89, /* string.eq */ + WASM_OP_STRING_IS_USV_SEQUENCE = 0x8a, /* string.is_usv_sequence */ + WASM_OP_STRING_NEW_LOSSY_UTF8 = 0x8b, /* string.new_lossy_utf8 */ + WASM_OP_STRING_NEW_WTF8 = 0x8c, /* string.new_wtf8 */ + WASM_OP_STRING_ENCODE_LOSSY_UTF8 = 0x8d, /* string.encode_lossy_utf8 */ + WASM_OP_STRING_ENCODE_WTF8 = 0x8e, /* string.encode_wtf8 */ + + WASM_OP_STRING_AS_WTF8 = 0x90, /* string.as_wtf8 */ + WASM_OP_STRINGVIEW_WTF8_ADVANCE = 0x91, /* stringview_wtf8.advance */ + WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8 = + 0x92, /* stringview_wtf8.encode_utf8 */ + WASM_OP_STRINGVIEW_WTF8_SLICE = 0x93, /* stringview_wtf8.slice */ + WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8 = + 0x94, /* stringview_wtf8.encode_lossy_utf8 */ + WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8 = + 0x95, /* stringview_wtf8.encode_wtf8 */ + + WASM_OP_STRING_AS_WTF16 = 0x98, /* string.as_wtf16 */ + WASM_OP_STRINGVIEW_WTF16_LENGTH = 0x99, /* stringview_wtf16.length */ + WASM_OP_STRINGVIEW_WTF16_GET_CODEUNIT = + 0x9a, /* stringview_wtf16.get_codeunit */ + WASM_OP_STRINGVIEW_WTF16_ENCODE = 0x9b, /* stringview_wtf16.encode */ + WASM_OP_STRINGVIEW_WTF16_SLICE = 0x9c, /* stringview_wtf16.slice */ + + WASM_OP_STRING_AS_ITER = 0xa0, /* string.as_iter */ + WASM_OP_STRINGVIEW_ITER_NEXT = 0xa1, /* stringview_iter.next */ + WASM_OP_STRINGVIEW_ITER_ADVANCE = 0xa2, /* stringview_iter.advance */ + WASM_OP_STRINGVIEW_ITER_REWIND = 0xa3, /* stringview_iter.rewind */ + WASM_OP_STRINGVIEW_ITER_SLICE = 0xa4, /* stringview_iter.slice */ + + WASM_OP_STRING_NEW_UTF8_ARRAY = 0xb0, /* string.new_utf8_array */ + WASM_OP_STRING_NEW_WTF16_ARRAY = 0xb1, /* string.new_wtf16_array */ + WASM_OP_STRING_ENCODE_UTF8_ARRAY = 0xb2, /* string.encode_utf8_array */ + WASM_OP_STRING_ENCODE_WTF16_ARRAY = 0xb3, /* string.encode_wtf16_array */ + WASM_OP_STRING_NEW_LOSSY_UTF8_ARRAY = + 0xb4, /* string.new_lossy_utf8_array */ + WASM_OP_STRING_NEW_WTF8_ARRAY = 0xb5, /* string.new_wtf8_array */ + WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY = + 0xb6, /* string.encode_lossy_utf8_array */ + WASM_OP_STRING_ENCODE_WTF8_ARRAY = 0xb7, /* string.encode_wtf8_array */ +} WASMGCEXTOpcode; + +typedef enum WASMMiscEXTOpcode { + WASM_OP_I32_TRUNC_SAT_S_F32 = 0x00, + WASM_OP_I32_TRUNC_SAT_U_F32 = 0x01, + WASM_OP_I32_TRUNC_SAT_S_F64 = 0x02, + WASM_OP_I32_TRUNC_SAT_U_F64 = 0x03, + WASM_OP_I64_TRUNC_SAT_S_F32 = 0x04, + WASM_OP_I64_TRUNC_SAT_U_F32 = 0x05, + WASM_OP_I64_TRUNC_SAT_S_F64 = 0x06, + WASM_OP_I64_TRUNC_SAT_U_F64 = 0x07, + WASM_OP_MEMORY_INIT = 0x08, + WASM_OP_DATA_DROP = 0x09, + WASM_OP_MEMORY_COPY = 0x0a, + WASM_OP_MEMORY_FILL = 0x0b, + WASM_OP_TABLE_INIT = 0x0c, + WASM_OP_ELEM_DROP = 0x0d, + WASM_OP_TABLE_COPY = 0x0e, + WASM_OP_TABLE_GROW = 0x0f, + WASM_OP_TABLE_SIZE = 0x10, + WASM_OP_TABLE_FILL = 0x11, +} WASMMiscEXTOpcode; + +typedef enum WASMSimdEXTOpcode { + /* memory instruction */ + SIMD_v128_load = 0x00, + SIMD_v128_load8x8_s = 0x01, + SIMD_v128_load8x8_u = 0x02, + SIMD_v128_load16x4_s = 0x03, + SIMD_v128_load16x4_u = 0x04, + SIMD_v128_load32x2_s = 0x05, + SIMD_v128_load32x2_u = 0x06, + SIMD_v128_load8_splat = 0x07, + SIMD_v128_load16_splat = 0x08, + SIMD_v128_load32_splat = 0x09, + SIMD_v128_load64_splat = 0x0a, + SIMD_v128_store = 0x0b, + + /* basic operation */ + SIMD_v128_const = 0x0c, + SIMD_v8x16_shuffle = 0x0d, + SIMD_v8x16_swizzle = 0x0e, + + /* splat operation */ + SIMD_i8x16_splat = 0x0f, + SIMD_i16x8_splat = 0x10, + SIMD_i32x4_splat = 0x11, + SIMD_i64x2_splat = 0x12, + SIMD_f32x4_splat = 0x13, + SIMD_f64x2_splat = 0x14, + + /* lane operation */ + SIMD_i8x16_extract_lane_s = 0x15, + SIMD_i8x16_extract_lane_u = 0x16, + SIMD_i8x16_replace_lane = 0x17, + SIMD_i16x8_extract_lane_s = 0x18, + SIMD_i16x8_extract_lane_u = 0x19, + SIMD_i16x8_replace_lane = 0x1a, + SIMD_i32x4_extract_lane = 0x1b, + SIMD_i32x4_replace_lane = 0x1c, + SIMD_i64x2_extract_lane = 0x1d, + SIMD_i64x2_replace_lane = 0x1e, + SIMD_f32x4_extract_lane = 0x1f, + SIMD_f32x4_replace_lane = 0x20, + SIMD_f64x2_extract_lane = 0x21, + SIMD_f64x2_replace_lane = 0x22, + + /* i8x16 compare operation */ + SIMD_i8x16_eq = 0x23, + SIMD_i8x16_ne = 0x24, + SIMD_i8x16_lt_s = 0x25, + SIMD_i8x16_lt_u = 0x26, + SIMD_i8x16_gt_s = 0x27, + SIMD_i8x16_gt_u = 0x28, + SIMD_i8x16_le_s = 0x29, + SIMD_i8x16_le_u = 0x2a, + SIMD_i8x16_ge_s = 0x2b, + SIMD_i8x16_ge_u = 0x2c, + + /* i16x8 compare operation */ + SIMD_i16x8_eq = 0x2d, + SIMD_i16x8_ne = 0x2e, + SIMD_i16x8_lt_s = 0x2f, + SIMD_i16x8_lt_u = 0x30, + SIMD_i16x8_gt_s = 0x31, + SIMD_i16x8_gt_u = 0x32, + SIMD_i16x8_le_s = 0x33, + SIMD_i16x8_le_u = 0x34, + SIMD_i16x8_ge_s = 0x35, + SIMD_i16x8_ge_u = 0x36, + + /* i32x4 compare operation */ + SIMD_i32x4_eq = 0x37, + SIMD_i32x4_ne = 0x38, + SIMD_i32x4_lt_s = 0x39, + SIMD_i32x4_lt_u = 0x3a, + SIMD_i32x4_gt_s = 0x3b, + SIMD_i32x4_gt_u = 0x3c, + SIMD_i32x4_le_s = 0x3d, + SIMD_i32x4_le_u = 0x3e, + SIMD_i32x4_ge_s = 0x3f, + SIMD_i32x4_ge_u = 0x40, + + /* f32x4 compare operation */ + SIMD_f32x4_eq = 0x41, + SIMD_f32x4_ne = 0x42, + SIMD_f32x4_lt = 0x43, + SIMD_f32x4_gt = 0x44, + SIMD_f32x4_le = 0x45, + SIMD_f32x4_ge = 0x46, + + /* f64x2 compare operation */ + SIMD_f64x2_eq = 0x47, + SIMD_f64x2_ne = 0x48, + SIMD_f64x2_lt = 0x49, + SIMD_f64x2_gt = 0x4a, + SIMD_f64x2_le = 0x4b, + SIMD_f64x2_ge = 0x4c, + + /* v128 operation */ + SIMD_v128_not = 0x4d, + SIMD_v128_and = 0x4e, + SIMD_v128_andnot = 0x4f, + SIMD_v128_or = 0x50, + SIMD_v128_xor = 0x51, + SIMD_v128_bitselect = 0x52, + SIMD_v128_any_true = 0x53, + + /* Load Lane Operation */ + SIMD_v128_load8_lane = 0x54, + SIMD_v128_load16_lane = 0x55, + SIMD_v128_load32_lane = 0x56, + SIMD_v128_load64_lane = 0x57, + SIMD_v128_store8_lane = 0x58, + SIMD_v128_store16_lane = 0x59, + SIMD_v128_store32_lane = 0x5a, + SIMD_v128_store64_lane = 0x5b, + SIMD_v128_load32_zero = 0x5c, + SIMD_v128_load64_zero = 0x5d, + + /* Float conversion */ + SIMD_f32x4_demote_f64x2_zero = 0x5e, + SIMD_f64x2_promote_low_f32x4_zero = 0x5f, + + /* i8x16 Operation */ + SIMD_i8x16_abs = 0x60, + SIMD_i8x16_neg = 0x61, + SIMD_i8x16_popcnt = 0x62, + SIMD_i8x16_all_true = 0x63, + SIMD_i8x16_bitmask = 0x64, + SIMD_i8x16_narrow_i16x8_s = 0x65, + SIMD_i8x16_narrow_i16x8_u = 0x66, + SIMD_f32x4_ceil = 0x67, + SIMD_f32x4_floor = 0x68, + SIMD_f32x4_trunc = 0x69, + SIMD_f32x4_nearest = 0x6a, + SIMD_i8x16_shl = 0x6b, + SIMD_i8x16_shr_s = 0x6c, + SIMD_i8x16_shr_u = 0x6d, + SIMD_i8x16_add = 0x6e, + SIMD_i8x16_add_sat_s = 0x6f, + SIMD_i8x16_add_sat_u = 0x70, + SIMD_i8x16_sub = 0x71, + SIMD_i8x16_sub_sat_s = 0x72, + SIMD_i8x16_sub_sat_u = 0x73, + SIMD_f64x2_ceil = 0x74, + SIMD_f64x2_floor = 0x75, + SIMD_i8x16_min_s = 0x76, + SIMD_i8x16_min_u = 0x77, + SIMD_i8x16_max_s = 0x78, + SIMD_i8x16_max_u = 0x79, + SIMD_f64x2_trunc = 0x7a, + SIMD_i8x16_avgr_u = 0x7b, + SIMD_i16x8_extadd_pairwise_i8x16_s = 0x7c, + SIMD_i16x8_extadd_pairwise_i8x16_u = 0x7d, + SIMD_i32x4_extadd_pairwise_i16x8_s = 0x7e, + SIMD_i32x4_extadd_pairwise_i16x8_u = 0x7f, + + /* i16x8 operation */ + SIMD_i16x8_abs = 0x80, + SIMD_i16x8_neg = 0x81, + SIMD_i16x8_q15mulr_sat_s = 0x82, + SIMD_i16x8_all_true = 0x83, + SIMD_i16x8_bitmask = 0x84, + SIMD_i16x8_narrow_i32x4_s = 0x85, + SIMD_i16x8_narrow_i32x4_u = 0x86, + SIMD_i16x8_extend_low_i8x16_s = 0x87, + SIMD_i16x8_extend_high_i8x16_s = 0x88, + SIMD_i16x8_extend_low_i8x16_u = 0x89, + SIMD_i16x8_extend_high_i8x16_u = 0x8a, + SIMD_i16x8_shl = 0x8b, + SIMD_i16x8_shr_s = 0x8c, + SIMD_i16x8_shr_u = 0x8d, + SIMD_i16x8_add = 0x8e, + SIMD_i16x8_add_sat_s = 0x8f, + SIMD_i16x8_add_sat_u = 0x90, + SIMD_i16x8_sub = 0x91, + SIMD_i16x8_sub_sat_s = 0x92, + SIMD_i16x8_sub_sat_u = 0x93, + SIMD_f64x2_nearest = 0x94, + SIMD_i16x8_mul = 0x95, + SIMD_i16x8_min_s = 0x96, + SIMD_i16x8_min_u = 0x97, + SIMD_i16x8_max_s = 0x98, + SIMD_i16x8_max_u = 0x99, + /* placeholder = 0x9a */ + SIMD_i16x8_avgr_u = 0x9b, + SIMD_i16x8_extmul_low_i8x16_s = 0x9c, + SIMD_i16x8_extmul_high_i8x16_s = 0x9d, + SIMD_i16x8_extmul_low_i8x16_u = 0x9e, + SIMD_i16x8_extmul_high_i8x16_u = 0x9f, + + /* i32x4 operation */ + SIMD_i32x4_abs = 0xa0, + SIMD_i32x4_neg = 0xa1, + /* placeholder = 0xa2 */ + SIMD_i32x4_all_true = 0xa3, + SIMD_i32x4_bitmask = 0xa4, + /* placeholder = 0xa5 */ + /* placeholder = 0xa6 */ + SIMD_i32x4_extend_low_i16x8_s = 0xa7, + SIMD_i32x4_extend_high_i16x8_s = 0xa8, + SIMD_i32x4_extend_low_i16x8_u = 0xa9, + SIMD_i32x4_extend_high_i16x8_u = 0xaa, + SIMD_i32x4_shl = 0xab, + SIMD_i32x4_shr_s = 0xac, + SIMD_i32x4_shr_u = 0xad, + SIMD_i32x4_add = 0xae, + /* placeholder = 0xaf */ + /* placeholder = 0xb0 */ + SIMD_i32x4_sub = 0xb1, + /* placeholder = 0xb2 */ + /* placeholder = 0xb3 */ + /* placeholder = 0xb4 */ + SIMD_i32x4_mul = 0xb5, + SIMD_i32x4_min_s = 0xb6, + SIMD_i32x4_min_u = 0xb7, + SIMD_i32x4_max_s = 0xb8, + SIMD_i32x4_max_u = 0xb9, + SIMD_i32x4_dot_i16x8_s = 0xba, + /* placeholder = 0xbb */ + SIMD_i32x4_extmul_low_i16x8_s = 0xbc, + SIMD_i32x4_extmul_high_i16x8_s = 0xbd, + SIMD_i32x4_extmul_low_i16x8_u = 0xbe, + SIMD_i32x4_extmul_high_i16x8_u = 0xbf, + + /* i64x2 operation */ + SIMD_i64x2_abs = 0xc0, + SIMD_i64x2_neg = 0xc1, + /* placeholder = 0xc2 */ + SIMD_i64x2_all_true = 0xc3, + SIMD_i64x2_bitmask = 0xc4, + /* placeholder = 0xc5 */ + /* placeholder = 0xc6 */ + SIMD_i64x2_extend_low_i32x4_s = 0xc7, + SIMD_i64x2_extend_high_i32x4_s = 0xc8, + SIMD_i64x2_extend_low_i32x4_u = 0xc9, + SIMD_i64x2_extend_high_i32x4_u = 0xca, + SIMD_i64x2_shl = 0xcb, + SIMD_i64x2_shr_s = 0xcc, + SIMD_i64x2_shr_u = 0xcd, + SIMD_i64x2_add = 0xce, + /* placeholder = 0xcf */ + /* placeholder = 0xd0 */ + SIMD_i64x2_sub = 0xd1, + /* placeholder = 0xd2 */ + /* placeholder = 0xd3 */ + /* placeholder = 0xd4 */ + SIMD_i64x2_mul = 0xd5, + SIMD_i64x2_eq = 0xd6, + SIMD_i64x2_ne = 0xd7, + SIMD_i64x2_lt_s = 0xd8, + SIMD_i64x2_gt_s = 0xd9, + SIMD_i64x2_le_s = 0xda, + SIMD_i64x2_ge_s = 0xdb, + SIMD_i64x2_extmul_low_i32x4_s = 0xdc, + SIMD_i64x2_extmul_high_i32x4_s = 0xdd, + SIMD_i64x2_extmul_low_i32x4_u = 0xde, + SIMD_i64x2_extmul_high_i32x4_u = 0xdf, + + /* f32x4 operation */ + SIMD_f32x4_abs = 0xe0, + SIMD_f32x4_neg = 0xe1, + /* placeholder = 0xe2 */ + SIMD_f32x4_sqrt = 0xe3, + SIMD_f32x4_add = 0xe4, + SIMD_f32x4_sub = 0xe5, + SIMD_f32x4_mul = 0xe6, + SIMD_f32x4_div = 0xe7, + SIMD_f32x4_min = 0xe8, + SIMD_f32x4_max = 0xe9, + SIMD_f32x4_pmin = 0xea, + SIMD_f32x4_pmax = 0xeb, + + /* f64x2 operation */ + SIMD_f64x2_abs = 0xec, + SIMD_f64x2_neg = 0xed, + /* placeholder = 0xee */ + SIMD_f64x2_sqrt = 0xef, + SIMD_f64x2_add = 0xf0, + SIMD_f64x2_sub = 0xf1, + SIMD_f64x2_mul = 0xf2, + SIMD_f64x2_div = 0xf3, + SIMD_f64x2_min = 0xf4, + SIMD_f64x2_max = 0xf5, + SIMD_f64x2_pmin = 0xf6, + SIMD_f64x2_pmax = 0xf7, + + /* conversion operation */ + SIMD_i32x4_trunc_sat_f32x4_s = 0xf8, + SIMD_i32x4_trunc_sat_f32x4_u = 0xf9, + SIMD_f32x4_convert_i32x4_s = 0xfa, + SIMD_f32x4_convert_i32x4_u = 0xfb, + SIMD_i32x4_trunc_sat_f64x2_s_zero = 0xfc, + SIMD_i32x4_trunc_sat_f64x2_u_zero = 0xfd, + SIMD_f64x2_convert_low_i32x4_s = 0xfe, + SIMD_f64x2_convert_low_i32x4_u = 0xff, +} WASMSimdEXTOpcode; + +typedef enum WASMAtomicEXTOpcode { + /* atomic wait and notify */ + WASM_OP_ATOMIC_NOTIFY = 0x00, + WASM_OP_ATOMIC_WAIT32 = 0x01, + WASM_OP_ATOMIC_WAIT64 = 0x02, + WASM_OP_ATOMIC_FENCE = 0x03, + /* atomic load and store */ + WASM_OP_ATOMIC_I32_LOAD = 0x10, + WASM_OP_ATOMIC_I64_LOAD = 0x11, + WASM_OP_ATOMIC_I32_LOAD8_U = 0x12, + WASM_OP_ATOMIC_I32_LOAD16_U = 0x13, + WASM_OP_ATOMIC_I64_LOAD8_U = 0x14, + WASM_OP_ATOMIC_I64_LOAD16_U = 0x15, + WASM_OP_ATOMIC_I64_LOAD32_U = 0x16, + WASM_OP_ATOMIC_I32_STORE = 0x17, + WASM_OP_ATOMIC_I64_STORE = 0x18, + WASM_OP_ATOMIC_I32_STORE8 = 0x19, + WASM_OP_ATOMIC_I32_STORE16 = 0x1a, + WASM_OP_ATOMIC_I64_STORE8 = 0x1b, + WASM_OP_ATOMIC_I64_STORE16 = 0x1c, + WASM_OP_ATOMIC_I64_STORE32 = 0x1d, + /* atomic add */ + WASM_OP_ATOMIC_RMW_I32_ADD = 0x1e, + WASM_OP_ATOMIC_RMW_I64_ADD = 0x1f, + WASM_OP_ATOMIC_RMW_I32_ADD8_U = 0x20, + WASM_OP_ATOMIC_RMW_I32_ADD16_U = 0x21, + WASM_OP_ATOMIC_RMW_I64_ADD8_U = 0x22, + WASM_OP_ATOMIC_RMW_I64_ADD16_U = 0x23, + WASM_OP_ATOMIC_RMW_I64_ADD32_U = 0x24, + /* atomic sub */ + WASM_OP_ATOMIC_RMW_I32_SUB = 0x25, + WASM_OP_ATOMIC_RMW_I64_SUB = 0x26, + WASM_OP_ATOMIC_RMW_I32_SUB8_U = 0x27, + WASM_OP_ATOMIC_RMW_I32_SUB16_U = 0x28, + WASM_OP_ATOMIC_RMW_I64_SUB8_U = 0x29, + WASM_OP_ATOMIC_RMW_I64_SUB16_U = 0x2a, + WASM_OP_ATOMIC_RMW_I64_SUB32_U = 0x2b, + /* atomic and */ + WASM_OP_ATOMIC_RMW_I32_AND = 0x2c, + WASM_OP_ATOMIC_RMW_I64_AND = 0x2d, + WASM_OP_ATOMIC_RMW_I32_AND8_U = 0x2e, + WASM_OP_ATOMIC_RMW_I32_AND16_U = 0x2f, + WASM_OP_ATOMIC_RMW_I64_AND8_U = 0x30, + WASM_OP_ATOMIC_RMW_I64_AND16_U = 0x31, + WASM_OP_ATOMIC_RMW_I64_AND32_U = 0x32, + /* atomic or */ + WASM_OP_ATOMIC_RMW_I32_OR = 0x33, + WASM_OP_ATOMIC_RMW_I64_OR = 0x34, + WASM_OP_ATOMIC_RMW_I32_OR8_U = 0x35, + WASM_OP_ATOMIC_RMW_I32_OR16_U = 0x36, + WASM_OP_ATOMIC_RMW_I64_OR8_U = 0x37, + WASM_OP_ATOMIC_RMW_I64_OR16_U = 0x38, + WASM_OP_ATOMIC_RMW_I64_OR32_U = 0x39, + /* atomic xor */ + WASM_OP_ATOMIC_RMW_I32_XOR = 0x3a, + WASM_OP_ATOMIC_RMW_I64_XOR = 0x3b, + WASM_OP_ATOMIC_RMW_I32_XOR8_U = 0x3c, + WASM_OP_ATOMIC_RMW_I32_XOR16_U = 0x3d, + WASM_OP_ATOMIC_RMW_I64_XOR8_U = 0x3e, + WASM_OP_ATOMIC_RMW_I64_XOR16_U = 0x3f, + WASM_OP_ATOMIC_RMW_I64_XOR32_U = 0x40, + /* atomic xchg */ + WASM_OP_ATOMIC_RMW_I32_XCHG = 0x41, + WASM_OP_ATOMIC_RMW_I64_XCHG = 0x42, + WASM_OP_ATOMIC_RMW_I32_XCHG8_U = 0x43, + WASM_OP_ATOMIC_RMW_I32_XCHG16_U = 0x44, + WASM_OP_ATOMIC_RMW_I64_XCHG8_U = 0x45, + WASM_OP_ATOMIC_RMW_I64_XCHG16_U = 0x46, + WASM_OP_ATOMIC_RMW_I64_XCHG32_U = 0x47, + /* atomic cmpxchg */ + WASM_OP_ATOMIC_RMW_I32_CMPXCHG = 0x48, + WASM_OP_ATOMIC_RMW_I64_CMPXCHG = 0x49, + WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U = 0x4a, + WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U = 0x4b, + WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U = 0x4c, + WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U = 0x4d, + WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U = 0x4e, +} WASMAtomicEXTOpcode; + +#if WASM_ENABLE_DEBUG_INTERP != 0 +#define DEF_DEBUG_BREAK_HANDLE() \ + [DEBUG_OP_BREAK] = HANDLE_OPCODE(DEBUG_OP_BREAK), /* 0xdb */ +#else +#define DEF_DEBUG_BREAK_HANDLE() +#endif +#define SET_GOTO_TABLE_ELEM(opcode) [opcode] = HANDLE_OPCODE(opcode) + +#if WASM_ENABLE_SIMDE != 0 +#define SET_GOTO_TABLE_SIMD_PREFIX_ELEM() \ + SET_GOTO_TABLE_ELEM(WASM_OP_SIMD_PREFIX), +#else +#define SET_GOTO_TABLE_SIMD_PREFIX_ELEM() +#endif + +#if WASM_ENABLE_SIMDE != 0 +#define DEF_EXT_V128_HANDLE() \ + SET_GOTO_TABLE_ELEM(EXT_OP_SET_LOCAL_FAST_V128), /* 0xdd */ \ + SET_GOTO_TABLE_ELEM(EXT_OP_TEE_LOCAL_FAST_V128), /* 0xde */ \ + SET_GOTO_TABLE_ELEM(EXT_OP_COPY_STACK_TOP_V128), /* 0xdf */ \ + SET_GOTO_TABLE_ELEM(WASM_OP_GET_GLOBAL_V128), /* 0xe0 */ \ + SET_GOTO_TABLE_ELEM(WASM_OP_SET_GLOBAL_V128), /* 0xe1 */ \ + SET_GOTO_TABLE_ELEM(WASM_OP_SELECT_128), /* 0xe2 */ + +#else +#define DEF_EXT_V128_HANDLE() +#endif +/* + * Macro used to generate computed goto tables for the C interpreter. + */ +#define WASM_INSTRUCTION_NUM 256 + +#define DEFINE_GOTO_TABLE(type, _name) \ + static type _name[WASM_INSTRUCTION_NUM] = { \ + HANDLE_OPCODE(WASM_OP_UNREACHABLE), /* 0x00 */ \ + HANDLE_OPCODE(WASM_OP_NOP), /* 0x01 */ \ + HANDLE_OPCODE(WASM_OP_BLOCK), /* 0x02 */ \ + HANDLE_OPCODE(WASM_OP_LOOP), /* 0x03 */ \ + HANDLE_OPCODE(WASM_OP_IF), /* 0x04 */ \ + HANDLE_OPCODE(WASM_OP_ELSE), /* 0x05 */ \ + HANDLE_OPCODE(WASM_OP_TRY), /* 0x06 */ \ + HANDLE_OPCODE(WASM_OP_CATCH), /* 0x07 */ \ + HANDLE_OPCODE(WASM_OP_THROW), /* 0x08 */ \ + HANDLE_OPCODE(WASM_OP_RETHROW), /* 0x09 */ \ + HANDLE_OPCODE(WASM_OP_UNUSED_0x0a), /* 0x0a */ \ + HANDLE_OPCODE(WASM_OP_END), /* 0x0b */ \ + HANDLE_OPCODE(WASM_OP_BR), /* 0x0c */ \ + HANDLE_OPCODE(WASM_OP_BR_IF), /* 0x0d */ \ + HANDLE_OPCODE(WASM_OP_BR_TABLE), /* 0x0e */ \ + HANDLE_OPCODE(WASM_OP_RETURN), /* 0x0f */ \ + HANDLE_OPCODE(WASM_OP_CALL), /* 0x10 */ \ + HANDLE_OPCODE(WASM_OP_CALL_INDIRECT), /* 0x11 */ \ + HANDLE_OPCODE(WASM_OP_RETURN_CALL), /* 0x12 */ \ + HANDLE_OPCODE(WASM_OP_RETURN_CALL_INDIRECT), /* 0x13 */ \ + HANDLE_OPCODE(WASM_OP_CALL_REF), /* 0x14 */ \ + HANDLE_OPCODE(WASM_OP_RETURN_CALL_REF), /* 0x15 */ \ + HANDLE_OPCODE(WASM_OP_UNUSED_0x16), /* 0x16 */ \ + HANDLE_OPCODE(WASM_OP_UNUSED_0x17), /* 0x17 */ \ + HANDLE_OPCODE(WASM_OP_DELEGATE), /* 0x18 */ \ + HANDLE_OPCODE(WASM_OP_CATCH_ALL), /* 0x19 */ \ + HANDLE_OPCODE(WASM_OP_DROP), /* 0x1a */ \ + HANDLE_OPCODE(WASM_OP_SELECT), /* 0x1b */ \ + HANDLE_OPCODE(WASM_OP_SELECT_T), /* 0x1c */ \ + HANDLE_OPCODE(WASM_OP_GET_GLOBAL_64), /* 0x1d */ \ + HANDLE_OPCODE(WASM_OP_SET_GLOBAL_64), /* 0x1e */ \ + HANDLE_OPCODE(WASM_OP_SET_GLOBAL_AUX_STACK), /* 0x1f */ \ + HANDLE_OPCODE(WASM_OP_GET_LOCAL), /* 0x20 */ \ + HANDLE_OPCODE(WASM_OP_SET_LOCAL), /* 0x21 */ \ + HANDLE_OPCODE(WASM_OP_TEE_LOCAL), /* 0x22 */ \ + HANDLE_OPCODE(WASM_OP_GET_GLOBAL), /* 0x23 */ \ + HANDLE_OPCODE(WASM_OP_SET_GLOBAL), /* 0x24 */ \ + HANDLE_OPCODE(WASM_OP_TABLE_GET), /* 0x25 */ \ + HANDLE_OPCODE(WASM_OP_TABLE_SET), /* 0x26 */ \ + HANDLE_OPCODE(WASM_OP_UNUSED_0x27), /* 0x27 */ \ + HANDLE_OPCODE(WASM_OP_I32_LOAD), /* 0x28 */ \ + HANDLE_OPCODE(WASM_OP_I64_LOAD), /* 0x29 */ \ + HANDLE_OPCODE(WASM_OP_F32_LOAD), /* 0x2a */ \ + HANDLE_OPCODE(WASM_OP_F64_LOAD), /* 0x2b */ \ + HANDLE_OPCODE(WASM_OP_I32_LOAD8_S), /* 0x2c */ \ + HANDLE_OPCODE(WASM_OP_I32_LOAD8_U), /* 0x2d */ \ + HANDLE_OPCODE(WASM_OP_I32_LOAD16_S), /* 0x2e */ \ + HANDLE_OPCODE(WASM_OP_I32_LOAD16_U), /* 0x2f */ \ + HANDLE_OPCODE(WASM_OP_I64_LOAD8_S), /* 0x30 */ \ + HANDLE_OPCODE(WASM_OP_I64_LOAD8_U), /* 0x31 */ \ + HANDLE_OPCODE(WASM_OP_I64_LOAD16_S), /* 0x32 */ \ + HANDLE_OPCODE(WASM_OP_I64_LOAD16_U), /* 0x33 */ \ + HANDLE_OPCODE(WASM_OP_I64_LOAD32_S), /* 0x34 */ \ + HANDLE_OPCODE(WASM_OP_I64_LOAD32_U), /* 0x35 */ \ + HANDLE_OPCODE(WASM_OP_I32_STORE), /* 0x36 */ \ + HANDLE_OPCODE(WASM_OP_I64_STORE), /* 0x37 */ \ + HANDLE_OPCODE(WASM_OP_F32_STORE), /* 0x38 */ \ + HANDLE_OPCODE(WASM_OP_F64_STORE), /* 0x39 */ \ + HANDLE_OPCODE(WASM_OP_I32_STORE8), /* 0x3a */ \ + HANDLE_OPCODE(WASM_OP_I32_STORE16), /* 0x3b */ \ + HANDLE_OPCODE(WASM_OP_I64_STORE8), /* 0x3c */ \ + HANDLE_OPCODE(WASM_OP_I64_STORE16), /* 0x3d */ \ + HANDLE_OPCODE(WASM_OP_I64_STORE32), /* 0x3e */ \ + HANDLE_OPCODE(WASM_OP_MEMORY_SIZE), /* 0x3f */ \ + HANDLE_OPCODE(WASM_OP_MEMORY_GROW), /* 0x40 */ \ + HANDLE_OPCODE(WASM_OP_I32_CONST), /* 0x41 */ \ + HANDLE_OPCODE(WASM_OP_I64_CONST), /* 0x42 */ \ + HANDLE_OPCODE(WASM_OP_F32_CONST), /* 0x43 */ \ + HANDLE_OPCODE(WASM_OP_F64_CONST), /* 0x44 */ \ + HANDLE_OPCODE(WASM_OP_I32_EQZ), /* 0x45 */ \ + HANDLE_OPCODE(WASM_OP_I32_EQ), /* 0x46 */ \ + HANDLE_OPCODE(WASM_OP_I32_NE), /* 0x47 */ \ + HANDLE_OPCODE(WASM_OP_I32_LT_S), /* 0x48 */ \ + HANDLE_OPCODE(WASM_OP_I32_LT_U), /* 0x49 */ \ + HANDLE_OPCODE(WASM_OP_I32_GT_S), /* 0x4a */ \ + HANDLE_OPCODE(WASM_OP_I32_GT_U), /* 0x4b */ \ + HANDLE_OPCODE(WASM_OP_I32_LE_S), /* 0x4c */ \ + HANDLE_OPCODE(WASM_OP_I32_LE_U), /* 0x4d */ \ + HANDLE_OPCODE(WASM_OP_I32_GE_S), /* 0x4e */ \ + HANDLE_OPCODE(WASM_OP_I32_GE_U), /* 0x4f */ \ + HANDLE_OPCODE(WASM_OP_I64_EQZ), /* 0x50 */ \ + HANDLE_OPCODE(WASM_OP_I64_EQ), /* 0x51 */ \ + HANDLE_OPCODE(WASM_OP_I64_NE), /* 0x52 */ \ + HANDLE_OPCODE(WASM_OP_I64_LT_S), /* 0x53 */ \ + HANDLE_OPCODE(WASM_OP_I64_LT_U), /* 0x54 */ \ + HANDLE_OPCODE(WASM_OP_I64_GT_S), /* 0x55 */ \ + HANDLE_OPCODE(WASM_OP_I64_GT_U), /* 0x56 */ \ + HANDLE_OPCODE(WASM_OP_I64_LE_S), /* 0x57 */ \ + HANDLE_OPCODE(WASM_OP_I64_LE_U), /* 0x58 */ \ + HANDLE_OPCODE(WASM_OP_I64_GE_S), /* 0x59 */ \ + HANDLE_OPCODE(WASM_OP_I64_GE_U), /* 0x5a */ \ + HANDLE_OPCODE(WASM_OP_F32_EQ), /* 0x5b */ \ + HANDLE_OPCODE(WASM_OP_F32_NE), /* 0x5c */ \ + HANDLE_OPCODE(WASM_OP_F32_LT), /* 0x5d */ \ + HANDLE_OPCODE(WASM_OP_F32_GT), /* 0x5e */ \ + HANDLE_OPCODE(WASM_OP_F32_LE), /* 0x5f */ \ + HANDLE_OPCODE(WASM_OP_F32_GE), /* 0x60 */ \ + HANDLE_OPCODE(WASM_OP_F64_EQ), /* 0x61 */ \ + HANDLE_OPCODE(WASM_OP_F64_NE), /* 0x62 */ \ + HANDLE_OPCODE(WASM_OP_F64_LT), /* 0x63 */ \ + HANDLE_OPCODE(WASM_OP_F64_GT), /* 0x64 */ \ + HANDLE_OPCODE(WASM_OP_F64_LE), /* 0x65 */ \ + HANDLE_OPCODE(WASM_OP_F64_GE), /* 0x66 */ \ + HANDLE_OPCODE(WASM_OP_I32_CLZ), /* 0x67 */ \ + HANDLE_OPCODE(WASM_OP_I32_CTZ), /* 0x68 */ \ + HANDLE_OPCODE(WASM_OP_I32_POPCNT), /* 0x69 */ \ + HANDLE_OPCODE(WASM_OP_I32_ADD), /* 0x6a */ \ + HANDLE_OPCODE(WASM_OP_I32_SUB), /* 0x6b */ \ + HANDLE_OPCODE(WASM_OP_I32_MUL), /* 0x6c */ \ + HANDLE_OPCODE(WASM_OP_I32_DIV_S), /* 0x6d */ \ + HANDLE_OPCODE(WASM_OP_I32_DIV_U), /* 0x6e */ \ + HANDLE_OPCODE(WASM_OP_I32_REM_S), /* 0x6f */ \ + HANDLE_OPCODE(WASM_OP_I32_REM_U), /* 0x70 */ \ + HANDLE_OPCODE(WASM_OP_I32_AND), /* 0x71 */ \ + HANDLE_OPCODE(WASM_OP_I32_OR), /* 0x72 */ \ + HANDLE_OPCODE(WASM_OP_I32_XOR), /* 0x73 */ \ + HANDLE_OPCODE(WASM_OP_I32_SHL), /* 0x74 */ \ + HANDLE_OPCODE(WASM_OP_I32_SHR_S), /* 0x75 */ \ + HANDLE_OPCODE(WASM_OP_I32_SHR_U), /* 0x76 */ \ + HANDLE_OPCODE(WASM_OP_I32_ROTL), /* 0x77 */ \ + HANDLE_OPCODE(WASM_OP_I32_ROTR), /* 0x78 */ \ + HANDLE_OPCODE(WASM_OP_I64_CLZ), /* 0x79 */ \ + HANDLE_OPCODE(WASM_OP_I64_CTZ), /* 0x7a */ \ + HANDLE_OPCODE(WASM_OP_I64_POPCNT), /* 0x7b */ \ + HANDLE_OPCODE(WASM_OP_I64_ADD), /* 0x7c */ \ + HANDLE_OPCODE(WASM_OP_I64_SUB), /* 0x7d */ \ + HANDLE_OPCODE(WASM_OP_I64_MUL), /* 0x7e */ \ + HANDLE_OPCODE(WASM_OP_I64_DIV_S), /* 0x7f */ \ + HANDLE_OPCODE(WASM_OP_I64_DIV_U), /* 0x80 */ \ + HANDLE_OPCODE(WASM_OP_I64_REM_S), /* 0x81 */ \ + HANDLE_OPCODE(WASM_OP_I64_REM_U), /* 0x82 */ \ + HANDLE_OPCODE(WASM_OP_I64_AND), /* 0x83 */ \ + HANDLE_OPCODE(WASM_OP_I64_OR), /* 0x84 */ \ + HANDLE_OPCODE(WASM_OP_I64_XOR), /* 0x85 */ \ + HANDLE_OPCODE(WASM_OP_I64_SHL), /* 0x86 */ \ + HANDLE_OPCODE(WASM_OP_I64_SHR_S), /* 0x87 */ \ + HANDLE_OPCODE(WASM_OP_I64_SHR_U), /* 0x88 */ \ + HANDLE_OPCODE(WASM_OP_I64_ROTL), /* 0x89 */ \ + HANDLE_OPCODE(WASM_OP_I64_ROTR), /* 0x8a */ \ + HANDLE_OPCODE(WASM_OP_F32_ABS), /* 0x8b */ \ + HANDLE_OPCODE(WASM_OP_F32_NEG), /* 0x8c */ \ + HANDLE_OPCODE(WASM_OP_F32_CEIL), /* 0x8d */ \ + HANDLE_OPCODE(WASM_OP_F32_FLOOR), /* 0x8e */ \ + HANDLE_OPCODE(WASM_OP_F32_TRUNC), /* 0x8f */ \ + HANDLE_OPCODE(WASM_OP_F32_NEAREST), /* 0x90 */ \ + HANDLE_OPCODE(WASM_OP_F32_SQRT), /* 0x91 */ \ + HANDLE_OPCODE(WASM_OP_F32_ADD), /* 0x92 */ \ + HANDLE_OPCODE(WASM_OP_F32_SUB), /* 0x93 */ \ + HANDLE_OPCODE(WASM_OP_F32_MUL), /* 0x94 */ \ + HANDLE_OPCODE(WASM_OP_F32_DIV), /* 0x95 */ \ + HANDLE_OPCODE(WASM_OP_F32_MIN), /* 0x96 */ \ + HANDLE_OPCODE(WASM_OP_F32_MAX), /* 0x97 */ \ + HANDLE_OPCODE(WASM_OP_F32_COPYSIGN), /* 0x98 */ \ + HANDLE_OPCODE(WASM_OP_F64_ABS), /* 0x99 */ \ + HANDLE_OPCODE(WASM_OP_F64_NEG), /* 0x9a */ \ + HANDLE_OPCODE(WASM_OP_F64_CEIL), /* 0x9b */ \ + HANDLE_OPCODE(WASM_OP_F64_FLOOR), /* 0x9c */ \ + HANDLE_OPCODE(WASM_OP_F64_TRUNC), /* 0x9d */ \ + HANDLE_OPCODE(WASM_OP_F64_NEAREST), /* 0x9e */ \ + HANDLE_OPCODE(WASM_OP_F64_SQRT), /* 0x9f */ \ + HANDLE_OPCODE(WASM_OP_F64_ADD), /* 0xa0 */ \ + HANDLE_OPCODE(WASM_OP_F64_SUB), /* 0xa1 */ \ + HANDLE_OPCODE(WASM_OP_F64_MUL), /* 0xa2 */ \ + HANDLE_OPCODE(WASM_OP_F64_DIV), /* 0xa3 */ \ + HANDLE_OPCODE(WASM_OP_F64_MIN), /* 0xa4 */ \ + HANDLE_OPCODE(WASM_OP_F64_MAX), /* 0xa5 */ \ + HANDLE_OPCODE(WASM_OP_F64_COPYSIGN), /* 0xa6 */ \ + HANDLE_OPCODE(WASM_OP_I32_WRAP_I64), /* 0xa7 */ \ + HANDLE_OPCODE(WASM_OP_I32_TRUNC_S_F32), /* 0xa8 */ \ + HANDLE_OPCODE(WASM_OP_I32_TRUNC_U_F32), /* 0xa9 */ \ + HANDLE_OPCODE(WASM_OP_I32_TRUNC_S_F64), /* 0xaa */ \ + HANDLE_OPCODE(WASM_OP_I32_TRUNC_U_F64), /* 0xab */ \ + HANDLE_OPCODE(WASM_OP_I64_EXTEND_S_I32), /* 0xac */ \ + HANDLE_OPCODE(WASM_OP_I64_EXTEND_U_I32), /* 0xad */ \ + HANDLE_OPCODE(WASM_OP_I64_TRUNC_S_F32), /* 0xae */ \ + HANDLE_OPCODE(WASM_OP_I64_TRUNC_U_F32), /* 0xaf */ \ + HANDLE_OPCODE(WASM_OP_I64_TRUNC_S_F64), /* 0xb0 */ \ + HANDLE_OPCODE(WASM_OP_I64_TRUNC_U_F64), /* 0xb1 */ \ + HANDLE_OPCODE(WASM_OP_F32_CONVERT_S_I32), /* 0xb2 */ \ + HANDLE_OPCODE(WASM_OP_F32_CONVERT_U_I32), /* 0xb3 */ \ + HANDLE_OPCODE(WASM_OP_F32_CONVERT_S_I64), /* 0xb4 */ \ + HANDLE_OPCODE(WASM_OP_F32_CONVERT_U_I64), /* 0xb5 */ \ + HANDLE_OPCODE(WASM_OP_F32_DEMOTE_F64), /* 0xb6 */ \ + HANDLE_OPCODE(WASM_OP_F64_CONVERT_S_I32), /* 0xb7 */ \ + HANDLE_OPCODE(WASM_OP_F64_CONVERT_U_I32), /* 0xb8 */ \ + HANDLE_OPCODE(WASM_OP_F64_CONVERT_S_I64), /* 0xb9 */ \ + HANDLE_OPCODE(WASM_OP_F64_CONVERT_U_I64), /* 0xba */ \ + HANDLE_OPCODE(WASM_OP_F64_PROMOTE_F32), /* 0xbb */ \ + HANDLE_OPCODE(WASM_OP_I32_REINTERPRET_F32), /* 0xbc */ \ + HANDLE_OPCODE(WASM_OP_I64_REINTERPRET_F64), /* 0xbd */ \ + HANDLE_OPCODE(WASM_OP_F32_REINTERPRET_I32), /* 0xbe */ \ + HANDLE_OPCODE(WASM_OP_F64_REINTERPRET_I64), /* 0xbf */ \ + HANDLE_OPCODE(WASM_OP_I32_EXTEND8_S), /* 0xc0 */ \ + HANDLE_OPCODE(WASM_OP_I32_EXTEND16_S), /* 0xc1 */ \ + HANDLE_OPCODE(WASM_OP_I64_EXTEND8_S), /* 0xc2 */ \ + HANDLE_OPCODE(WASM_OP_I64_EXTEND16_S), /* 0xc3 */ \ + HANDLE_OPCODE(WASM_OP_I64_EXTEND32_S), /* 0xc4 */ \ + HANDLE_OPCODE(WASM_OP_DROP_64), /* 0xc5 */ \ + HANDLE_OPCODE(WASM_OP_SELECT_64), /* 0xc6 */ \ + HANDLE_OPCODE(EXT_OP_GET_LOCAL_FAST), /* 0xc7 */ \ + HANDLE_OPCODE(EXT_OP_SET_LOCAL_FAST_I64), /* 0xc8 */ \ + HANDLE_OPCODE(EXT_OP_SET_LOCAL_FAST), /* 0xc9 */ \ + HANDLE_OPCODE(EXT_OP_TEE_LOCAL_FAST), /* 0xca */ \ + HANDLE_OPCODE(EXT_OP_TEE_LOCAL_FAST_I64), /* 0xcb */ \ + HANDLE_OPCODE(EXT_OP_COPY_STACK_TOP), /* 0xcc */ \ + HANDLE_OPCODE(EXT_OP_COPY_STACK_TOP_I64), /* 0xcd */ \ + HANDLE_OPCODE(EXT_OP_COPY_STACK_VALUES), /* 0xce */ \ + HANDLE_OPCODE(WASM_OP_IMPDEP), /* 0xcf */ \ + HANDLE_OPCODE(WASM_OP_REF_NULL), /* 0xd0 */ \ + HANDLE_OPCODE(WASM_OP_REF_IS_NULL), /* 0xd1 */ \ + HANDLE_OPCODE(WASM_OP_REF_FUNC), /* 0xd2 */ \ + HANDLE_OPCODE(WASM_OP_REF_EQ), /* 0xd3 */ \ + HANDLE_OPCODE(WASM_OP_REF_AS_NON_NULL), /* 0xd4 */ \ + HANDLE_OPCODE(WASM_OP_BR_ON_NULL), /* 0xd5 */ \ + HANDLE_OPCODE(WASM_OP_BR_ON_NON_NULL), /* 0xd6 */ \ + HANDLE_OPCODE(EXT_OP_BLOCK), /* 0xd7 */ \ + HANDLE_OPCODE(EXT_OP_LOOP), /* 0xd8 */ \ + HANDLE_OPCODE(EXT_OP_IF), /* 0xd9 */ \ + HANDLE_OPCODE(EXT_OP_BR_TABLE_CACHE), /* 0xda */ \ + HANDLE_OPCODE(EXT_OP_TRY), /* 0xdb */ \ + SET_GOTO_TABLE_ELEM(WASM_OP_GC_PREFIX), /* 0xfb */ \ + SET_GOTO_TABLE_ELEM(WASM_OP_MISC_PREFIX), /* 0xfc */ \ + SET_GOTO_TABLE_SIMD_PREFIX_ELEM() /* 0xfd */ \ + SET_GOTO_TABLE_ELEM(WASM_OP_ATOMIC_PREFIX), /* 0xfe */ \ + DEF_DEBUG_BREAK_HANDLE() DEF_EXT_V128_HANDLE() \ + }; + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_OPCODE_H */ diff --git a/wamr/include/wasm_runtime.h b/wamr/include/wasm_runtime.h new file mode 100644 index 0000000..da69114 --- /dev/null +++ b/wamr/include/wasm_runtime.h @@ -0,0 +1,912 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_RUNTIME_H +#define _WASM_RUNTIME_H + +#include "wasm.h" // wasm.h +#include "bh_atomic.h" // bh_atomic.h +#include "bh_bitmap.h" // bh_bitmap.h +#include "bh_hashmap.h" // bh_hashmap.h +#include "wasm_runtime_common.h" // ../common/wasm_runtime_common.h +#include "wasm_exec_env.h" // ../common/wasm_exec_env.h + +#ifdef __cplusplus +extern "C" { +#endif + +#define EXCEPTION_BUF_LEN 128 + +typedef struct WASMModuleInstance WASMModuleInstance; +typedef struct WASMFunctionInstance WASMFunctionInstance; +typedef struct WASMMemoryInstance WASMMemoryInstance; +typedef struct WASMTableInstance WASMTableInstance; +typedef struct WASMGlobalInstance WASMGlobalInstance; +#if WASM_ENABLE_TAGS != 0 +typedef struct WASMTagInstance WASMTagInstance; +#endif + +/** + * When LLVM JIT, WAMR compiler or AOT is enabled, we should ensure that + * some offsets of the same field in the interpreter module instance and + * aot module instance are the same, so that the LLVM JITed/AOTed code + * can smoothly access the interpreter module instance. + * Same for the memory instance and table instance. + * We use the macro DefPointer to define some related pointer fields. + */ +#if (WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 \ + || WASM_ENABLE_AOT != 0) \ + && UINTPTR_MAX == UINT32_MAX +/* Add u32 padding if LLVM JIT, WAMR compiler or AOT is enabled on + 32-bit platform */ +#define DefPointer(type, field) \ + type field; \ + uint32 field##_padding +#else +#define DefPointer(type, field) type field +#endif + +typedef enum WASMExceptionID { + EXCE_UNREACHABLE = 0, + EXCE_OUT_OF_MEMORY, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, + EXCE_INTEGER_OVERFLOW, + EXCE_INTEGER_DIVIDE_BY_ZERO, + EXCE_INVALID_CONVERSION_TO_INTEGER, + EXCE_INVALID_FUNCTION_TYPE_INDEX, + EXCE_INVALID_FUNCTION_INDEX, + EXCE_UNDEFINED_ELEMENT, + EXCE_UNINITIALIZED_ELEMENT, + EXCE_CALL_UNLINKED_IMPORT_FUNC, + EXCE_NATIVE_STACK_OVERFLOW, + EXCE_UNALIGNED_ATOMIC, + EXCE_AUX_STACK_OVERFLOW, + EXCE_AUX_STACK_UNDERFLOW, + EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, + EXCE_OPERAND_STACK_OVERFLOW, + EXCE_FAILED_TO_COMPILE_FAST_JIT_FUNC, + /* GC related exceptions */ + EXCE_NULL_FUNC_OBJ, + EXCE_NULL_STRUCT_OBJ, + EXCE_NULL_ARRAY_OBJ, + EXCE_NULL_I31_OBJ, + EXCE_NULL_REFERENCE, + EXCE_FAILED_TO_CREATE_RTT_TYPE, + EXCE_FAILED_TO_CREATE_STRUCT_OBJ, + EXCE_FAILED_TO_CREATE_ARRAY_OBJ, + EXCE_FAILED_TO_CREATE_EXTERNREF_OBJ, + EXCE_CAST_FAILURE, + EXCE_ARRAY_IDX_OOB, + EXCE_FAILED_TO_CREATE_STRING, + EXCE_FAILED_TO_CREATE_STRINGREF, + EXCE_FAILED_TO_CREATE_STRINGVIEW, + EXCE_FAILED_TO_ENCODE_STRING, + EXCE_ALREADY_THROWN, + EXCE_NUM, +} WASMExceptionID; + +typedef union { + uint64 u64; + uint32 u32[2]; +} MemBound; + +typedef struct WASMSharedHeap { + /* The global shared heap list maintained in runtime, used for runtime + * destroy */ + DefPointer(struct WASMSharedHeap *, next); + /* The logical shared heap chain the shared heap in */ + DefPointer(struct WASMSharedHeap *, chain_next); + /* Will be null if shared heap is created from pre allocated memory chunk + * and don't need to dynamic malloc and free */ + DefPointer(void *, heap_handle); + DefPointer(uint8 *, base_addr); + uint64 size; + uint64 start_off_mem64; + uint64 start_off_mem32; + /* The number of wasm apps it attached to, for a shared heap chain, only the + * list head need to maintain the valid attached_count */ + uint8 attached_count; +} WASMSharedHeap; + +struct WASMMemoryInstance { + /* Module type */ + uint32 module_type; + + /* Whether the memory is shared */ + uint8 is_shared_memory; + + /* Whether the memory has 64-bit memory addresses */ + uint8 is_memory64; + + /* Reference count of the memory instance: + 0: non-shared memory, > 0: shared memory */ + bh_atomic_16_t ref_count; + + /* Four-byte paddings to ensure the layout of WASMMemoryInstance is the same + * in both 64-bit and 32-bit */ + uint8 _paddings[4]; + + /* Number bytes per page */ + uint32 num_bytes_per_page; + /* Current page count */ + uint32 cur_page_count; + /* Maximum page count */ + uint32 max_page_count; + /* Memory data size */ + uint64 memory_data_size; + /** + * Memory data begin address, Note: + * the app-heap might be inserted in to the linear memory, + * when memory is re-allocated, the heap data and memory data + * must be copied to new memory also + */ + DefPointer(uint8 *, memory_data); + /* Memory data end address */ + DefPointer(uint8 *, memory_data_end); + + /* Heap data base address */ + DefPointer(uint8 *, heap_data); + /* Heap data end address */ + DefPointer(uint8 *, heap_data_end); + /* The heap created */ + DefPointer(void *, heap_handle); + /* TODO: use it to replace the g_shared_memory_lock */ + DefPointer(korp_mutex *, memory_lock); + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_AOT != 0 + MemBound mem_bound_check_1byte; + MemBound mem_bound_check_2bytes; + MemBound mem_bound_check_4bytes; + MemBound mem_bound_check_8bytes; + MemBound mem_bound_check_16bytes; +#endif +}; + +/* WASMTableInstance is used to represent table instance in + * runtime, to compute the table element address with index + * we need to know the element type and the element ref type. + * For pointer type, it's 32-bit or 64-bit, align up to 8 bytes + * to simplify the computation. + * And each struct member should be 4-byte or 8-byte aligned. + */ +struct WASMTableInstance { + /* The element type */ + uint8 elem_type; + uint8 is_table64; + uint8 __padding__[6]; + union { +#if WASM_ENABLE_GC != 0 + WASMRefType *elem_ref_type; +#endif + uint64 __padding__; + } elem_ref_type; + /* Current size */ + uint32 cur_size; + /* Maximum size */ + uint32 max_size; + /* Table elements */ + table_elem_type_t elems[1]; +}; + +struct WASMGlobalInstance { + /* value type, VALUE_TYPE_I32/I64/F32/F64 */ + uint8 type; + /* mutable or constant */ + bool is_mutable; + /* data offset to the address of initial_value, started from the end of + * WASMMemoryInstance(start of WASMGlobalInstance)*/ + uint32 data_offset; + /* initial value */ + WASMValue initial_value; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type; +#endif +#if WASM_ENABLE_MULTI_MODULE != 0 + /* just for import, keep the reference here */ + WASMModuleInstance *import_module_inst; + WASMGlobalInstance *import_global_inst; +#endif +}; + +struct WASMFunctionInstance { + /* whether it is import function or WASM function */ + bool is_import_func; + /* parameter count */ + uint16 param_count; + /* local variable count, 0 for import function */ + uint16 local_count; + /* cell num of parameters */ + uint16 param_cell_num; + /* cell num of return type */ + uint16 ret_cell_num; + /* cell num of local variables, 0 for import function */ + uint16 local_cell_num; +#if WASM_ENABLE_FAST_INTERP != 0 + /* cell num of consts */ + uint16 const_cell_num; +#endif + uint16 *local_offsets; + /* parameter types */ + uint8 *param_types; + /* local types, NULL for import function */ + uint8 *local_types; + union { + WASMFunctionImport *func_import; + WASMFunction *func; + } u; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModuleInstance *import_module_inst; + WASMFunctionInstance *import_func_inst; +#endif +#if WASM_ENABLE_PERF_PROFILING != 0 + /* total execution time */ + uint64 total_exec_time; + /* total execution count */ + uint32 total_exec_cnt; + /* children execution time */ + uint64 children_exec_time; +#endif +}; + +#if WASM_ENABLE_TAGS != 0 +struct WASMTagInstance { + bool is_import_tag; + /* tag attribute */ + uint8 attribute; + /* tag type index */ + uint32 type; + union { + WASMTagImport *tag_import; + WASMTag *tag; + } u; + +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModuleInstance *import_module_inst; + WASMTagInstance *import_tag_inst; +#endif +}; +#endif + +#if WASM_ENABLE_EXCE_HANDLING != 0 +#define INVALID_TAGINDEX ((uint32)0xFFFFFFFF) +#define SET_INVALID_TAGINDEX(tag) (tag = INVALID_TAGINDEX) +#define IS_INVALID_TAGINDEX(tag) ((tag & INVALID_TAGINDEX) == INVALID_TAGINDEX) +#endif +typedef struct WASMExportFuncInstance { + char *name; + WASMFunctionInstance *function; +} WASMExportFuncInstance; + +typedef struct WASMExportGlobInstance { + char *name; + WASMGlobalInstance *global; +} WASMExportGlobInstance; + +typedef struct WASMExportTabInstance { + char *name; + WASMTableInstance *table; +} WASMExportTabInstance; + +typedef struct WASMExportMemInstance { + char *name; + WASMMemoryInstance *memory; +} WASMExportMemInstance; + +#if WASM_ENABLE_TAGS != 0 +typedef struct WASMExportTagInstance { + char *name; + WASMTagInstance *tag; +} WASMExportTagInstance; +#endif + +/* wasm-c-api import function info */ +typedef struct CApiFuncImport { + /* host func pointer after linked */ + void *func_ptr_linked; + /* whether the host func has env argument */ + bool with_env_arg; + /* the env argument of the host func */ + void *env_arg; +} CApiFuncImport; + +/* The common part of WASMModuleInstanceExtra and AOTModuleInstanceExtra */ +typedef struct WASMModuleInstanceExtraCommon { +#if WASM_ENABLE_MODULE_INST_CONTEXT != 0 + void *contexts[WASM_MAX_INSTANCE_CONTEXTS]; +#endif +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 + /* Disable bounds checks or not */ + bool disable_bounds_checks; +#endif +#if WASM_ENABLE_BULK_MEMORY != 0 + bh_bitmap *data_dropped; +#endif +#if WASM_ENABLE_REF_TYPES != 0 + bh_bitmap *elem_dropped; +#endif + +#if WASM_ENABLE_GC != 0 + /* The gc heap memory pool */ + uint8 *gc_heap_pool; + /* The gc heap created */ + void *gc_heap_handle; +#endif +} WASMModuleInstanceExtraCommon; + +/* Extra info of WASM module instance for interpreter/jit mode */ +typedef struct WASMModuleInstanceExtra { + WASMModuleInstanceExtraCommon common; + + WASMGlobalInstance *globals; + WASMFunctionInstance *functions; + + uint32 global_count; + uint32 function_count; + + WASMFunctionInstance *start_function; + WASMFunctionInstance *malloc_function; + WASMFunctionInstance *free_function; + WASMFunctionInstance *retain_function; + + RunningMode running_mode; + +#if WASM_ENABLE_MULTI_MODULE != 0 + bh_list sub_module_inst_list_head; + bh_list *sub_module_inst_list; + /* linked table instances of import table instances */ + WASMTableInstance **table_insts_linked; +#endif + +#if WASM_ENABLE_TAGS != 0 + uint32 tag_count; + uint32 export_tag_count; + WASMTagInstance *tags; + WASMExportTagInstance *export_tags; + void **import_tag_ptrs; +#endif + +#if WASM_ENABLE_MEMORY_PROFILING != 0 + uint32 max_aux_stack_used; +#endif + +#if WASM_ENABLE_SHARED_HEAP != 0 + /* + * Adjusted shared heap based addr to simple the calculation + * in the aot code. The value is: + * shared_heap->base_addr - shared_heap->start_off + */ + uint8 *shared_heap_base_addr_adj; + MemBound shared_heap_start_off; + MemBound shared_heap_end_off; + WASMSharedHeap *shared_heap; +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 \ + || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0) + WASMModuleInstance *next; +#endif +} WASMModuleInstanceExtra; + +struct AOTFuncPerfProfInfo; + +struct WASMModuleInstance { + /* Module instance type, for module instance loaded from + WASM bytecode binary, this field is Wasm_Module_Bytecode; + for module instance loaded from AOT file, this field is + Wasm_Module_AoT, and this structure should be treated as + AOTModuleInstance structure. */ + uint32 module_type; + + uint32 memory_count; + DefPointer(WASMMemoryInstance **, memories); + + /* global and table info */ + uint32 global_data_size; + uint32 table_count; + DefPointer(uint8 *, global_data); + /* For AOTModuleInstance, it denotes `AOTTableInstance *` */ + DefPointer(WASMTableInstance **, tables); + + /* import func ptrs + llvm jit func ptrs */ + DefPointer(void **, func_ptrs); + + /* function type indexes */ + DefPointer(uint32 *, func_type_indexes); + + uint32 export_func_count; + uint32 export_global_count; + uint32 export_memory_count; + uint32 export_table_count; + /* For AOTModuleInstance, it denotes `AOTFunctionInstance *` */ + DefPointer(WASMExportFuncInstance *, export_functions); + DefPointer(WASMExportGlobInstance *, export_globals); + DefPointer(WASMExportMemInstance *, export_memories); + DefPointer(WASMExportTabInstance *, export_tables); + + /* The exception buffer of wasm interpreter for current thread. */ + char cur_exception[EXCEPTION_BUF_LEN]; + + /* The WASM module or AOT module, for AOTModuleInstance, + it denotes `AOTModule *` */ + DefPointer(WASMModule *, module); + + DefPointer(WASMExecEnv *, exec_env_singleton); + /* Array of function pointers to import functions, + not available in AOTModuleInstance */ + DefPointer(void **, import_func_ptrs); + /* Array of function pointers to fast jit functions, + not available in AOTModuleInstance: + Only when the multi-tier JIT macros are all enabled and the running + mode of current module instance is set to Mode_Fast_JIT, runtime + will allocate new memory for it, otherwise it always points to the + module->fast_jit_func_ptrs */ + DefPointer(void **, fast_jit_func_ptrs); + /* The custom data that can be set/get by wasm_{get|set}_custom_data */ + DefPointer(void *, custom_data); + /* Stack frames, used in call stack dump and perf profiling */ + DefPointer(Vector *, frames); + /* Function performance profiling info list, only available + in AOTModuleInstance */ + DefPointer(struct AOTFuncPerfProfInfo *, func_perf_profilings); + DefPointer(CApiFuncImport *, c_api_func_imports); + /* Pointer to the exec env currently used */ + DefPointer(WASMExecEnv *, cur_exec_env); + /* WASM/AOT module extra info, for AOTModuleInstance, + it denotes `AOTModuleInstanceExtra *` */ + DefPointer(WASMModuleInstanceExtra *, e); + + /* Default WASM operand stack size */ + uint32 default_wasm_stack_size; + uint32 reserved[7]; + + /* + * +------------------------------+ <-- memories + * | WASMMemoryInstance[mem_count], mem_count is always 1 for LLVM JIT/AOT + * +------------------------------+ <-- global_data + * | global data + * +------------------------------+ <-- tables + * | WASMTableInstance[table_count] + * +------------------------------+ <-- e + * | WASMModuleInstanceExtra + * +------------------------------+ + */ + union { + uint64 _make_it_8_byte_aligned_; + WASMMemoryInstance memory_instances[1]; + uint8 bytes[1]; + } global_table_data; +}; + +struct WASMInterpFrame; +typedef struct WASMInterpFrame WASMRuntimeFrame; + +#if WASM_ENABLE_MULTI_MODULE != 0 +typedef struct WASMSubModInstNode { + bh_list_link l; + /* point to a string pool */ + const char *module_name; + WASMModuleInstance *module_inst; +} WASMSubModInstNode; +#endif + +/** + * Return the code block of a function. + * + * @param func the WASM function instance + * + * @return the code block of the function + */ +static inline uint8 * +wasm_get_func_code(WASMFunctionInstance *func) +{ +#if WASM_ENABLE_FAST_INTERP == 0 + return func->is_import_func ? NULL : func->u.func->code; +#else + return func->is_import_func ? NULL : func->u.func->code_compiled; +#endif +} + +/** + * Return the code block end of a function. + * + * @param func the WASM function instance + * + * @return the code block end of the function + */ +static inline uint8 * +wasm_get_func_code_end(WASMFunctionInstance *func) +{ +#if WASM_ENABLE_FAST_INTERP == 0 + return func->is_import_func ? NULL + : func->u.func->code + func->u.func->code_size; +#else + return func->is_import_func + ? NULL + : func->u.func->code_compiled + func->u.func->code_compiled_size; +#endif +} + +WASMModule * +wasm_load(uint8 *buf, uint32 size, +#if WASM_ENABLE_MULTI_MODULE != 0 + bool main_module, +#endif + const LoadArgs *args, char *error_buf, uint32 error_buf_size); + +WASMModule * +wasm_load_from_sections(WASMSection *section_list, char *error_buf, + uint32 error_buf_size); + +void +wasm_unload(WASMModule *module); + +bool +wasm_resolve_symbols(WASMModule *module); + +bool +wasm_resolve_import_func(const WASMModule *module, + WASMFunctionImport *function); + +WASMModuleInstance * +wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, + WASMExecEnv *exec_env_main, uint32 stack_size, + uint32 heap_size, uint32 max_memory_pages, char *error_buf, + uint32 error_buf_size); + +void +wasm_dump_perf_profiling(const WASMModuleInstance *module_inst); + +double +wasm_summarize_wasm_execute_time(const WASMModuleInstance *inst); + +double +wasm_get_wasm_func_exec_time(const WASMModuleInstance *inst, + const char *func_name); + +void +wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst); + +bool +wasm_set_running_mode(WASMModuleInstance *module_inst, + RunningMode running_mode); + +WASMFunctionInstance * +wasm_lookup_function(const WASMModuleInstance *module_inst, const char *name); + +WASMMemoryInstance * +wasm_lookup_memory(const WASMModuleInstance *module_inst, const char *name); + +#if WASM_ENABLE_MULTI_MODULE != 0 +WASMGlobalInstance * +wasm_lookup_global(const WASMModuleInstance *module_inst, const char *name); + +WASMTableInstance * +wasm_lookup_table(const WASMModuleInstance *module_inst, const char *name); + +#if WASM_ENABLE_TAGS != 0 +WASMTagInstance * +wasm_lookup_tag(const WASMModuleInstance *module_inst, const char *name, + const char *signature); +#endif + +#endif + +bool +wasm_call_function(WASMExecEnv *exec_env, WASMFunctionInstance *function, + unsigned argc, uint32 argv[]); + +void +wasm_set_exception(WASMModuleInstance *module, const char *exception); + +void +wasm_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id); + +const char * +wasm_get_exception(WASMModuleInstance *module); + +/** + * @brief Copy exception in buffer passed as parameter. Thread-safe version of + * `wasm_get_exception()` + * @note Buffer size must be no smaller than EXCEPTION_BUF_LEN + * @return true if exception found + */ +bool +wasm_copy_exception(WASMModuleInstance *module_inst, char *exception_buf); + +uint64 +wasm_module_malloc_internal(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, uint64 size, + void **p_native_addr); + +uint64 +wasm_module_realloc_internal(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, uint64 ptr, uint64 size, + void **p_native_addr); + +void +wasm_module_free_internal(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, uint64 ptr); + +uint64 +wasm_module_malloc(WASMModuleInstance *module_inst, uint64 size, + void **p_native_addr); + +uint64 +wasm_module_realloc(WASMModuleInstance *module_inst, uint64 ptr, uint64 size, + void **p_native_addr); + +void +wasm_module_free(WASMModuleInstance *module_inst, uint64 ptr); + +uint64 +wasm_module_dup_data(WASMModuleInstance *module_inst, const char *src, + uint64 size); + +/** + * Check whether the app address and the buf is inside the linear memory, + * and convert the app address into native address + */ +bool +wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, + uint64 app_buf_addr, uint64 app_buf_size, + void **p_native_addr); + +WASMMemoryInstance * +wasm_get_default_memory(WASMModuleInstance *module_inst); + +WASMMemoryInstance * +wasm_get_memory_with_idx(WASMModuleInstance *module_inst, uint32 index); + +bool +wasm_enlarge_memory(WASMModuleInstance *module_inst, uint32 inc_page_count); + +bool +wasm_enlarge_memory_with_idx(WASMModuleInstance *module_inst, + uint32 inc_page_count, uint32 memidx); + +bool +wasm_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 argc, uint32 argv[]); + +#if WASM_ENABLE_THREAD_MGR != 0 +bool +wasm_set_aux_stack(WASMExecEnv *exec_env, uint64 start_offset, uint32 size); + +bool +wasm_get_aux_stack(WASMExecEnv *exec_env, uint64 *start_offset, uint32 *size); +#endif + +void +wasm_get_module_mem_consumption(const WASMModule *module, + WASMModuleMemConsumption *mem_conspn); + +void +wasm_get_module_inst_mem_consumption(const WASMModuleInstance *module, + WASMModuleInstMemConsumption *mem_conspn); + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 +static inline bool +wasm_elem_is_active(uint32 mode) +{ + return (mode & 0x1) == 0x0; +} + +static inline bool +wasm_elem_is_passive(uint32 mode) +{ + return (mode & 0x1) == 0x1; +} + +static inline bool +wasm_elem_is_declarative(uint32 mode) +{ + return (mode & 0x3) == 0x3; +} + +bool +wasm_enlarge_table(WASMModuleInstance *module_inst, uint32 table_idx, + uint32 inc_entries, table_elem_type_t init_val); +#endif /* WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_GC != 0 +void * +wasm_create_func_obj(WASMModuleInstance *module_inst, uint32 func_idx, + bool throw_exce, char *error_buf, uint32 error_buf_size); + +bool +wasm_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap); + +#endif + +static inline WASMTableInstance * +wasm_get_table_inst(const WASMModuleInstance *module_inst, uint32 tbl_idx) +{ + /* careful, it might be a table in another module */ + WASMTableInstance *tbl_inst = module_inst->tables[tbl_idx]; +#if WASM_ENABLE_MULTI_MODULE != 0 + if (tbl_idx < module_inst->module->import_table_count + && module_inst->e->table_insts_linked[tbl_idx]) { + tbl_inst = module_inst->e->table_insts_linked[tbl_idx]; + } +#endif + bh_assert(tbl_inst); + return tbl_inst; +} + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + +#if WASM_ENABLE_COPY_CALL_STACK != 0 +uint32 +wasm_interp_copy_callstack(WASMExecEnv *exec_env, WASMCApiFrame *buffer, + uint32 length, uint32 skip_n, char *error_buf, + uint32_t error_buf_size); +#endif // WASM_ENABLE_COPY_CALL_STACK + +bool +wasm_interp_create_call_stack(struct WASMExecEnv *exec_env); + +/** + * @brief Dump wasm call stack or get the size + * + * @param exec_env the execution environment + * @param print whether to print to stdout or not + * @param buf buffer to store the dumped content + * @param len length of the buffer + * + * @return when print is true, return the bytes printed out to stdout; when + * print is false and buf is NULL, return the size required to store the + * callstack content; when print is false and buf is not NULL, return the size + * dumped to the buffer, 0 means error and data in buf may be invalid + */ +uint32 +wasm_interp_dump_call_stack(struct WASMExecEnv *exec_env, bool print, char *buf, + uint32 len); +#endif + +const uint8 * +wasm_loader_get_custom_section(WASMModule *module, const char *name, + uint32 *len); + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 +void +jit_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id); + +/** + * Check whether the app address and the buf is inside the linear memory, + * and convert the app address into native address + */ +bool +jit_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, + uint64 app_buf_addr, uint64 app_buf_size, + void **p_native_addr); +#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 */ + +#if WASM_ENABLE_FAST_JIT != 0 +bool +fast_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 type_idx, uint32 argc, uint32 *argv); + +bool +fast_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, + struct WASMInterpFrame *prev_frame); +#endif + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 +bool +llvm_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 argc, uint32 *argv); + +bool +llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, + uint32 *argv); + +#if WASM_ENABLE_BULK_MEMORY != 0 +bool +llvm_jit_memory_init(WASMModuleInstance *module_inst, uint32 seg_index, + uint32 offset, uint32 len, size_t dst); + +bool +llvm_jit_data_drop(WASMModuleInstance *module_inst, uint32 seg_index); +#endif + +#if WASM_ENABLE_REF_TYPES != 0 +void +llvm_jit_drop_table_seg(WASMModuleInstance *module_inst, uint32 tbl_seg_idx); + +void +llvm_jit_table_init(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 tbl_seg_idx, uint32 length, uint32 src_offset, + uint32 dst_offset); + +void +llvm_jit_table_copy(WASMModuleInstance *module_inst, uint32 src_tbl_idx, + uint32 dst_tbl_idx, uint32 length, uint32 src_offset, + uint32 dst_offset); + +void +llvm_jit_table_fill(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 length, uintptr_t val, uint32 data_offset); + +uint32 +llvm_jit_table_grow(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 inc_entries, uintptr_t init_val); +#endif + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0 \ + || WASM_ENABLE_AOT_STACK_FRAME != 0 +bool +llvm_jit_alloc_frame(WASMExecEnv *exec_env, uint32 func_index); + +void +llvm_jit_free_frame(WASMExecEnv *exec_env); + +void +llvm_jit_frame_update_profile_info(WASMExecEnv *exec_env, bool alloc_frame); +#endif + +#if WASM_ENABLE_GC != 0 +void * +llvm_jit_create_func_obj(WASMModuleInstance *module_inst, uint32 func_idx, + bool throw_exce, char *error_buf, + uint32 error_buf_size); + +bool +llvm_jit_obj_is_instance_of(WASMModuleInstance *module_inst, + WASMObjectRef gc_obj, uint32 type_index); + +/* Whether func type1 is one of super types of func type2 */ +bool +llvm_jit_func_type_is_super_of(WASMModuleInstance *module_inst, + uint32 type_idx1, uint32 type_idx2); + +WASMRttTypeRef +llvm_jit_rtt_type_new(WASMModuleInstance *module_inst, uint32 type_index); + +bool +llvm_array_init_with_data(WASMModuleInstance *module_inst, uint32 seg_index, + uint32 data_seg_offset, WASMArrayObjectRef array_obj, + uint32 elem_size, uint32 array_len); +#endif +#endif /* end of WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 */ + +#if WASM_ENABLE_LIBC_WASI != 0 && WASM_ENABLE_MULTI_MODULE != 0 +void +wasm_propagate_wasi_args(WASMModule *module); +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 +void +exception_lock(WASMModuleInstance *module_inst); +void +exception_unlock(WASMModuleInstance *module_inst); +#else +#define exception_lock(module_inst) (void)(module_inst) +#define exception_unlock(module_inst) (void)(module_inst) +#endif + +bool +wasm_check_utf8_str(const uint8 *str, uint32 len); + +char * +wasm_const_str_list_insert(const uint8 *str, uint32 len, WASMModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size); + +bool +wasm_set_module_name(WASMModule *module, const char *name, char *error_buf, + uint32_t error_buf_size); + +const char * +wasm_get_module_name(WASMModule *module); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_RUNTIME_H */ diff --git a/wamr/include/wasm_runtime_common.h b/wamr/include/wasm_runtime_common.h new file mode 100644 index 0000000..6ec4704 --- /dev/null +++ b/wamr/include/wasm_runtime_common.h @@ -0,0 +1,1390 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_COMMON_H +#define _WASM_COMMON_H + +#include "bh_platform.h" // bh_platform.h +#include "bh_common.h" // bh_common.h +#include "wasm_exec_env.h" // wasm_exec_env.h +#include "wasm_native.h" // wasm_native.h +#include "wasm_export.h" // ../include/wasm_export.h +#include "wasm.h" // ../interpreter/wasm.h + + +#if WASM_ENABLE_LIBC_WASI != 0 +#if WASM_ENABLE_UVWASI == 0 +#include "posix.h" // posix.h +#else + +#endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* Internal use for setting default running mode */ +#define Mode_Default 0 + +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + +#define PUT_I64_TO_ADDR(addr, value) \ + do { \ + *(int64 *)(addr) = (int64)(value); \ + } while (0) +#define PUT_V128_TO_ADDR(addr, value) \ + do { \ + *(V128 *)(addr) = (value); \ + } while (0) +#define PUT_F64_TO_ADDR(addr, value) \ + do { \ + *(float64 *)(addr) = (float64)(value); \ + } while (0) +#define PUT_REF_TO_ADDR(addr, value) \ + do { \ + *(void **)(addr) = (void *)(value); \ + } while (0) + +#define GET_I64_FROM_ADDR(addr) (*(int64 *)(addr)) +#define GET_F64_FROM_ADDR(addr) (*(float64 *)(addr)) +#define GET_REF_FROM_ADDR(addr) (*(void **)(addr)) +#define GET_V128_FROM_ADDR(addr) (*(V128 *)(addr)) + +/* For STORE opcodes */ +#define STORE_I64 PUT_I64_TO_ADDR +static inline void +STORE_U32(void *addr, uint32_t value) +{ + *(uint32_t *)(addr) = (uint32_t)(value); +} +static inline void +STORE_U16(void *addr, uint16_t value) +{ + *(uint16_t *)(addr) = (uint16_t)(value); +} +static inline void +STORE_U8(void *addr, uint8_t value) +{ + *(uint8 *)addr = value; +} + +static inline void +STORE_V128(void *addr, V128 value) +{ + *(V128 *)addr = value; +} + +/* For LOAD opcodes */ +#define LOAD_I64(addr) (*(int64 *)(addr)) +#define LOAD_F64(addr) (*(float64 *)(addr)) +#define LOAD_I32(addr) (*(int32 *)(addr)) +#define LOAD_U32(addr) (*(uint32 *)(addr)) +#define LOAD_I16(addr) (*(int16 *)(addr)) +#define LOAD_U16(addr) (*(uint16 *)(addr)) +#define LOAD_V128(addr) (*(V128 *)(addr)) + +#define STORE_PTR(addr, ptr) \ + do { \ + *(void **)addr = (void *)ptr; \ + } while (0) + +#else /* WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 */ + +#define PUT_V128_TO_ADDR(addr, value) \ + do { \ + uint32 *addr_u32 = (uint32 *)(addr); \ + addr_u32[0] = (value).i32x4[0]; \ + addr_u32[1] = (value).i32x4[1]; \ + addr_u32[2] = (value).i32x4[2]; \ + addr_u32[3] = (value).i32x4[3]; \ + } while (0) + +#define PUT_I64_TO_ADDR(addr, value) \ + do { \ + uint32 *addr_u32 = (uint32 *)(addr); \ + union { \ + int64 val; \ + uint32 parts[2]; \ + } u; \ + u.val = (int64)(value); \ + addr_u32[0] = u.parts[0]; \ + addr_u32[1] = u.parts[1]; \ + } while (0) +#define PUT_F64_TO_ADDR(addr, value) \ + do { \ + uint32 *addr_u32 = (uint32 *)(addr); \ + union { \ + float64 val; \ + uint32 parts[2]; \ + } u; \ + u.val = (value); \ + addr_u32[0] = u.parts[0]; \ + addr_u32[1] = u.parts[1]; \ + } while (0) +#if UINTPTR_MAX == UINT64_MAX +#define PUT_REF_TO_ADDR(addr, value) \ + do { \ + uint32 *addr_u32 = (uint32 *)(addr); \ + union { \ + void *val; \ + uint32 parts[2]; \ + } u; \ + u.val = (void *)(value); \ + addr_u32[0] = u.parts[0]; \ + addr_u32[1] = u.parts[1]; \ + } while (0) +#else +#define PUT_REF_TO_ADDR(addr, value) \ + do { \ + *(void **)(addr) = (void *)(value); \ + } while (0) +#endif + +static inline V128 +GET_V128_FROM_ADDR(uint32 *addr) +{ + V128 ret; + ret.i32x4[0] = addr[0]; + ret.i32x4[1] = addr[1]; + ret.i32x4[2] = addr[2]; + ret.i32x4[3] = addr[3]; + return ret; +} + +static inline int64 +GET_I64_FROM_ADDR(uint32 *addr) +{ + union { + int64 val; + uint32 parts[2]; + } u; + u.parts[0] = addr[0]; + u.parts[1] = addr[1]; + return u.val; +} + +static inline float64 +GET_F64_FROM_ADDR(uint32 *addr) +{ + union { + float64 val; + uint32 parts[2]; + } u; + u.parts[0] = addr[0]; + u.parts[1] = addr[1]; + return u.val; +} + +#if UINTPTR_MAX == UINT64_MAX +static inline void * +GET_REF_FROM_ADDR(uint32 *addr) +{ + union { + void *val; + uint32 parts[2]; + } u; + u.parts[0] = addr[0]; + u.parts[1] = addr[1]; + return u.val; +} +#else +#define GET_REF_FROM_ADDR(addr) (*(void **)(addr)) +#endif + +/* For STORE opcodes */ +#define STORE_I64(addr, value) \ + do { \ + uintptr_t addr_ = (uintptr_t)(addr); \ + union { \ + int64 val; \ + uint32 u32[2]; \ + uint16 u16[4]; \ + uint8 u8[8]; \ + } u; \ + if ((addr_ & (uintptr_t)7) == 0) \ + *(int64 *)(addr) = (int64)(value); \ + else { \ + u.val = (int64)(value); \ + if ((addr_ & (uintptr_t)3) == 0) { \ + ((uint32 *)(addr))[0] = u.u32[0]; \ + ((uint32 *)(addr))[1] = u.u32[1]; \ + } \ + else if ((addr_ & (uintptr_t)1) == 0) { \ + ((uint16 *)(addr))[0] = u.u16[0]; \ + ((uint16 *)(addr))[1] = u.u16[1]; \ + ((uint16 *)(addr))[2] = u.u16[2]; \ + ((uint16 *)(addr))[3] = u.u16[3]; \ + } \ + else { \ + int32 t; \ + for (t = 0; t < 8; t++) \ + ((uint8 *)(addr))[t] = u.u8[t]; \ + } \ + } \ + } while (0) + +static inline void +STORE_U32(void *addr, uint32_t value) +{ + uintptr_t addr_ = (uintptr_t)(addr); + union { + uint32_t val; + uint16_t u16[2]; + uint8_t u8[4]; + } u; + if ((addr_ & (uintptr_t)3) == 0) + *(uint32_t *)(addr) = (uint32_t)(value); + else { + u.val = (uint32_t)(value); + if ((addr_ & (uintptr_t)1) == 0) { + ((uint16_t *)(addr))[0] = u.u16[0]; + ((uint16_t *)(addr))[1] = u.u16[1]; + } + else { + ((uint8_t *)(addr))[0] = u.u8[0]; + ((uint8_t *)(addr))[1] = u.u8[1]; + ((uint8_t *)(addr))[2] = u.u8[2]; + ((uint8_t *)(addr))[3] = u.u8[3]; + } + } +} + +static inline void +STORE_U8(void *addr, uint8_t value) +{ + *(uint8 *)addr = value; +} + +static inline void +STORE_U16(void *addr, uint16_t value) +{ + union { + uint16_t val; + uint8_t u8[2]; + } u; + u.val = (uint16_t)(value); + ((uint8_t *)(addr))[0] = u.u8[0]; + ((uint8_t *)(addr))[1] = u.u8[1]; +} + +static inline void +STORE_V128(void *addr, V128 value) +{ + uintptr_t addr_ = (uintptr_t)(addr); + union { + V128 val; + uint64 u64[2]; + uint32 u32[4]; + uint16 u16[8]; + uint8 u8[16]; + } u; + + if ((addr_ & (uintptr_t)15) == 0) { + *(V128 *)addr = value; + } + else if ((addr_ & (uintptr_t)7) == 0) { + u.val = value; + ((uint64 *)(addr))[0] = u.u64[0]; + ((uint64 *)(addr))[1] = u.u64[1]; + } + else if ((addr_ & (uintptr_t)3) == 0) { + u.val = value; + ((uint32 *)addr)[0] = u.u32[0]; + ((uint32 *)addr)[1] = u.u32[1]; + ((uint32 *)addr)[2] = u.u32[2]; + ((uint32 *)addr)[3] = u.u32[3]; + } + else if ((addr_ & (uintptr_t)1) == 0) { + u.val = value; + ((uint16 *)addr)[0] = u.u16[0]; + ((uint16 *)addr)[1] = u.u16[1]; + ((uint16 *)addr)[2] = u.u16[2]; + ((uint16 *)addr)[3] = u.u16[3]; + ((uint16 *)addr)[4] = u.u16[4]; + ((uint16 *)addr)[5] = u.u16[5]; + ((uint16 *)addr)[6] = u.u16[6]; + ((uint16 *)addr)[7] = u.u16[7]; + } + else { + u.val = value; + for (int i = 0; i < 16; i++) + ((uint8 *)addr)[i] = u.u8[i]; + } +} + +/* For LOAD opcodes */ +static inline V128 +LOAD_V128(void *addr) +{ + uintptr_t addr1 = (uintptr_t)addr; + union { + V128 val; + uint64 u64[2]; + uint32 u32[4]; + uint16 u16[8]; + uint8 u8[16]; + } u; + if ((addr1 & (uintptr_t)15) == 0) + return *(V128 *)addr; + + if ((addr1 & (uintptr_t)7) == 0) { + u.u64[0] = ((uint64 *)addr)[0]; + u.u64[1] = ((uint64 *)addr)[1]; + } + else if ((addr1 & (uintptr_t)3) == 0) { + u.u32[0] = ((uint32 *)addr)[0]; + u.u32[1] = ((uint32 *)addr)[1]; + u.u32[2] = ((uint32 *)addr)[2]; + u.u32[3] = ((uint32 *)addr)[3]; + } + else if ((addr1 & (uintptr_t)1) == 0) { + u.u16[0] = ((uint16 *)addr)[0]; + u.u16[1] = ((uint16 *)addr)[1]; + u.u16[2] = ((uint16 *)addr)[2]; + u.u16[3] = ((uint16 *)addr)[3]; + u.u16[4] = ((uint16 *)addr)[4]; + u.u16[5] = ((uint16 *)addr)[5]; + u.u16[6] = ((uint16 *)addr)[6]; + u.u16[7] = ((uint16 *)addr)[7]; + } + else { + for (int i = 0; i < 16; i++) + u.u8[i] = ((uint8 *)addr)[i]; + } + return u.val; +} + +static inline int64 +LOAD_I64(void *addr) +{ + uintptr_t addr1 = (uintptr_t)addr; + union { + int64 val; + uint32 u32[2]; + uint16 u16[4]; + uint8 u8[8]; + } u; + if ((addr1 & (uintptr_t)7) == 0) + return *(int64 *)addr; + + if ((addr1 & (uintptr_t)3) == 0) { + u.u32[0] = ((uint32 *)addr)[0]; + u.u32[1] = ((uint32 *)addr)[1]; + } + else if ((addr1 & (uintptr_t)1) == 0) { + u.u16[0] = ((uint16 *)addr)[0]; + u.u16[1] = ((uint16 *)addr)[1]; + u.u16[2] = ((uint16 *)addr)[2]; + u.u16[3] = ((uint16 *)addr)[3]; + } + else { + int32 t; + for (t = 0; t < 8; t++) + u.u8[t] = ((uint8 *)addr)[t]; + } + return u.val; +} + +static inline float64 +LOAD_F64(void *addr) +{ + uintptr_t addr1 = (uintptr_t)addr; + union { + float64 val; + uint32 u32[2]; + uint16 u16[4]; + uint8 u8[8]; + } u; + if ((addr1 & (uintptr_t)7) == 0) + return *(float64 *)addr; + + if ((addr1 & (uintptr_t)3) == 0) { + u.u32[0] = ((uint32 *)addr)[0]; + u.u32[1] = ((uint32 *)addr)[1]; + } + else if ((addr1 & (uintptr_t)1) == 0) { + u.u16[0] = ((uint16 *)addr)[0]; + u.u16[1] = ((uint16 *)addr)[1]; + u.u16[2] = ((uint16 *)addr)[2]; + u.u16[3] = ((uint16 *)addr)[3]; + } + else { + int32 t; + for (t = 0; t < 8; t++) + u.u8[t] = ((uint8 *)addr)[t]; + } + return u.val; +} + +static inline int32 +LOAD_I32(void *addr) +{ + uintptr_t addr1 = (uintptr_t)addr; + union { + int32 val; + uint16 u16[2]; + uint8 u8[4]; + } u; + if ((addr1 & (uintptr_t)3) == 0) + return *(int32 *)addr; + + if ((addr1 & (uintptr_t)1) == 0) { + u.u16[0] = ((uint16 *)addr)[0]; + u.u16[1] = ((uint16 *)addr)[1]; + } + else { + u.u8[0] = ((uint8 *)addr)[0]; + u.u8[1] = ((uint8 *)addr)[1]; + u.u8[2] = ((uint8 *)addr)[2]; + u.u8[3] = ((uint8 *)addr)[3]; + } + return u.val; +} + +static inline int16 +LOAD_I16(void *addr) +{ + uintptr_t addr1 = (uintptr_t)addr; + union { + int16 val; + uint8 u8[2]; + } u; + if ((addr1 & (uintptr_t)1)) { + u.u8[0] = ((uint8 *)addr)[0]; + u.u8[1] = ((uint8 *)addr)[1]; + return u.val; + } + return *(int16 *)addr; +} + +#define LOAD_U32(addr) ((uint32)LOAD_I32(addr)) +#define LOAD_U16(addr) ((uint16)LOAD_I16(addr)) + +#if UINTPTR_MAX == UINT32_MAX +#define STORE_PTR(addr, ptr) STORE_U32(addr, (uintptr_t)ptr) +#elif UINTPTR_MAX == UINT64_MAX +#define STORE_PTR(addr, ptr) STORE_I64(addr, (uintptr_t)ptr) +#endif + +#endif /* WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 */ + +#if WASM_ENABLE_SHARED_MEMORY != 0 +#define SHARED_MEMORY_LOCK(memory) shared_memory_lock(memory) +#define SHARED_MEMORY_UNLOCK(memory) shared_memory_unlock(memory) +#else +#define SHARED_MEMORY_LOCK(memory) (void)0 +#define SHARED_MEMORY_UNLOCK(memory) (void)0 +#endif + +#define CLAMP_U64_TO_U32(value) \ + ((value) > UINT32_MAX ? UINT32_MAX : (uint32)(value)) + +typedef struct WASMModuleCommon { + /* Module type, for module loaded from WASM bytecode binary, + this field is Wasm_Module_Bytecode, and this structure should + be treated as WASMModule structure; + for module loaded from AOT binary, this field is + Wasm_Module_AoT, and this structure should be treated as + AOTModule structure. */ + uint32 module_type; + + /* The following uint8[1] member is a dummy just to indicate + some module_type dependent members follow. + Typically, it should be accessed by casting to the corresponding + actual module_type dependent structure, not via this member. */ + uint8 module_data[1]; +} WASMModuleCommon; + +typedef struct WASMModuleInstanceCommon { + /* Module instance type, for module instance loaded from WASM + bytecode binary, this field is Wasm_Module_Bytecode, and this + structure should be treated as WASMModuleInstance structure; + for module instance loaded from AOT binary, this field is + Wasm_Module_AoT, and this structure should be treated as + AOTModuleInstance structure. */ + uint32 module_type; + + /* The following uint8[1] member is a dummy just to indicate + some module_type dependent members follow. + Typically, it should be accessed by casting to the corresponding + actual module_type dependent structure, not via this member. */ + uint8 module_inst_data[1]; +} WASMModuleInstanceCommon; + +typedef struct WASMModuleMemConsumption { + uint32 total_size; + uint32 module_struct_size; + uint32 types_size; + uint32 imports_size; + uint32 functions_size; + uint32 tables_size; + uint32 memories_size; + uint32 globals_size; + uint32 exports_size; + uint32 table_segs_size; + uint32 data_segs_size; + uint32 const_strs_size; +#if WASM_ENABLE_AOT != 0 + uint32 aot_code_size; +#endif +} WASMModuleMemConsumption; + +typedef struct WASMModuleInstMemConsumption { + uint64 total_size; + uint32 module_inst_struct_size; + uint32 app_heap_size; + uint64 memories_size; + uint32 tables_size; + uint32 globals_size; + uint32 functions_size; + uint32 exports_size; +} WASMModuleInstMemConsumption; + +#if WASM_ENABLE_LIBC_WASI != 0 +#if WASM_ENABLE_UVWASI == 0 +typedef struct WASIContext { + struct fd_table *curfds; + struct fd_prestats *prestats; + struct argv_environ_values *argv_environ; + struct addr_pool *addr_pool; + char *ns_lookup_buf; + char **ns_lookup_list; + char *argv_buf; + char **argv_list; + char *env_buf; + char **env_list; + uint32_t exit_code; +} WASIContext; +#else +typedef struct WASIContext { + uvwasi_t uvwasi; + uint32_t exit_code; +} WASIContext; +#endif +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 +typedef struct WASMRegisteredModule { + bh_list_link l; + /* point to a string pool */ + const char *module_name; + WASMModuleCommon *module; + /* to store the original module file buffer address */ + uint8 *orig_file_buf; + uint32 orig_file_buf_size; +} WASMRegisteredModule; +#endif + +typedef package_type_t PackageType; +typedef wasm_section_t WASMSection, AOTSection; + +#if WASM_ENABLE_JIT != 0 +typedef struct LLVMJITOptions { + uint32 opt_level; + uint32 size_level; + uint32 segue_flags; + bool quick_invoke_c_api_import; +} LLVMJITOptions; +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK +/* Signal info passing to interp/aot signal handler */ +typedef struct WASMSignalInfo { + WASMExecEnv *exec_env_tls; +#ifndef BH_PLATFORM_WINDOWS + void *sig_addr; +#else + EXCEPTION_POINTERS *exce_info; +#endif +} WASMSignalInfo; + +/* Set exec_env of thread local storage */ +void +wasm_runtime_set_exec_env_tls(WASMExecEnv *exec_env); + +/* Get exec_env of thread local storage */ +WASMExecEnv * +wasm_runtime_get_exec_env_tls(void); +#endif + +struct InstantiationArgs2 { + InstantiationArgs v1; +}; + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_init(void); + +/* Internal API */ +RunningMode +wasm_runtime_get_default_running_mode(void); + +#if WASM_ENABLE_JIT != 0 +/* Internal API */ +LLVMJITOptions * +wasm_runtime_get_llvm_jit_options(void); +#endif + +#if WASM_ENABLE_GC != 0 +/* Internal API */ +uint32 +wasm_runtime_get_gc_heap_size_default(void); +#endif + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_full_init(RuntimeInitArgs *init_args); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_running_mode_supported(RunningMode running_mode); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_set_default_running_mode(RunningMode running_mode); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_destroy(void); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN PackageType +get_package_type(const uint8 *buf, uint32 size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_xip_file(const uint8 *buf, uint32 size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMModuleCommon * +wasm_runtime_load(uint8 *buf, uint32 size, char *error_buf, + uint32 error_buf_size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMModuleCommon * +wasm_runtime_load_from_sections(WASMSection *section_list, bool is_aot, + char *error_buf, uint32 error_buf_size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_unload(WASMModuleCommon *module); + +/* Internal API */ +uint32 +wasm_runtime_get_max_mem(uint32 max_memory_pages, uint32 module_init_page_count, + uint32 module_max_page_count); + +/* Internal API */ +WASMModuleInstanceCommon * +wasm_runtime_instantiate_internal(WASMModuleCommon *module, + WASMModuleInstanceCommon *parent, + WASMExecEnv *exec_env_main, uint32 stack_size, + uint32 heap_size, uint32 max_memory_pages, + char *error_buf, uint32 error_buf_size); + +/* Internal API */ +void +wasm_runtime_deinstantiate_internal(WASMModuleInstanceCommon *module_inst, + bool is_sub_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMModuleInstanceCommon * +wasm_runtime_instantiate(WASMModuleCommon *module, uint32 default_stack_size, + uint32 host_managed_heap_size, char *error_buf, + uint32 error_buf_size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMModuleInstanceCommon * +wasm_runtime_instantiate_ex(WASMModuleCommon *module, + const InstantiationArgs *args, char *error_buf, + uint32 error_buf_size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN +bool +wasm_runtime_instantiation_args_create(struct InstantiationArgs2 **p); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN +void +wasm_runtime_instantiation_args_destroy(struct InstantiationArgs2 *p); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN +void +wasm_runtime_instantiation_args_set_default_stack_size( + struct InstantiationArgs2 *p, uint32 v); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN +void +wasm_runtime_instantiation_args_set_host_managed_heap_size( + struct InstantiationArgs2 *p, uint32 v); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN +void +wasm_runtime_instantiation_args_set_max_memory_pages( + struct InstantiationArgs2 *p, uint32 v); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMModuleInstanceCommon * +wasm_runtime_instantiate_ex2(WASMModuleCommon *module, + const struct InstantiationArgs2 *args, + char *error_buf, uint32 error_buf_size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_set_running_mode(wasm_module_inst_t module_inst, + RunningMode running_mode); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN RunningMode +wasm_runtime_get_running_mode(wasm_module_inst_t module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_deinstantiate(WASMModuleInstanceCommon *module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMModuleCommon * +wasm_runtime_get_module(WASMModuleInstanceCommon *module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMFunctionInstanceCommon * +wasm_runtime_lookup_function(WASMModuleInstanceCommon *const module_inst, + const char *name); + +/* Internal API */ +WASMFuncType * +wasm_runtime_get_function_type(const WASMFunctionInstanceCommon *function, + uint32 module_type); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN uint32 +wasm_func_get_param_count(WASMFunctionInstanceCommon *const func_inst, + WASMModuleInstanceCommon *const module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN uint32 +wasm_func_get_result_count(WASMFunctionInstanceCommon *const func_inst, + WASMModuleInstanceCommon *const module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_func_get_param_types(WASMFunctionInstanceCommon *const func_inst, + WASMModuleInstanceCommon *const module_inst, + wasm_valkind_t *param_types); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_func_get_result_types(WASMFunctionInstanceCommon *const func_inst, + WASMModuleInstanceCommon *const module_inst, + wasm_valkind_t *result_types); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMExecEnv * +wasm_runtime_create_exec_env(WASMModuleInstanceCommon *module_inst, + uint32 stack_size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_destroy_exec_env(WASMExecEnv *exec_env); + +#if WASM_ENABLE_COPY_CALL_STACK != 0 +WASM_RUNTIME_API_EXTERN uint32_t +wasm_copy_callstack(const wasm_exec_env_t exec_env, WASMCApiFrame *buffer, + const uint32 length, const uint32 skip_n, char *error_buf, + uint32 error_buf_size); +#endif // WASM_ENABLE_COPY_CALL_STACK + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMModuleInstanceCommon * +wasm_runtime_get_module_inst(WASMExecEnv *exec_env); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_module_inst(WASMExecEnv *exec_env, + WASMModuleInstanceCommon *const module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_get_function_attachment(WASMExecEnv *exec_env); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_user_data(WASMExecEnv *exec_env, void *user_data); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_get_user_data(WASMExecEnv *exec_env); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_native_stack_boundary(WASMExecEnv *exec_env, + uint8 *native_stack_boundary); + +#if WASM_ENABLE_INSTRUCTION_METERING != 0 +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_instruction_count_limit(WASMExecEnv *exec_env, + int instructions_to_execute); +#endif + +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN +void +wasm_runtime_set_bounds_checks(WASMModuleInstanceCommon *module_inst, + bool enable); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst); +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK +/* Access exception check guard page to trigger the signal handler */ +void +wasm_runtime_access_exce_check_guard_page(); +#endif + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_wasm(WASMExecEnv *exec_env, + WASMFunctionInstanceCommon *function, uint32 argc, + uint32 argv[]); + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_wasm_a(WASMExecEnv *exec_env, + WASMFunctionInstanceCommon *function, + uint32 num_results, wasm_val_t *results, + uint32 num_args, wasm_val_t *args); + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_wasm_v(WASMExecEnv *exec_env, + WASMFunctionInstanceCommon *function, + uint32 num_results, wasm_val_t *results, + uint32 num_args, ...); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_call_indirect(WASMExecEnv *exec_env, uint32 element_index, + uint32 argc, uint32 argv[]); + +#if WASM_ENABLE_DEBUG_INTERP != 0 +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN uint32 +wasm_runtime_start_debug_instance_with_port(WASMExecEnv *exec_env, + int32_t port); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN uint32 +wasm_runtime_start_debug_instance(WASMExecEnv *exec_env); +#endif + +bool +wasm_runtime_create_exec_env_singleton(WASMModuleInstanceCommon *module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMExecEnv * +wasm_runtime_get_exec_env_singleton(WASMModuleInstanceCommon *module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_application_execute_main(WASMModuleInstanceCommon *module_inst, int32 argc, + char *argv[]); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_application_execute_func(WASMModuleInstanceCommon *module_inst, + const char *name, int32 argc, char *argv[]); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_exception(WASMModuleInstanceCommon *module, + const char *exception); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN const char * +wasm_runtime_get_exception(WASMModuleInstanceCommon *module); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_clear_exception(WASMModuleInstanceCommon *module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_terminate(WASMModuleInstanceCommon *module); + +/* Internal API */ +void +wasm_runtime_set_custom_data_internal(WASMModuleInstanceCommon *module_inst, + void *custom_data); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_custom_data(WASMModuleInstanceCommon *module_inst, + void *custom_data); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_get_custom_data(WASMModuleInstanceCommon *module_inst); + +/* Internal API */ +uint64 +wasm_runtime_module_malloc_internal(WASMModuleInstanceCommon *module_inst, + WASMExecEnv *exec_env, uint64 size, + void **p_native_addr); + +/* Internal API */ +uint64 +wasm_runtime_module_realloc_internal(WASMModuleInstanceCommon *module_inst, + WASMExecEnv *exec_env, uint64 ptr, + uint64 size, void **p_native_addr); + +/* Internal API */ +void +wasm_runtime_module_free_internal(WASMModuleInstanceCommon *module_inst, + WASMExecEnv *exec_env, uint64 ptr); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN uint64 +wasm_runtime_module_malloc(WASMModuleInstanceCommon *module_inst, uint64 size, + void **p_native_addr); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_module_free(WASMModuleInstanceCommon *module_inst, uint64 ptr); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN uint64 +wasm_runtime_module_dup_data(WASMModuleInstanceCommon *module_inst, + const char *src, uint64 size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst, + uint64 app_offset, uint64 size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst, + uint64 app_str_offset); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst, + void *native_ptr, uint64 size); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst, + uint64 app_offset); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN uint64 +wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst, + void *native_ptr); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_get_app_addr_range(WASMModuleInstanceCommon *module_inst, + uint64 app_offset, uint64 *p_app_start_offset, + uint64 *p_app_end_offset); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_get_native_addr_range(WASMModuleInstanceCommon *module_inst, + uint8 *native_ptr, + uint8 **p_native_start_addr, + uint8 **p_native_end_addr); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN const uint8 * +wasm_runtime_get_custom_section(WASMModuleCommon *const module_comm, + const char *name, uint32 *len); + +#if WASM_ENABLE_MULTI_MODULE != 0 +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_module_reader(const module_reader reader, + const module_destroyer destroyer); + +module_reader +wasm_runtime_get_module_reader(void); + +module_destroyer +wasm_runtime_get_module_destroyer(void); + +bool +wasm_runtime_register_module_internal(const char *module_name, + WASMModuleCommon *module, + uint8 *orig_file_buf, + uint32 orig_file_buf_size, + char *error_buf, uint32 error_buf_size); + +void +wasm_runtime_unregister_module(const WASMModuleCommon *module); + +WASMModuleCommon * +wasm_runtime_find_module_registered(const char *module_name); + +bool +wasm_runtime_add_loading_module(const char *module_name, char *error_buf, + uint32 error_buf_size); + +void +wasm_runtime_delete_loading_module(const char *module_name); + +bool +wasm_runtime_is_loading_module(const char *module_name); + +void +wasm_runtime_destroy_loading_module_list(void); + +WASMModuleCommon * +wasm_runtime_search_sub_module(const WASMModuleCommon *parent_module, + const char *sub_module_name); + +bool +wasm_runtime_register_sub_module(const WASMModuleCommon *parent_module, + const char *sub_module_name, + WASMModuleCommon *sub_module); + +WASMModuleCommon * +wasm_runtime_load_depended_module(const WASMModuleCommon *parent_module, + const char *sub_module_name, char *error_buf, + uint32 error_buf_size); + +bool +wasm_runtime_sub_module_instantiate(WASMModuleCommon *module, + WASMModuleInstanceCommon *module_inst, + uint32 stack_size, uint32 heap_size, + uint32 max_memory_pages, char *error_buf, + uint32 error_buf_size); +void +wasm_runtime_sub_module_deinstantiate(WASMModuleInstanceCommon *module_inst); +#endif + +#if WASM_ENABLE_LIBC_WASI != 0 || WASM_ENABLE_MULTI_MODULE != 0 +WASMExport * +loader_find_export(const WASMModuleCommon *module, const char *module_name, + const char *field_name, uint8 export_kind, char *error_buf, + uint32 error_buf_size); +#endif /* WASM_ENABLE_MULTI_MODULE */ + +bool +wasm_runtime_is_built_in_module(const char *module_name); + +#if WASM_ENABLE_THREAD_MGR != 0 +bool +wasm_exec_env_get_aux_stack(WASMExecEnv *exec_env, uint64 *start_offset, + uint32 *size); + +bool +wasm_exec_env_set_aux_stack(WASMExecEnv *exec_env, uint64 start_offset, + uint32 size); +#endif + +#if WASM_ENABLE_LIBC_WASI != 0 +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_wasi_args_ex(WASMModuleCommon *module, const char *dir_list[], + uint32 dir_count, const char *map_dir_list[], + uint32 map_dir_count, const char *env_list[], + uint32 env_count, char *argv[], int argc, + int64 stdinfd, int64 stdoutfd, int64 stderrfd); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_wasi_args(WASMModuleCommon *module, const char *dir_list[], + uint32 dir_count, const char *map_dir_list[], + uint32 map_dir_count, const char *env_list[], + uint32 env_count, char *argv[], int argc); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_wasi_mode(WASMModuleInstanceCommon *module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMFunctionInstanceCommon * +wasm_runtime_lookup_wasi_start_function(WASMModuleInstanceCommon *module_inst); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN uint32_t +wasm_runtime_get_wasi_exit_code(WASMModuleInstanceCommon *module_inst); + +void +wasi_args_set_defaults(WASIArguments *args); + +bool +wasm_runtime_init_wasi(WASMModuleInstanceCommon *module_inst, + const char *dir_list[], uint32 dir_count, + const char *map_dir_list[], uint32 map_dir_count, + const char *env[], uint32 env_count, + const char *addr_pool[], uint32 addr_pool_size, + const char *ns_lookup_pool[], uint32 ns_lookup_pool_size, + char *argv[], uint32 argc, os_raw_file_handle stdinfd, + os_raw_file_handle stdoutfd, os_raw_file_handle stderrfd, + char *error_buf, uint32 error_buf_size); + +void +wasm_runtime_destroy_wasi(WASMModuleInstanceCommon *module_inst); + +void +wasm_runtime_set_wasi_ctx(WASMModuleInstanceCommon *module_inst, + WASIContext *wasi_ctx); + +WASIContext * +wasm_runtime_get_wasi_ctx(WASMModuleInstanceCommon *module_inst); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_wasi_addr_pool(wasm_module_t module, const char *addr_pool[], + uint32 addr_pool_size); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_wasi_ns_lookup_pool(wasm_module_t module, + const char *ns_lookup_pool[], + uint32 ns_lookup_pool_size); +#endif /* end of WASM_ENABLE_LIBC_WASI */ + +#if WASM_ENABLE_GC != 0 +void +wasm_runtime_set_gc_heap_handle(WASMModuleInstanceCommon *module_inst, + void *gc_heap_handle); + +void * +wasm_runtime_get_gc_heap_handle(WASMModuleInstanceCommon *module_inst); +#endif + +#if WASM_ENABLE_REF_TYPES != 0 +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_obj2ref(WASMModuleInstanceCommon *module_inst, void *extern_obj, + uint32 *p_externref_idx); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_ref2obj(uint32 externref_idx, void **p_extern_obj); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_retain(uint32 externref_idx); + +/** + * Reclaim the externref objects/indexes which are not used by + * module instance + */ +void +wasm_externref_reclaim(WASMModuleInstanceCommon *module_inst); + +/** + * Cleanup the externref objects/indexes of the module instance + */ +void +wasm_externref_cleanup(WASMModuleInstanceCommon *module_inst); +#endif /* end of WASM_ENABLE_REF_TYPES */ + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 +/** + * @brief Internal implementation for dumping or printing callstack line + * + * @note if dump_or_print is true, then print to stdout directly; + * if dump_or_print is false, but *buf is NULL, then return the length of the + * line; + * if dump_or_print is false, and *buf is not NULL, then dump content to + * the memory pointed by *buf, and adjust *buf and *len according to actual + * bytes dumped, and return the actual dumped length + * + * @param line_buf current line to dump or print + * @param dump_or_print whether to print to stdout or dump to buf + * @param buf [INOUT] pointer to the buffer + * @param len [INOUT] pointer to remaining length + * @return bytes printed to stdout or dumped to buf + */ +uint32 +wasm_runtime_dump_line_buf_impl(const char *line_buf, bool dump_or_print, + char **buf, uint32 *len); +#endif /* end of WASM_ENABLE_DUMP_CALL_STACK != 0 */ + +/* Get module of the current exec_env */ +WASMModuleCommon * +wasm_exec_env_get_module(WASMExecEnv *exec_env); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_register_natives(const char *module_name, + NativeSymbol *native_symbols, + uint32 n_native_symbols); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_register_natives_raw(const char *module_name, + NativeSymbol *native_symbols, + uint32 n_native_symbols); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_unregister_natives(const char *module_name, + NativeSymbol *native_symbols); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_create_context_key(void (*dtor)(WASMModuleInstanceCommon *inst, + void *ctx)); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_destroy_context_key(void *key); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_context(WASMModuleInstanceCommon *inst, void *key, void *ctx); +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_set_context_spread(WASMModuleInstanceCommon *inst, void *key, + void *ctx); +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN void * +wasm_runtime_get_context(WASMModuleInstanceCommon *inst, void *key); + +bool +wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, + const WASMFuncType *func_type, const char *signature, + void *attachment, uint32 *argv, uint32 argc, + uint32 *ret); + +bool +wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, + const WASMFuncType *func_type, + const char *signature, void *attachment, + uint32 *argv, uint32 argc, uint32 *ret); + +void +wasm_runtime_read_v128(const uint8 *bytes, uint64 *ret1, uint64 *ret2); + +void +wasm_runtime_dump_module_mem_consumption(const WASMModuleCommon *module); + +void +wasm_runtime_dump_module_inst_mem_consumption( + const WASMModuleInstanceCommon *module_inst); + +void +wasm_runtime_dump_exec_env_mem_consumption(const WASMExecEnv *exec_env); + +bool +wasm_runtime_get_table_elem_type(const WASMModuleCommon *module_comm, + uint32 table_idx, uint8 *out_elem_type, +#if WASM_ENABLE_GC != 0 + WASMRefType **out_ref_type, +#endif + uint32 *out_min_size, uint32 *out_max_size); + +bool +wasm_runtime_get_table_inst_elem_type( + const WASMModuleInstanceCommon *module_inst_comm, uint32 table_idx, + uint8 *out_elem_type, +#if WASM_ENABLE_GC != 0 + WASMRefType **out_ref_type, +#endif + uint32 *out_min_size, uint32 *out_max_size); + +bool +wasm_runtime_get_export_func_type(const WASMModuleCommon *module_comm, + const WASMExport *export_, + WASMFuncType **out); + +bool +wasm_runtime_get_export_global_type(const WASMModuleCommon *module_comm, + const WASMExport *export_, + uint8 *out_val_type, bool *out_mutability); + +bool +wasm_runtime_get_export_memory_type(const WASMModuleCommon *module_comm, + const WASMExport *export_, + uint32 *out_min_page, uint32 *out_max_page); + +bool +wasm_runtime_get_export_table_type(const WASMModuleCommon *module_comm, + const WASMExport *export_, + uint8 *out_elem_type, +#if WASM_ENABLE_GC != 0 + WASMRefType **out_ref_type, +#endif + uint32 *out_min_size, uint32 *out_max_size); + +bool +wasm_runtime_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, + void *func_ptr, WASMFuncType *func_type, + uint32 argc, uint32 *argv, bool with_env, + void *wasm_c_api_env); + +struct CApiFuncImport; +/* A quick version of wasm_runtime_invoke_c_api_native to directly invoke + wasm-c-api import function from jitted code to improve performance */ +bool +wasm_runtime_quick_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, + struct CApiFuncImport *c_api_import, + wasm_val_t *params, uint32 param_count, + wasm_val_t *results, + uint32 result_count); + +void +wasm_runtime_show_app_heap_corrupted_prompt(void); + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 +void +wasm_runtime_destroy_custom_sections(WASMCustomSection *section_list); +#endif + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_import_func_linked(const char *module_name, + const char *func_name); + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_import_global_linked(const char *module_name, + const char *global_name); + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_begin_blocking_op(WASMExecEnv *exec_env); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_end_blocking_op(WASMExecEnv *exec_env); + +void +wasm_runtime_interrupt_blocking_op(WASMExecEnv *exec_env); + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_detect_native_stack_overflow(WASMExecEnv *exec_env); + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_detect_native_stack_overflow_size(WASMExecEnv *exec_env, + uint32 requested_size); + +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_is_underlying_binary_freeable(WASMModuleCommon *const module); + +#if WASM_ENABLE_LINUX_PERF != 0 +bool +wasm_runtime_get_linux_perf(void); + +void +wasm_runtime_set_linux_perf(bool flag); +#endif + +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +wasm_runtime_check_and_update_last_used_shared_heap( + WASMModuleInstanceCommon *module_inst, uintptr_t app_offset, size_t bytes, + uintptr_t *shared_heap_start_off_p, uintptr_t *shared_heap_end_off_p, + uint8 **shared_heap_base_addr_adj_p, bool is_memory64); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_COMMON_H */ diff --git a/wamr/include/wasm_shared_memory.h b/wamr/include/wasm_shared_memory.h new file mode 100644 index 0000000..c8a4a9d --- /dev/null +++ b/wamr/include/wasm_shared_memory.h @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_SHARED_MEMORY_H +#define _WASM_SHARED_MEMORY_H + +#include "bh_common.h" // bh_common.h +#include "wasm_runtime.h" // ../interpreter/wasm_runtime.h +#include "wasm_runtime_common.h" // wasm_runtime_common.h + +#ifdef __cplusplus +extern "C" { +#endif + +extern korp_mutex g_shared_memory_lock; + +bool +wasm_shared_memory_init(void); + +void +wasm_shared_memory_destroy(void); + +uint16 +shared_memory_inc_reference(WASMMemoryInstance *memory); + +uint16 +shared_memory_dec_reference(WASMMemoryInstance *memory); + +#define shared_memory_is_shared(memory) memory->is_shared_memory + +#define shared_memory_lock(memory) \ + do { \ + /* \ + * Note: exception logic is currently abusing this lock. \ + * cf. \ + * https://github.com/bytecodealliance/wasm-micro-runtime/issues/2407 \ + */ \ + bh_assert(memory != NULL); \ + if (memory->is_shared_memory) \ + os_mutex_lock(&g_shared_memory_lock); \ + } while (0) + +#define shared_memory_unlock(memory) \ + do { \ + if (memory->is_shared_memory) \ + os_mutex_unlock(&g_shared_memory_lock); \ + } while (0) + +uint32 +wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, + uint64 expect, int64 timeout, bool wait64); + +uint32 +wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address, + uint32 count); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_SHARED_MEMORY_H */ diff --git a/wamr/include/wasm_suspend_flags.h b/wamr/include/wasm_suspend_flags.h new file mode 100644 index 0000000..845a081 --- /dev/null +++ b/wamr/include/wasm_suspend_flags.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2023 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _WASM_SUSPEND_FLAGS_H +#define _WASM_SUSPEND_FLAGS_H + +#include "bh_atomic.h" // bh_atomic.h + +#ifdef __cplusplus +extern "C" { +#endif + +/* Need to terminate */ +#define WASM_SUSPEND_FLAG_TERMINATE 0x1 +/* Need to suspend */ +#define WASM_SUSPEND_FLAG_SUSPEND 0x2 +/* Need to go into breakpoint */ +#define WASM_SUSPEND_FLAG_BREAKPOINT 0x4 +/* Return from pthread_exit */ +#define WASM_SUSPEND_FLAG_EXIT 0x8 +/* The thread might be blocking */ +#define WASM_SUSPEND_FLAG_BLOCKING 0x10 + +typedef union WASMSuspendFlags { + bh_atomic_32_t flags; + uintptr_t __padding__; +} WASMSuspendFlags; + +#define WASM_SUSPEND_FLAGS_IS_ATOMIC BH_ATOMIC_32_IS_ATOMIC +#define WASM_SUSPEND_FLAGS_GET(s_flags) BH_ATOMIC_32_LOAD(s_flags.flags) +#define WASM_SUSPEND_FLAGS_FETCH_OR(s_flags, val) \ + BH_ATOMIC_32_FETCH_OR(s_flags.flags, val) +#define WASM_SUSPEND_FLAGS_FETCH_AND(s_flags, val) \ + BH_ATOMIC_32_FETCH_AND(s_flags.flags, val) + +#define WASM_SUSPEND_FLAG_INHERIT_MASK (~WASM_SUSPEND_FLAG_BLOCKING) + +#if WASM_SUSPEND_FLAGS_IS_ATOMIC != 0 +#define WASM_SUSPEND_FLAGS_LOCK(lock) (void)0 +#define WASM_SUSPEND_FLAGS_UNLOCK(lock) (void)0 +#else /* else of WASM_SUSPEND_FLAGS_IS_ATOMIC */ +#define WASM_SUSPEND_FLAGS_LOCK(lock) os_mutex_lock(&lock) +#define WASM_SUSPEND_FLAGS_UNLOCK(lock) os_mutex_unlock(&lock); +#endif /* WASM_SUSPEND_FLAGS_IS_ATOMIC */ + +#ifdef __cplusplus +} +#endif + +#endif /* end of _WASM_SUSPEND_FLAGS_H */ diff --git a/wamr/src/bh_assert.c b/wamr/src/bh_assert.c new file mode 100644 index 0000000..78e2aa8 --- /dev/null +++ b/wamr/src/bh_assert.c @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_assert.h" // bh_assert.h + +void +bh_assert_internal(int64 v, const char *file_name, int line_number, + const char *expr_string) +{ + if (v) + return; + + if (!file_name) + file_name = "NULL FILENAME"; + + if (!expr_string) + expr_string = "NULL EXPR_STRING"; + + LOG_ERROR("\nASSERTION FAILED: %s, at file %s, line %d\n", expr_string, + file_name, line_number); + + abort(); +} diff --git a/wamr/src/bh_common.c b/wamr/src/bh_common.c new file mode 100644 index 0000000..a121e00 --- /dev/null +++ b/wamr/src/bh_common.c @@ -0,0 +1,217 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_common.h" // bh_common.h + +static char * +align_ptr(char *src, unsigned int b) +{ + uintptr_t v = (uintptr_t)src; + uintptr_t m = b - 1; + return (char *)((v + m) & ~m); +} + +/* +Memory copy, with word alignment +*/ +int +b_memcpy_wa(void *s1, unsigned int s1max, const void *s2, unsigned int n) +{ + char *dest = (char *)s1; + char *src = (char *)s2; + + char *pa = align_ptr(src, 4); + char *pb = align_ptr((src + n), 4); + + unsigned int buff; + const char *p_byte_read; + + unsigned int *p; + char *ps; + + if (n == 0) { + return 0; + } + + if (pa > src) { + pa -= 4; + } + + for (p = (unsigned int *)pa; p < (unsigned int *)pb; p++) { + buff = *(p); + p_byte_read = ((char *)&buff); + + /* read leading word */ + if ((char *)p <= src) { + for (ps = src; ps < ((char *)p + 4); ps++) { + if (ps >= src + n) { + break; + } + p_byte_read = ((char *)&buff) + (ps - (char *)p); + *dest++ = *p_byte_read; + } + } + /* read trailing word */ + else if ((char *)p >= pb - 4) { + for (ps = (char *)p; ps < src + n; ps++) { + *dest++ = *p_byte_read++; + } + } + /* read remaining word(s) */ + else { + if ((char *)p + 4 >= src + n) { + for (ps = (char *)p; ps < src + n; ps++) { + *dest++ = *p_byte_read++; + } + } + else { + *(unsigned int *)dest = buff; + dest += 4; + } + } + } + + return 0; +} + +int +b_memcpy_s(void *s1, unsigned int s1max, const void *s2, unsigned int n) +{ + char *dest = (char *)s1; + char *src = (char *)s2; + if (n == 0) { + return 0; + } + + if (s1 == NULL) { + return -1; + } + if (s2 == NULL || n > s1max) { + memset(dest, 0, s1max); + return -1; + } + memcpy(dest, src, n); + return 0; +} + +int +b_memmove_s(void *s1, unsigned int s1max, const void *s2, unsigned int n) +{ + char *dest = (char *)s1; + char *src = (char *)s2; + if (n == 0) { + return 0; + } + + if (s1 == NULL) { + return -1; + } + if (s2 == NULL || n > s1max) { + memset(dest, 0, s1max); + return -1; + } + memmove(dest, src, n); + return 0; +} + +int +b_strcat_s(char *s1, unsigned int s1max, const char *s2) +{ + if (NULL == s1 || NULL == s2 || s1max < (strlen(s1) + strlen(s2) + 1)) { + return -1; + } + + memcpy(s1 + strlen(s1), s2, strlen(s2) + 1); + return 0; +} + +int +b_strcpy_s(char *s1, unsigned int s1max, const char *s2) +{ + if (NULL == s1 || NULL == s2 || s1max < (strlen(s2) + 1)) { + return -1; + } + + memcpy(s1, s2, strlen(s2) + 1); + return 0; +} + +char * +bh_strdup(const char *s) +{ + uint32 size; + char *s1 = NULL; + + if (s) { + size = (uint32)(strlen(s) + 1); + if ((s1 = BH_MALLOC(size))) + bh_memcpy_s(s1, size, s, size); + } + return s1; +} + +char * +wa_strdup(const char *s) +{ + uint32 size; + char *s1 = NULL; + + if (s) { + size = (uint32)(strlen(s) + 1); + if ((s1 = WA_MALLOC(size))) + bh_memcpy_s(s1, size, s, size); + } + return s1; +} + +#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 +int +bh_system(const char *cmd) +{ + int ret; + +#if !(defined(_WIN32) || defined(_WIN32_)) + ret = system(cmd); +#else + ret = _spawnlp(_P_WAIT, "cmd.exe", "/c", cmd, NULL); +#endif + + return ret; +} + +#if defined(_WIN32) || defined(_WIN32_) +errno_t +_mktemp_s(char *nameTemplate, size_t sizeInChars); +#endif + +bool +bh_mkstemp(char *file_name, size_t name_len) +{ + int fd; + +#if !(defined(_WIN32) || defined(_WIN32_)) + (void)name_len; + /* On Linux, it generates a unique temporary filename from template, creates + * and opens the file, and returns an open file descriptor for the file. */ + if ((fd = mkstemp(file_name)) <= 0) { + goto fail; + } + + /* close and remove temp file */ + close(fd); + unlink(file_name); +#else + /* On Windows, it generates a unique temporary file name but does not create + * or open the file */ + if (_mktemp_s(file_name, name_len) != 0) { + goto fail; + } +#endif + + return true; +fail: + return false; +} +#endif /* End of WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 */ diff --git a/wamr/src/bh_leb128.c b/wamr/src/bh_leb128.c new file mode 100644 index 0000000..aae738a --- /dev/null +++ b/wamr/src/bh_leb128.c @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_leb128.h" // bh_leb128.h + +bh_leb_read_status_t +bh_leb_read(const uint8 *buf, const uint8 *buf_end, uint32 maxbits, bool sign, + uint64 *p_result, size_t *p_offset) +{ + uint64 result = 0; + uint32 shift = 0; + uint32 offset = 0, bcnt = 0; + uint64 byte; + + while (true) { + /* uN or SN must not exceed ceil(N/7) bytes */ + if (bcnt + 1 > (maxbits + 6) / 7) { + return BH_LEB_READ_TOO_LONG; + } + + if ((uintptr_t)buf + offset + 1 < (uintptr_t)buf + || (uintptr_t)buf + offset + 1 > (uintptr_t)buf_end) { + return BH_LEB_READ_UNEXPECTED_END; + } + byte = buf[offset]; + offset += 1; + result |= ((byte & 0x7f) << shift); + shift += 7; + bcnt += 1; + if ((byte & 0x80) == 0) { + break; + } + } + + if (!sign && maxbits == 32 && shift >= maxbits) { + /* The top bits set represent values > 32 bits */ + if (((uint8)byte) & 0xf0) + return BH_LEB_READ_OVERFLOW; + } + else if (sign && maxbits == 32) { + if (shift < maxbits) { + /* Sign extend, second-highest bit is the sign bit */ + if ((uint8)byte & 0x40) + result |= (~((uint64)0)) << shift; + } + else { + /* The top bits should be a sign-extension of the sign bit */ + bool sign_bit_set = ((uint8)byte) & 0x8; + int top_bits = ((uint8)byte) & 0xf0; + if ((sign_bit_set && top_bits != 0x70) + || (!sign_bit_set && top_bits != 0)) + return BH_LEB_READ_OVERFLOW; + } + } + else if (sign && maxbits == 64) { + if (shift < maxbits) { + /* Sign extend, second-highest bit is the sign bit */ + if ((uint8)byte & 0x40) + result |= (~((uint64)0)) << shift; + } + else { + /* The top bits should be a sign-extension of the sign bit */ + bool sign_bit_set = ((uint8)byte) & 0x1; + int top_bits = ((uint8)byte) & 0xfe; + + if ((sign_bit_set && top_bits != 0x7e) + || (!sign_bit_set && top_bits != 0)) + return BH_LEB_READ_OVERFLOW; + } + } + + *p_offset = offset; + *p_result = result; + return BH_LEB_READ_SUCCESS; +} \ No newline at end of file diff --git a/wamr/src/bh_vector.c b/wamr/src/bh_vector.c new file mode 100644 index 0000000..86b3a29 --- /dev/null +++ b/wamr/src/bh_vector.c @@ -0,0 +1,279 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_vector.h" // bh_vector.h + +static uint8 * +alloc_vector_data(size_t length, size_t size_elem) +{ + uint64 total_size = ((uint64)size_elem) * length; + uint8 *data; + + if (length > UINT32_MAX || size_elem > UINT32_MAX + || total_size > UINT32_MAX) { + return NULL; + } + + if ((data = BH_MALLOC((uint32)total_size))) { + memset(data, 0, (uint32)total_size); + } + + return data; +} + +/** + * every caller of `extend_vector` must provide + * a thread-safe environment. + */ +static bool +extend_vector(Vector *vector, size_t length) +{ + uint8 *data; + + if (length <= vector->max_elems) + return true; + + if (length < vector->max_elems * 3 / 2) + length = vector->max_elems * 3 / 2; + + if (!(data = alloc_vector_data(length, vector->size_elem))) { + return false; + } + + bh_memcpy_s(data, (uint32)(vector->size_elem * length), vector->data, + (uint32)(vector->size_elem * vector->max_elems)); + BH_FREE(vector->data); + + vector->data = data; + vector->max_elems = length; + return true; +} + +bool +bh_vector_init(Vector *vector, size_t init_length, size_t size_elem, + bool use_lock) +{ + if (!vector) { + LOG_ERROR("Init vector failed: vector is NULL.\n"); + return false; + } + + if (init_length == 0) { + init_length = 4; + } + + if (!(vector->data = alloc_vector_data(init_length, size_elem))) { + LOG_ERROR("Init vector failed: alloc memory failed.\n"); + return false; + } + + vector->size_elem = size_elem; + vector->max_elems = init_length; + vector->num_elems = 0; + vector->lock = NULL; + + if (use_lock) { + if (!(vector->lock = BH_MALLOC(sizeof(korp_mutex)))) { + LOG_ERROR("Init vector failed: alloc locker failed.\n"); + bh_vector_destroy(vector); + return false; + } + + if (BHT_OK != os_mutex_init(vector->lock)) { + LOG_ERROR("Init vector failed: init locker failed.\n"); + + BH_FREE(vector->lock); + vector->lock = NULL; + + bh_vector_destroy(vector); + return false; + } + } + + return true; +} + +bool +bh_vector_set(Vector *vector, uint32 index, const void *elem_buf) +{ + if (!vector || !elem_buf) { + LOG_ERROR("Set vector elem failed: vector or elem buf is NULL.\n"); + return false; + } + + if (index >= vector->num_elems) { + LOG_ERROR("Set vector elem failed: invalid elem index.\n"); + return false; + } + + if (vector->lock) + os_mutex_lock(vector->lock); + bh_memcpy_s(vector->data + vector->size_elem * index, + (uint32)vector->size_elem, elem_buf, (uint32)vector->size_elem); + if (vector->lock) + os_mutex_unlock(vector->lock); + return true; +} + +bool +bh_vector_get(Vector *vector, uint32 index, void *elem_buf) +{ + if (!vector || !elem_buf) { + LOG_ERROR("Get vector elem failed: vector or elem buf is NULL.\n"); + return false; + } + + if (index >= vector->num_elems) { + LOG_ERROR("Get vector elem failed: invalid elem index.\n"); + return false; + } + + if (vector->lock) + os_mutex_lock(vector->lock); + bh_memcpy_s(elem_buf, (uint32)vector->size_elem, + vector->data + vector->size_elem * index, + (uint32)vector->size_elem); + if (vector->lock) + os_mutex_unlock(vector->lock); + return true; +} + +bool +bh_vector_insert(Vector *vector, uint32 index, const void *elem_buf) +{ + size_t i; + uint8 *p; + bool ret = false; + + if (!vector || !elem_buf) { + LOG_ERROR("Insert vector elem failed: vector or elem buf is NULL.\n"); + goto just_return; + } + + if (index >= vector->num_elems) { + LOG_ERROR("Insert vector elem failed: invalid elem index.\n"); + goto just_return; + } + + if (vector->lock) + os_mutex_lock(vector->lock); + + if (!extend_vector(vector, vector->num_elems + 1)) { + LOG_ERROR("Insert vector elem failed: extend vector failed.\n"); + goto unlock_return; + } + + p = vector->data + vector->size_elem * vector->num_elems; + for (i = vector->num_elems - 1; i > index; i--) { + bh_memcpy_s(p, (uint32)vector->size_elem, p - vector->size_elem, + (uint32)vector->size_elem); + p -= vector->size_elem; + } + + bh_memcpy_s(p, (uint32)vector->size_elem, elem_buf, + (uint32)vector->size_elem); + vector->num_elems++; + ret = true; + +unlock_return: + if (vector->lock) + os_mutex_unlock(vector->lock); +just_return: + return ret; +} + +bool +bh_vector_append(Vector *vector, const void *elem_buf) +{ + bool ret = false; + + if (!vector || !elem_buf) { + LOG_ERROR("Append vector elem failed: vector or elem buf is NULL.\n"); + goto just_return; + } + + /* make sure one more slot is used by the thread who allocates it */ + if (vector->lock) + os_mutex_lock(vector->lock); + + if (!extend_vector(vector, vector->num_elems + 1)) { + LOG_ERROR("Append vector elem failed: extend vector failed.\n"); + goto unlock_return; + } + + bh_memcpy_s(vector->data + vector->size_elem * vector->num_elems, + (uint32)vector->size_elem, elem_buf, (uint32)vector->size_elem); + vector->num_elems++; + ret = true; + +unlock_return: + if (vector->lock) + os_mutex_unlock(vector->lock); +just_return: + return ret; +} + +bool +bh_vector_remove(Vector *vector, uint32 index, void *old_elem_buf) +{ + uint32 i; + uint8 *p; + + if (!vector) { + LOG_ERROR("Remove vector elem failed: vector is NULL.\n"); + return false; + } + + if (index >= vector->num_elems) { + LOG_ERROR("Remove vector elem failed: invalid elem index.\n"); + return false; + } + + if (vector->lock) + os_mutex_lock(vector->lock); + p = vector->data + vector->size_elem * index; + + if (old_elem_buf) { + bh_memcpy_s(old_elem_buf, (uint32)vector->size_elem, p, + (uint32)vector->size_elem); + } + + for (i = index; i < vector->num_elems - 1; i++) { + bh_memcpy_s(p, (uint32)vector->size_elem, p + vector->size_elem, + (uint32)vector->size_elem); + p += vector->size_elem; + } + + vector->num_elems--; + if (vector->lock) + os_mutex_unlock(vector->lock); + return true; +} + +size_t +bh_vector_size(const Vector *vector) +{ + return vector ? vector->num_elems : 0; +} + +bool +bh_vector_destroy(Vector *vector) +{ + if (!vector) { + LOG_ERROR("Destroy vector elem failed: vector is NULL.\n"); + return false; + } + + if (vector->data) + BH_FREE(vector->data); + + if (vector->lock) { + os_mutex_destroy(vector->lock); + BH_FREE(vector->lock); + } + + memset(vector, 0, sizeof(Vector)); + return true; +} diff --git a/wamr/src/ems_alloc.c b/wamr/src/ems_alloc.c new file mode 100644 index 0000000..3142248 --- /dev/null +++ b/wamr/src/ems_alloc.c @@ -0,0 +1,1169 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "ems_gc_internal.h" // ems_gc_internal.h + +#if WASM_ENABLE_GC != 0 +#define LOCK_HEAP(heap) \ + do { \ + if (!heap->is_doing_reclaim) \ + /* If the heap is doing reclaim, it must have been locked, \ + we should not lock the heap again. */ \ + os_mutex_lock(&heap->lock); \ + } while (0) +#define UNLOCK_HEAP(heap) \ + do { \ + if (!heap->is_doing_reclaim) \ + /* If the heap is doing reclaim, it must have been locked, \ + and will be unlocked after reclaim, we should not \ + unlock the heap again. */ \ + os_mutex_unlock(&heap->lock); \ + } while (0) +#else +#define LOCK_HEAP(heap) os_mutex_lock(&heap->lock) +#define UNLOCK_HEAP(heap) os_mutex_unlock(&heap->lock) +#endif + +static inline bool +hmu_is_in_heap(void *hmu, gc_uint8 *heap_base_addr, gc_uint8 *heap_end_addr) +{ + gc_uint8 *addr = (gc_uint8 *)hmu; + return (addr >= heap_base_addr && addr < heap_end_addr) ? true : false; +} + +/** + * Remove a node from the tree it belongs to + * + * @param p the node to remove, can not be NULL, can not be the ROOT node + * the node will be removed from the tree, and the left, right and + * parent pointers of the node @p will be set to be NULL. Other fields + * won't be touched. The tree will be re-organized so that the order + * conditions are still satisfied. + */ +static bool +remove_tree_node(gc_heap_t *heap, hmu_tree_node_t *p) +{ + hmu_tree_node_t *q = NULL, **slot = NULL; +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + hmu_tree_node_t *root = heap->kfc_tree_root, *parent; + gc_uint8 *base_addr = heap->base_addr; + gc_uint8 *end_addr = base_addr + heap->current_size; +#endif + + bh_assert(p); + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + parent = p->parent; + if (!parent || p == root /* p can not be the ROOT node */ + || !hmu_is_in_heap(p, base_addr, end_addr) + || (parent != root && !hmu_is_in_heap(parent, base_addr, end_addr))) { + goto fail; + } +#endif + + /* get the slot which holds pointer to node p */ + if (p == p->parent->right) { + /* Don't use `slot = &p->parent->right` to avoid compiler warning */ + slot = (hmu_tree_node_t **)((uint8 *)p->parent + + offsetof(hmu_tree_node_t, right)); + } + else if (p == p->parent->left) { + /* p should be a child of its parent */ + /* Don't use `slot = &p->parent->left` to avoid compiler warning */ + slot = (hmu_tree_node_t **)((uint8 *)p->parent + + offsetof(hmu_tree_node_t, left)); + } + else { + goto fail; + } + + /** + * algorithms used to remove node p + * case 1: if p has no left child, replace p with its right child + * case 2: if p has no right child, replace p with its left child + * case 3: otherwise, find p's predecessor, remove it from the tree + * and replace p with it. + * use predecessor can keep the left <= root < right condition. + */ + + if (!p->left) { + /* move right child up*/ + *slot = p->right; + if (p->right) { +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(p->right, base_addr, end_addr)) { + goto fail; + } +#endif + p->right->parent = p->parent; + } + + p->left = p->right = p->parent = NULL; + return true; + } + + if (!p->right) { + /* move left child up*/ + *slot = p->left; +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(p->left, base_addr, end_addr)) { + goto fail; + } +#endif + /* p->left can never be NULL unless it is corrupted. */ + p->left->parent = p->parent; + + p->left = p->right = p->parent = NULL; + return true; + } + + /* both left & right exist, find p's predecessor at first*/ + q = p->left; +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(q, base_addr, end_addr)) { + goto fail; + } +#endif + while (q->right) { + q = q->right; +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(q, base_addr, end_addr)) { + goto fail; + } +#endif + } + + /* remove from the tree*/ + if (!remove_tree_node(heap, q)) + return false; + + *slot = q; + q->parent = p->parent; + q->left = p->left; + q->right = p->right; + if (q->left) { +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(q->left, base_addr, end_addr)) { + goto fail; + } +#endif + q->left->parent = q; + } + if (q->right) { +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(q->right, base_addr, end_addr)) { + goto fail; + } +#endif + q->right->parent = q; + } + + p->left = p->right = p->parent = NULL; + + return true; +fail: +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + heap->is_heap_corrupted = true; +#endif + return false; +} + +static bool +unlink_hmu(gc_heap_t *heap, hmu_t *hmu) +{ +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + gc_uint8 *base_addr, *end_addr; +#endif + gc_size_t size; + + bh_assert(gci_is_heap_valid(heap)); + bh_assert(hmu && (gc_uint8 *)hmu >= heap->base_addr + && (gc_uint8 *)hmu < heap->base_addr + heap->current_size); + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (hmu_get_ut(hmu) != HMU_FC) { + heap->is_heap_corrupted = true; + return false; + } +#endif + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; +#endif + size = hmu_get_size(hmu); + + if (HMU_IS_FC_NORMAL(size)) { + uint32 node_idx = size >> 3; + hmu_normal_node_t *node_prev = NULL, *node_next; + hmu_normal_node_t *node = heap->kfc_normal_list[node_idx].next; + + while (node) { +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(node, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return false; + } +#endif + node_next = get_hmu_normal_node_next(node); + if ((hmu_t *)node == hmu) { + if (!node_prev) /* list head */ + heap->kfc_normal_list[node_idx].next = node_next; + else + set_hmu_normal_node_next(node_prev, node_next); + break; + } + node_prev = node; + node = node_next; + } + + if (!node) { + LOG_ERROR("[GC_ERROR]couldn't find the node in the normal list\n"); + } + } + else { + if (!remove_tree_node(heap, (hmu_tree_node_t *)hmu)) + return false; + } + return true; +} + +static void +hmu_set_free_size(hmu_t *hmu) +{ + gc_size_t size; + bh_assert(hmu && hmu_get_ut(hmu) == HMU_FC); + + size = hmu_get_size(hmu); + *((uint32 *)((char *)hmu + size) - 1) = size; +} + +/** + * Add free chunk back to KFC + * + * @param heap should not be NULL and it should be a valid heap + * @param hmu should not be NULL and it should be a HMU of length @size inside + * @heap hmu should be 8-bytes aligned + * @param size should be positive and multiple of 8 + * hmu with size @size will be added into KFC as a new FC. + */ +bool +gci_add_fc(gc_heap_t *heap, hmu_t *hmu, gc_size_t size) +{ +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + gc_uint8 *base_addr, *end_addr; +#endif + hmu_normal_node_t *np = NULL; + hmu_tree_node_t *root = NULL, *tp = NULL, *node = NULL; + uint32 node_idx; + + bh_assert(gci_is_heap_valid(heap)); + bh_assert(hmu && (gc_uint8 *)hmu >= heap->base_addr + && (gc_uint8 *)hmu < heap->base_addr + heap->current_size); + bh_assert(((gc_uint32)(uintptr_t)hmu_to_obj(hmu) & 7) == 0); + bh_assert(size > 0 + && ((gc_uint8 *)hmu) + size + <= heap->base_addr + heap->current_size); + bh_assert(!(size & 7)); + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; +#endif + + hmu_set_ut(hmu, HMU_FC); + hmu_set_size(hmu, size); + hmu_set_free_size(hmu); + + if (HMU_IS_FC_NORMAL(size)) { + np = (hmu_normal_node_t *)hmu; +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(np, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return false; + } +#endif + + node_idx = size >> 3; + set_hmu_normal_node_next(np, heap->kfc_normal_list[node_idx].next); + heap->kfc_normal_list[node_idx].next = np; + return true; + } + + /* big block */ + node = (hmu_tree_node_t *)hmu; + node->size = size; + node->left = node->right = node->parent = NULL; + + /* find proper node to link this new node to */ + root = heap->kfc_tree_root; + tp = root; + bh_assert(tp->size < size); + while (1) { + if (tp->size < size) { + if (!tp->right) { + tp->right = node; + node->parent = tp; + break; + } + tp = tp->right; + } + else { /* tp->size >= size */ + if (!tp->left) { + tp->left = node; + node->parent = tp; + break; + } + tp = tp->left; + } +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(tp, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return false; + } +#endif + } + return true; +} + +/** + * Find a proper hmu for required memory size + * + * @param heap should not be NULL and should be a valid heap + * @param size should cover the header and should be 8 bytes aligned + * GC will not be performed here. + * Heap extension will not be performed here. + * + * @return hmu allocated if success, which will be aligned to 8 bytes, + * NULL otherwise + */ +static hmu_t * +alloc_hmu(gc_heap_t *heap, gc_size_t size) +{ + gc_uint8 *base_addr, *end_addr; + hmu_normal_list_t *normal_head = NULL; + hmu_normal_node_t *p = NULL; + uint32 node_idx = 0, init_node_idx = 0; + hmu_tree_node_t *root = NULL, *tp = NULL, *last_tp = NULL; + hmu_t *next, *rest; + uintptr_t tp_ret; + + bh_assert(gci_is_heap_valid(heap)); + bh_assert(size > 0 && !(size & 7)); + +#if WASM_ENABLE_GC != 0 + /* In doing reclaim, gc must not alloc memory again. */ + bh_assert(!heap->is_doing_reclaim); +#endif + + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; + + if (size < GC_SMALLEST_SIZE) + size = GC_SMALLEST_SIZE; + + /* check normal list at first*/ + if (HMU_IS_FC_NORMAL(size)) { + /* find a non-empty slot in normal_node_list with good size*/ + init_node_idx = (size >> 3); + for (node_idx = init_node_idx; node_idx < HMU_NORMAL_NODE_CNT; + node_idx++) { + normal_head = heap->kfc_normal_list + node_idx; + if (normal_head->next) + break; + normal_head = NULL; + } + + /* found in normal list*/ + if (normal_head) { + bh_assert(node_idx >= init_node_idx); + + p = normal_head->next; +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(p, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return NULL; + } +#endif + normal_head->next = get_hmu_normal_node_next(p); +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (((gc_int32)(uintptr_t)hmu_to_obj(p) & 7) != 0) { + heap->is_heap_corrupted = true; + return NULL; + } +#endif + + if ((gc_size_t)node_idx != (uint32)init_node_idx + /* with bigger size*/ + && ((gc_size_t)node_idx << 3) >= size + GC_SMALLEST_SIZE) { + rest = (hmu_t *)(((char *)p) + size); + if (!gci_add_fc(heap, rest, (node_idx << 3) - size)) { + return NULL; + } + hmu_mark_pinuse(rest); + } + else { + size = node_idx << 3; + next = (hmu_t *)((char *)p + size); + if (hmu_is_in_heap(next, base_addr, end_addr)) + hmu_mark_pinuse(next); + } + + heap->total_free_size -= size; + if ((heap->current_size - heap->total_free_size) + > heap->highmark_size) + heap->highmark_size = + heap->current_size - heap->total_free_size; + + hmu_set_size((hmu_t *)p, size); + return (hmu_t *)p; + } + } + + /* need to find a node in tree*/ + root = heap->kfc_tree_root; + + /* find the best node*/ + bh_assert(root); + tp = root->right; + while (tp) { +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (!hmu_is_in_heap(tp, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return NULL; + } +#endif + + if (tp->size < size) { + tp = tp->right; + continue; + } + + /* record the last node with size equal to or bigger than given size*/ + last_tp = tp; + tp = tp->left; + } + + if (last_tp) { + bh_assert(last_tp->size >= size); + + /* alloc in last_p*/ + + /* remove node last_p from tree*/ + if (!remove_tree_node(heap, last_tp)) + return NULL; + + if (last_tp->size >= size + GC_SMALLEST_SIZE) { + rest = (hmu_t *)((char *)last_tp + size); + if (!gci_add_fc(heap, rest, last_tp->size - size)) + return NULL; + hmu_mark_pinuse(rest); + } + else { + size = last_tp->size; + next = (hmu_t *)((char *)last_tp + size); + if (hmu_is_in_heap(next, base_addr, end_addr)) + hmu_mark_pinuse(next); + } + + heap->total_free_size -= size; + if ((heap->current_size - heap->total_free_size) > heap->highmark_size) + heap->highmark_size = heap->current_size - heap->total_free_size; + + hmu_set_size((hmu_t *)last_tp, size); + tp_ret = (uintptr_t)last_tp; + return (hmu_t *)tp_ret; + } + + return NULL; +} + +#if WASM_ENABLE_GC != 0 +static int +do_gc_heap(gc_heap_t *heap) +{ + int ret = GC_SUCCESS; +#if WASM_ENABLE_GC_PERF_PROFILING != 0 + uint64 start = 0, end = 0, time = 0; + + start = os_time_get_boot_microsecond(); +#endif + if (heap->is_reclaim_enabled) { + UNLOCK_HEAP(heap); + ret = gci_gc_heap(heap); + LOCK_HEAP(heap); + } +#if WASM_ENABLE_GC_PERF_PROFILING != 0 + end = os_time_get_boot_microsecond(); + time = end - start; + heap->total_gc_time += time; + if (time > heap->max_gc_time) { + heap->max_gc_time = time; + } + heap->total_gc_count += 1; +#endif + return ret; +} +#endif + +/** + * Find a proper HMU with given size + * + * @param heap should not be NULL and should be a valid heap + * @param size should cover the header and should be 8 bytes aligned + * + * Note: This function will try several ways to satisfy the allocation request: + * 1. Find a proper on available HMUs. + * 2. GC will be triggered if 1 failed. + * 3. Find a proper on available HMUS. + * 4. Return NULL if 3 failed + * + * @return hmu allocated if success, which will be aligned to 8 bytes, + * NULL otherwise + */ +static hmu_t * +alloc_hmu_ex(gc_heap_t *heap, gc_size_t size) +{ + bh_assert(gci_is_heap_valid(heap)); + bh_assert(size > 0 && !(size & 7)); + +#if WASM_ENABLE_GC != 0 +#if GC_IN_EVERY_ALLOCATION != 0 + if (GC_SUCCESS != do_gc_heap(heap)) + return NULL; +#else + if (heap->total_free_size < heap->gc_threshold) { + if (GC_SUCCESS != do_gc_heap(heap)) + return NULL; + } + else { + hmu_t *ret = NULL; + if ((ret = alloc_hmu(heap, size))) { + return ret; + } + if (GC_SUCCESS != do_gc_heap(heap)) + return NULL; + } +#endif +#endif + + return alloc_hmu(heap, size); +} + +#if BH_ENABLE_GC_VERIFY == 0 +gc_object_t +gc_alloc_vo(void *vheap, gc_size_t size) +#else +gc_object_t +gc_alloc_vo_internal(void *vheap, gc_size_t size, const char *file, int line) +#endif +{ + gc_heap_t *heap = (gc_heap_t *)vheap; + hmu_t *hmu = NULL; + gc_object_t ret = (gc_object_t)NULL; + gc_size_t tot_size = 0, tot_size_unaligned; + + /* hmu header + prefix + obj + suffix */ + tot_size_unaligned = HMU_SIZE + OBJ_PREFIX_SIZE + size + OBJ_SUFFIX_SIZE; + /* aligned size*/ + tot_size = GC_ALIGN_8(tot_size_unaligned); + if (tot_size < size) + /* integer overflow */ + return NULL; + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (heap->is_heap_corrupted) { + LOG_ERROR("[GC_ERROR]Heap is corrupted, allocate memory failed.\n"); + return NULL; + } +#endif + + LOCK_HEAP(heap); + + hmu = alloc_hmu_ex(heap, tot_size); + if (!hmu) + goto finish; + + bh_assert(hmu_get_size(hmu) >= tot_size); + /* the total size allocated may be larger than + the required size, reset it here */ + tot_size = hmu_get_size(hmu); + +#if GC_STAT_DATA != 0 + heap->total_size_allocated += tot_size; +#endif + + hmu_set_ut(hmu, HMU_VO); + hmu_unfree_vo(hmu); + +#if BH_ENABLE_GC_VERIFY != 0 + hmu_init_prefix_and_suffix(hmu, tot_size, file, line); +#endif + + ret = hmu_to_obj(hmu); + if (tot_size > tot_size_unaligned) + /* clear buffer appended by GC_ALIGN_8() */ + memset((uint8 *)ret + size, 0, tot_size - tot_size_unaligned); + +finish: + UNLOCK_HEAP(heap); + return ret; +} + +#if BH_ENABLE_GC_VERIFY == 0 +gc_object_t +gc_realloc_vo(void *vheap, void *ptr, gc_size_t size) +#else +gc_object_t +gc_realloc_vo_internal(void *vheap, void *ptr, gc_size_t size, const char *file, + int line) +#endif +{ + gc_heap_t *heap = (gc_heap_t *)vheap; + hmu_t *hmu = NULL, *hmu_old = NULL, *hmu_next; + gc_object_t ret = (gc_object_t)NULL, obj_old = (gc_object_t)ptr; + gc_size_t tot_size, tot_size_unaligned, tot_size_old = 0, tot_size_next; + gc_size_t obj_size, obj_size_old; + gc_uint8 *base_addr, *end_addr; + hmu_type_t ut; + + /* hmu header + prefix + obj + suffix */ + tot_size_unaligned = HMU_SIZE + OBJ_PREFIX_SIZE + size + OBJ_SUFFIX_SIZE; + /* aligned size*/ + tot_size = GC_ALIGN_8(tot_size_unaligned); + if (tot_size < size) + /* integer overflow */ + return NULL; + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (heap->is_heap_corrupted) { + LOG_ERROR("[GC_ERROR]Heap is corrupted, allocate memory failed.\n"); + return NULL; + } +#endif + + if (obj_old) { + hmu_old = obj_to_hmu(obj_old); + tot_size_old = hmu_get_size(hmu_old); + if (tot_size <= tot_size_old) + /* current node already meets requirement */ + return obj_old; + } + + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; + + LOCK_HEAP(heap); + + if (hmu_old) { + hmu_next = (hmu_t *)((char *)hmu_old + tot_size_old); + if (hmu_is_in_heap(hmu_next, base_addr, end_addr)) { + ut = hmu_get_ut(hmu_next); + tot_size_next = hmu_get_size(hmu_next); + if (ut == HMU_FC && tot_size <= tot_size_old + tot_size_next) { + /* current node and next node meets requirement */ + if (!unlink_hmu(heap, hmu_next)) { + UNLOCK_HEAP(heap); + return NULL; + } + hmu_set_size(hmu_old, tot_size); + memset((char *)hmu_old + tot_size_old, 0, + tot_size - tot_size_old); +#if BH_ENABLE_GC_VERIFY != 0 + hmu_init_prefix_and_suffix(hmu_old, tot_size, file, line); +#endif + if (tot_size < tot_size_old + tot_size_next) { + hmu_next = (hmu_t *)((char *)hmu_old + tot_size); + tot_size_next = tot_size_old + tot_size_next - tot_size; + if (!gci_add_fc(heap, hmu_next, tot_size_next)) { + UNLOCK_HEAP(heap); + return NULL; + } + hmu_mark_pinuse(hmu_next); + } + UNLOCK_HEAP(heap); + return obj_old; + } + } + } + + hmu = alloc_hmu_ex(heap, tot_size); + if (!hmu) + goto finish; + + bh_assert(hmu_get_size(hmu) >= tot_size); + /* the total size allocated may be larger than + the required size, reset it here */ + tot_size = hmu_get_size(hmu); + +#if GC_STAT_DATA != 0 + heap->total_size_allocated += tot_size; +#endif + + hmu_set_ut(hmu, HMU_VO); + hmu_unfree_vo(hmu); + +#if BH_ENABLE_GC_VERIFY != 0 + hmu_init_prefix_and_suffix(hmu, tot_size, file, line); +#endif + + ret = hmu_to_obj(hmu); + +finish: + + if (ret) { + obj_size = tot_size - HMU_SIZE - OBJ_PREFIX_SIZE - OBJ_SUFFIX_SIZE; + memset(ret, 0, obj_size); + if (obj_old) { + obj_size_old = + tot_size_old - HMU_SIZE - OBJ_PREFIX_SIZE - OBJ_SUFFIX_SIZE; + bh_memcpy_s(ret, obj_size, obj_old, obj_size_old); + } + } + + UNLOCK_HEAP(heap); + + if (ret && obj_old) + gc_free_vo(vheap, obj_old); + + return ret; +} + +#if GC_MANUALLY != 0 +void +gc_free_wo(void *vheap, void *ptr) +{ + gc_heap_t *heap = (gc_heap_t *)vheap; + gc_object_t *obj = (gc_object_t *)ptr; + hmu_t *hmu = obj_to_hmu(obj); + + bh_assert(gci_is_heap_valid(heap)); + bh_assert(obj); + bh_assert((gc_uint8 *)hmu >= heap->base_addr + && (gc_uint8 *)hmu < heap->base_addr + heap->current_size); + bh_assert(hmu_get_ut(hmu) == HMU_WO); + + hmu_unmark_wo(hmu); + (void)heap; +} +#endif + +/* see ems_gc.h for description*/ +#if BH_ENABLE_GC_VERIFY == 0 +gc_object_t +gc_alloc_wo(void *vheap, gc_size_t size) +#else +gc_object_t +gc_alloc_wo_internal(void *vheap, gc_size_t size, const char *file, int line) +#endif +{ + gc_heap_t *heap = (gc_heap_t *)vheap; + hmu_t *hmu = NULL; + gc_object_t ret = (gc_object_t)NULL; + gc_size_t tot_size = 0, tot_size_unaligned; + + /* hmu header + prefix + obj + suffix */ + tot_size_unaligned = HMU_SIZE + OBJ_PREFIX_SIZE + size + OBJ_SUFFIX_SIZE; + /* aligned size*/ + tot_size = GC_ALIGN_8(tot_size_unaligned); + if (tot_size < size) + /* integer overflow */ + return NULL; + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (heap->is_heap_corrupted) { + os_printf("[GC_ERROR]Heap is corrupted, allocate memory failed.\n"); + return NULL; + } +#endif + + LOCK_HEAP(heap); + + hmu = alloc_hmu_ex(heap, tot_size); + if (!hmu) + goto finish; + + /* Don't memset the memory to improve performance, the caller should + decide whether to memset it or not */ + + bh_assert(hmu_get_size(hmu) >= tot_size); + /* the total size allocated may be larger than + the required size, reset it here */ + tot_size = hmu_get_size(hmu); + +#if GC_STAT_DATA != 0 + heap->total_size_allocated += tot_size; +#endif + + hmu_set_ut(hmu, HMU_WO); +#if GC_MANUALLY != 0 + hmu_mark_wo(hmu); +#else + hmu_unmark_wo(hmu); +#endif + +#if BH_ENABLE_GC_VERIFY != 0 + hmu_init_prefix_and_suffix(hmu, tot_size, file, line); +#endif + + ret = hmu_to_obj(hmu); + if (tot_size > tot_size_unaligned) + /* clear buffer appended by GC_ALIGN_8() */ + memset((uint8 *)ret + size, 0, tot_size - tot_size_unaligned); + +finish: + UNLOCK_HEAP(heap); + return ret; +} + +/** + * Do some checking to see if given pointer is a possible valid heap + * @return GC_TRUE if all checking passed, GC_FALSE otherwise + */ +int +gci_is_heap_valid(gc_heap_t *heap) +{ + if (!heap) + return GC_FALSE; + if (heap->heap_id != (gc_handle_t)heap) + return GC_FALSE; + + return GC_TRUE; +} + +#if BH_ENABLE_GC_VERIFY == 0 +int +gc_free_vo(void *vheap, gc_object_t obj) +#else +int +gc_free_vo_internal(void *vheap, gc_object_t obj, const char *file, int line) +#endif +{ + gc_heap_t *heap = (gc_heap_t *)vheap; + gc_uint8 *base_addr, *end_addr; + hmu_t *hmu = NULL; + hmu_t *prev = NULL; + hmu_t *next = NULL; + gc_size_t size = 0; + hmu_type_t ut; + int ret = GC_SUCCESS; + + if (!obj) { + return GC_SUCCESS; + } + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (heap->is_heap_corrupted) { + LOG_ERROR("[GC_ERROR]Heap is corrupted, free memory failed.\n"); + return GC_ERROR; + } +#endif + + hmu = obj_to_hmu(obj); + + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; + + LOCK_HEAP(heap); + + if (hmu_is_in_heap(hmu, base_addr, end_addr)) { +#if BH_ENABLE_GC_VERIFY != 0 + hmu_verify(heap, hmu); +#endif + ut = hmu_get_ut(hmu); + if (ut == HMU_VO) { + if (hmu_is_vo_freed(hmu)) { + bh_assert(0); + ret = GC_ERROR; + goto out; + } + + size = hmu_get_size(hmu); + + heap->total_free_size += size; + +#if GC_STAT_DATA != 0 + heap->total_size_freed += size; +#endif + + if (!hmu_get_pinuse(hmu)) { + prev = (hmu_t *)((char *)hmu - *((int *)hmu - 1)); + + if (hmu_is_in_heap(prev, base_addr, end_addr) + && hmu_get_ut(prev) == HMU_FC) { + size += hmu_get_size(prev); + hmu = prev; + if (!unlink_hmu(heap, prev)) { + ret = GC_ERROR; + goto out; + } + } + } + + next = (hmu_t *)((char *)hmu + size); + if (hmu_is_in_heap(next, base_addr, end_addr)) { + if (hmu_get_ut(next) == HMU_FC) { + size += hmu_get_size(next); + if (!unlink_hmu(heap, next)) { + ret = GC_ERROR; + goto out; + } + next = (hmu_t *)((char *)hmu + size); + } + } + + if (!gci_add_fc(heap, hmu, size)) { + ret = GC_ERROR; + goto out; + } + + if (hmu_is_in_heap(next, base_addr, end_addr)) { + hmu_unmark_pinuse(next); + } + } + else { + ret = GC_ERROR; + goto out; + } + ret = GC_SUCCESS; + goto out; + } + +out: + UNLOCK_HEAP(heap); + return ret; +} + +void +gc_dump_heap_stats(gc_heap_t *heap) +{ + os_printf("heap: %p, heap start: %p\n", heap, heap->base_addr); + os_printf("total free: %" PRIu32 ", current: %" PRIu32 + ", highmark: %" PRIu32 "\n", + heap->total_free_size, heap->current_size, heap->highmark_size); +#if GC_STAT_DATA != 0 + os_printf("total size allocated: %" PRIu64 ", total size freed: %" PRIu64 + ", total occupied: %" PRIu64 "\n", + heap->total_size_allocated, heap->total_size_freed, + heap->total_size_allocated - heap->total_size_freed); +#endif +} + +uint32 +gc_get_heap_highmark_size(gc_heap_t *heap) +{ + return heap->highmark_size; +} + +void +gci_dump(gc_heap_t *heap) +{ + hmu_t *cur = NULL, *end = NULL; + hmu_type_t ut; + gc_size_t size; + int i = 0, p, mark; + char inuse = 'U'; + + cur = (hmu_t *)heap->base_addr; + end = (hmu_t *)((char *)heap->base_addr + heap->current_size); + + while (cur < end) { + ut = hmu_get_ut(cur); + size = hmu_get_size(cur); + p = hmu_get_pinuse(cur); + mark = hmu_is_wo_marked(cur); + + if (ut == HMU_VO) + inuse = 'V'; + else if (ut == HMU_WO) + inuse = hmu_is_wo_marked(cur) ? 'W' : 'w'; + else if (ut == HMU_FC) + inuse = 'F'; + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (size == 0 || size > (uint32)((uint8 *)end - (uint8 *)cur)) { + LOG_ERROR("[GC_ERROR]Heap is corrupted, heap dump failed.\n"); + heap->is_heap_corrupted = true; + return; + } +#endif + + os_printf("#%d %08" PRIx32 " %" PRIx32 " %d %d" + " %c %" PRId32 "\n", + i, (uint32)((char *)cur - (char *)heap->base_addr), + (uint32)ut, p, mark, inuse, (int32)hmu_obj_size(size)); +#if BH_ENABLE_GC_VERIFY != 0 + if (inuse == 'V') { + gc_object_prefix_t *prefix = (gc_object_prefix_t *)(cur + 1); + os_printf("#%s:%d\n", prefix->file_name, prefix->line_no); + } +#endif + + cur = (hmu_t *)((char *)cur + size); + i++; + } + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (cur != end) { + LOG_ERROR("[GC_ERROR]Heap is corrupted, heap dump failed.\n"); + heap->is_heap_corrupted = true; + } +#else + bh_assert(cur == end); +#endif +} + +#if WASM_ENABLE_GC != 0 +extra_info_node_t * +gc_search_extra_info_node(gc_handle_t handle, gc_object_t obj, + gc_size_t *p_index) +{ + gc_heap_t *vheap = (gc_heap_t *)handle; + int32 low = 0, high = vheap->extra_info_node_cnt - 1; + int32 mid; + extra_info_node_t *node; + + if (!vheap->extra_info_nodes) + return NULL; + + while (low <= high) { + mid = (low + high) / 2; + node = vheap->extra_info_nodes[mid]; + + if (obj == node->obj) { + if (p_index) { + *p_index = mid; + } + return node; + } + else if (obj < node->obj) { + high = mid - 1; + } + else { + low = mid + 1; + } + } + + if (p_index) { + *p_index = low; + } + return NULL; +} + +static bool +insert_extra_info_node(gc_heap_t *vheap, extra_info_node_t *node) +{ + gc_size_t index; + extra_info_node_t *orig_node; + + if (!vheap->extra_info_nodes) { + vheap->extra_info_nodes = vheap->extra_info_normal_nodes; + vheap->extra_info_node_capacity = sizeof(vheap->extra_info_normal_nodes) + / sizeof(extra_info_node_t *); + vheap->extra_info_nodes[0] = node; + vheap->extra_info_node_cnt = 1; + return true; + } + + /* extend array */ + if (vheap->extra_info_node_cnt == vheap->extra_info_node_capacity) { + extra_info_node_t **new_nodes = NULL; + gc_size_t new_capacity = vheap->extra_info_node_capacity * 3 / 2; + gc_size_t total_size = sizeof(extra_info_node_t *) * new_capacity; + + new_nodes = (extra_info_node_t **)BH_MALLOC(total_size); + if (!new_nodes) { + LOG_ERROR("alloc extra info nodes failed"); + return false; + } + + bh_memcpy_s(new_nodes, total_size, vheap->extra_info_nodes, + sizeof(extra_info_node_t *) * vheap->extra_info_node_cnt); + if (vheap->extra_info_nodes != vheap->extra_info_normal_nodes) { + BH_FREE(vheap->extra_info_nodes); + } + + vheap->extra_info_nodes = new_nodes; + vheap->extra_info_node_capacity = new_capacity; + } + + orig_node = gc_search_extra_info_node(vheap, node->obj, &index); + if (orig_node) { + /* replace the old node */ + vheap->extra_info_nodes[index] = node; + BH_FREE(orig_node); + } + else { + bh_memmove_s(vheap->extra_info_nodes + index + 1, + (vheap->extra_info_node_capacity - index - 1) + * sizeof(extra_info_node_t *), + vheap->extra_info_nodes + index, + (vheap->extra_info_node_cnt - index) + * sizeof(extra_info_node_t *)); + vheap->extra_info_nodes[index] = node; + vheap->extra_info_node_cnt += 1; + } + + return true; +} + +bool +gc_set_finalizer(gc_handle_t handle, gc_object_t obj, gc_finalizer_t cb, + void *data) +{ + extra_info_node_t *node = NULL; + gc_heap_t *vheap = (gc_heap_t *)handle; + + node = (extra_info_node_t *)BH_MALLOC(sizeof(extra_info_node_t)); + + if (!node) { + LOG_ERROR("alloc a new extra info node failed"); + return GC_FALSE; + } + memset(node, 0, sizeof(extra_info_node_t)); + + node->finalizer = cb; + node->obj = obj; + node->data = data; + + LOCK_HEAP(vheap); + if (!insert_extra_info_node(vheap, node)) { + BH_FREE(node); + UNLOCK_HEAP(vheap); + return GC_FALSE; + } + UNLOCK_HEAP(vheap); + + gct_vm_set_extra_info_flag(obj, true); + return GC_TRUE; +} + +void +gc_unset_finalizer(gc_handle_t handle, gc_object_t obj) +{ + gc_size_t index; + gc_heap_t *vheap = (gc_heap_t *)handle; + extra_info_node_t *node; + + LOCK_HEAP(vheap); + node = gc_search_extra_info_node(vheap, obj, &index); + + if (!node) { + UNLOCK_HEAP(vheap); + return; + } + + BH_FREE(node); + bh_memmove_s( + vheap->extra_info_nodes + index, + (vheap->extra_info_node_capacity - index) * sizeof(extra_info_node_t *), + vheap->extra_info_nodes + index + 1, + (vheap->extra_info_node_cnt - index - 1) * sizeof(extra_info_node_t *)); + vheap->extra_info_node_cnt -= 1; + UNLOCK_HEAP(vheap); + + gct_vm_set_extra_info_flag(obj, false); +} +#endif diff --git a/wamr/src/ems_kfc.c b/wamr/src/ems_kfc.c new file mode 100644 index 0000000..f3165fb --- /dev/null +++ b/wamr/src/ems_kfc.c @@ -0,0 +1,547 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "ems_gc_internal.h" // ems_gc_internal.h + +static gc_handle_t +gc_init_internal(gc_heap_t *heap, char *base_addr, gc_size_t heap_max_size) +{ + hmu_tree_node_t *root = NULL, *q = NULL; + int ret; + + memset(heap, 0, sizeof *heap); + memset(base_addr, 0, heap_max_size); + + ret = os_mutex_init(&heap->lock); + if (ret != BHT_OK) { + LOG_ERROR("[GC_ERROR]failed to init lock\n"); + return NULL; + } + + /* init all data structures*/ + heap->current_size = heap_max_size; + heap->base_addr = (gc_uint8 *)base_addr; + heap->heap_id = (gc_handle_t)heap; + + heap->total_free_size = heap->current_size; + heap->highmark_size = 0; +#if WASM_ENABLE_GC != 0 + heap->gc_threshold_factor = GC_DEFAULT_THRESHOLD_FACTOR; + gc_update_threshold(heap); +#endif + + root = heap->kfc_tree_root = (hmu_tree_node_t *)heap->kfc_tree_root_buf; + memset(root, 0, sizeof *root); + root->size = sizeof *root; + hmu_set_ut(&root->hmu_header, HMU_FC); + hmu_set_size(&root->hmu_header, sizeof *root); + + q = (hmu_tree_node_t *)heap->base_addr; + memset(q, 0, sizeof *q); + hmu_set_ut(&q->hmu_header, HMU_FC); + hmu_set_size(&q->hmu_header, heap->current_size); + + ASSERT_TREE_NODE_ALIGNED_ACCESS(q); + ASSERT_TREE_NODE_ALIGNED_ACCESS(root); + + hmu_mark_pinuse(&q->hmu_header); + root->right = q; + q->parent = root; + q->size = heap->current_size; + + bh_assert(root->size <= HMU_FC_NORMAL_MAX_SIZE); + + return heap; +} + +gc_handle_t +gc_init_with_pool(char *buf, gc_size_t buf_size) +{ + char *buf_end = buf + buf_size; + char *buf_aligned = (char *)(((uintptr_t)buf + 7) & (uintptr_t)~7); + char *base_addr = buf_aligned + sizeof(gc_heap_t); + gc_heap_t *heap = (gc_heap_t *)buf_aligned; + gc_size_t heap_max_size; + + if (buf_size < APP_HEAP_SIZE_MIN) { + LOG_ERROR("[GC_ERROR]heap init buf size (%" PRIu32 ") < %" PRIu32 "\n", + buf_size, (uint32)APP_HEAP_SIZE_MIN); + return NULL; + } + + base_addr = + (char *)(((uintptr_t)base_addr + 7) & (uintptr_t)~7) + GC_HEAD_PADDING; + heap_max_size = (uint32)(buf_end - base_addr) & (uint32)~7; + +#if WASM_ENABLE_MEMORY_TRACING != 0 + os_printf("Heap created, total size: %u\n", buf_size); + os_printf(" heap struct size: %u\n", sizeof(gc_heap_t)); + os_printf(" actual heap size: %u\n", heap_max_size); + os_printf(" padding bytes: %u\n", + buf_size - sizeof(gc_heap_t) - heap_max_size); +#endif + return gc_init_internal(heap, base_addr, heap_max_size); +} + +gc_handle_t +gc_init_with_struct_and_pool(char *struct_buf, gc_size_t struct_buf_size, + char *pool_buf, gc_size_t pool_buf_size) +{ + gc_heap_t *heap = (gc_heap_t *)struct_buf; + char *base_addr = pool_buf + GC_HEAD_PADDING; + char *pool_buf_end = pool_buf + pool_buf_size; + gc_size_t heap_max_size; + + if ((((uintptr_t)struct_buf) & 7) != 0) { + LOG_ERROR("[GC_ERROR]heap init struct buf not 8-byte aligned\n"); + return NULL; + } + + if (struct_buf_size < sizeof(gc_handle_t)) { + LOG_ERROR("[GC_ERROR]heap init struct buf size (%" PRIu32 ") < %zu\n", + struct_buf_size, sizeof(gc_handle_t)); + return NULL; + } + + if ((((uintptr_t)pool_buf) & 7) != 0) { + LOG_ERROR("[GC_ERROR]heap init pool buf not 8-byte aligned\n"); + return NULL; + } + + if (pool_buf_size < APP_HEAP_SIZE_MIN) { + LOG_ERROR("[GC_ERROR]heap init buf size (%" PRIu32 ") < %u\n", + pool_buf_size, APP_HEAP_SIZE_MIN); + return NULL; + } + + heap_max_size = (uint32)(pool_buf_end - base_addr) & (uint32)~7; + +#if WASM_ENABLE_MEMORY_TRACING != 0 + os_printf("Heap created, total size: %u\n", + struct_buf_size + pool_buf_size); + os_printf(" heap struct size: %u\n", sizeof(gc_heap_t)); + os_printf(" actual heap size: %u\n", heap_max_size); + os_printf(" padding bytes: %u\n", pool_buf_size - heap_max_size); +#endif + return gc_init_internal(heap, base_addr, heap_max_size); +} + +int +gc_destroy_with_pool(gc_handle_t handle) +{ + gc_heap_t *heap = (gc_heap_t *)handle; + int ret = GC_SUCCESS; + +#if WASM_ENABLE_GC != 0 + gc_size_t i = 0; + + if (heap->extra_info_node_cnt > 0) { + for (i = 0; i < heap->extra_info_node_cnt; i++) { + extra_info_node_t *node = heap->extra_info_nodes[i]; +#if BH_ENABLE_GC_VERIFY != 0 + os_printf("Memory leak detected: gc object [%p] not claimed\n", + node->obj); +#endif + bh_assert(heap->is_reclaim_enabled); + node->finalizer(node->obj, node->data); + + BH_FREE(heap->extra_info_nodes[i]); + } + + if (heap->extra_info_nodes != heap->extra_info_normal_nodes) { + BH_FREE(heap->extra_info_nodes); + } + } +#endif + +#if BH_ENABLE_GC_VERIFY != 0 + hmu_t *cur = (hmu_t *)heap->base_addr; + hmu_t *end = (hmu_t *)((char *)heap->base_addr + heap->current_size); + + if ( +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + !heap->is_heap_corrupted && +#endif + (hmu_t *)((char *)cur + hmu_get_size(cur)) != end) { + LOG_WARNING("Memory leak detected:\n"); + gci_dump(heap); + ret = GC_ERROR; + } +#endif + + os_mutex_destroy(&heap->lock); + memset(heap->base_addr, 0, heap->current_size); + memset(heap, 0, sizeof(gc_heap_t)); + return ret; +} + +#if WASM_ENABLE_GC != 0 +#if WASM_ENABLE_THREAD_MGR == 0 +void +gc_enable_gc_reclaim(gc_handle_t handle, void *exec_env) +{ + gc_heap_t *heap = (gc_heap_t *)handle; + + heap->is_reclaim_enabled = 1; + heap->exec_env = exec_env; +} +#else +void +gc_enable_gc_reclaim(gc_handle_t handle, void *cluster) +{ + gc_heap_t *heap = (gc_heap_t *)handle; + + heap->is_reclaim_enabled = 1; + heap->cluster = cluster; +} +#endif +#endif + +uint32 +gc_get_heap_struct_size() +{ + return sizeof(gc_heap_t); +} + +static void +adjust_ptr(uint8 **p_ptr, intptr_t offset) +{ + if ((!*p_ptr)) { + return; + } + + /* + * to resolve a possible signed integer overflow issue + * when p_ptr is over 0x8000000000000000 by not using + * `(intptr_t)` + */ + uintptr_t offset_val = 0; +#if UINTPTR_MAX == UINT64_MAX + offset_val = labs(offset); +#else + offset_val = abs(offset); +#endif + + if (offset > 0) { + *p_ptr = (uint8 *)((uintptr_t)(*p_ptr) + offset_val); + } + else { + *p_ptr = (uint8 *)((uintptr_t)(*p_ptr) - offset_val); + } +} + +int +gc_migrate(gc_handle_t handle, char *pool_buf_new, gc_size_t pool_buf_size) +{ + gc_heap_t *heap = (gc_heap_t *)handle; + char *base_addr_new = pool_buf_new + GC_HEAD_PADDING; + char *pool_buf_end = pool_buf_new + pool_buf_size; + intptr_t offset = (uint8 *)base_addr_new - (uint8 *)heap->base_addr; + hmu_t *cur = NULL, *end = NULL; + hmu_tree_node_t *tree_node; + uint8 **p_left, **p_right, **p_parent; + gc_size_t heap_max_size, size; + + if ((((uintptr_t)pool_buf_new) & 7) != 0) { + LOG_ERROR("[GC_ERROR]heap migrate pool buf not 8-byte aligned\n"); + return GC_ERROR; + } + + heap_max_size = (uint32)(pool_buf_end - base_addr_new) & (uint32)~7; + + if (pool_buf_end < base_addr_new || heap_max_size < heap->current_size) { + LOG_ERROR("[GC_ERROR]heap migrate invalid pool buf size\n"); + return GC_ERROR; + } + + if (offset == 0) + return 0; + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (heap->is_heap_corrupted) { + LOG_ERROR("[GC_ERROR]Heap is corrupted, heap migrate failed.\n"); + return GC_ERROR; + } +#endif + + heap->base_addr = (uint8 *)base_addr_new; + + ASSERT_TREE_NODE_ALIGNED_ACCESS(heap->kfc_tree_root); + + p_left = (uint8 **)((uint8 *)heap->kfc_tree_root + + offsetof(hmu_tree_node_t, left)); + p_right = (uint8 **)((uint8 *)heap->kfc_tree_root + + offsetof(hmu_tree_node_t, right)); + p_parent = (uint8 **)((uint8 *)heap->kfc_tree_root + + offsetof(hmu_tree_node_t, parent)); + adjust_ptr(p_left, offset); + adjust_ptr(p_right, offset); + adjust_ptr(p_parent, offset); + + cur = (hmu_t *)heap->base_addr; + end = (hmu_t *)((char *)heap->base_addr + heap->current_size); + + while (cur < end) { + size = hmu_get_size(cur); + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (size <= 0 || size > (uint32)((uint8 *)end - (uint8 *)cur)) { + LOG_ERROR("[GC_ERROR]Heap is corrupted, heap migrate failed.\n"); + heap->is_heap_corrupted = true; + return GC_ERROR; + } +#endif + + if (hmu_get_ut(cur) == HMU_FC && !HMU_IS_FC_NORMAL(size)) { + tree_node = (hmu_tree_node_t *)cur; + + ASSERT_TREE_NODE_ALIGNED_ACCESS(tree_node); + + p_left = (uint8 **)((uint8 *)tree_node + + offsetof(hmu_tree_node_t, left)); + p_right = (uint8 **)((uint8 *)tree_node + + offsetof(hmu_tree_node_t, right)); + p_parent = (uint8 **)((uint8 *)tree_node + + offsetof(hmu_tree_node_t, parent)); + adjust_ptr(p_left, offset); + adjust_ptr(p_right, offset); + if (tree_node->parent != heap->kfc_tree_root) + /* The root node belongs to heap structure, + it is fixed part and isn't changed. */ + adjust_ptr(p_parent, offset); + } + cur = (hmu_t *)((char *)cur + size); + } + +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + if (cur != end) { + LOG_ERROR("[GC_ERROR]Heap is corrupted, heap migrate failed.\n"); + heap->is_heap_corrupted = true; + return GC_ERROR; + } +#else + bh_assert(cur == end); +#endif + + return 0; +} + +bool +gc_is_heap_corrupted(gc_handle_t handle) +{ +#if BH_ENABLE_GC_CORRUPTION_CHECK != 0 + gc_heap_t *heap = (gc_heap_t *)handle; + + return heap->is_heap_corrupted ? true : false; +#else + return false; +#endif +} + +#if BH_ENABLE_GC_VERIFY != 0 +void +gci_verify_heap(gc_heap_t *heap) +{ + hmu_t *cur = NULL, *end = NULL; + + bh_assert(heap && gci_is_heap_valid(heap)); + cur = (hmu_t *)heap->base_addr; + end = (hmu_t *)(heap->base_addr + heap->current_size); + while (cur < end) { + hmu_verify(heap, cur); + cur = (hmu_t *)((gc_uint8 *)cur + hmu_get_size(cur)); + } + bh_assert(cur == end); +} +#endif + +void +gc_heap_stat(void *heap_ptr, gc_stat_t *stat) +{ + hmu_t *cur = NULL, *end = NULL; + hmu_type_t ut; + gc_size_t size; + gc_heap_t *heap = (gc_heap_t *)heap_ptr; + + memset(stat, 0, sizeof(gc_stat_t)); + cur = (hmu_t *)heap->base_addr; + end = (hmu_t *)((char *)heap->base_addr + heap->current_size); + + while (cur < end) { + ut = hmu_get_ut(cur); + size = hmu_get_size(cur); + bh_assert(size > 0); + + if (ut == HMU_FC || ut == HMU_FM + || (ut == HMU_VO && hmu_is_vo_freed(cur)) + || (ut == HMU_WO && !hmu_is_wo_marked(cur))) { + if (ut == HMU_VO) + stat->vo_free += size; + if (ut == HMU_WO) + stat->wo_free += size; + stat->free += size; + stat->free_block++; + if (size / sizeof(int) < GC_HEAP_STAT_SIZE - 1) + stat->free_sizes[size / sizeof(int)] += 1; + else + stat->free_sizes[GC_HEAP_STAT_SIZE - 1] += 1; + } + else { + if (ut == HMU_VO) + stat->vo_usage += size; + if (ut == HMU_WO) + stat->wo_usage += size; + stat->usage += size; + stat->usage_block++; + if (size / sizeof(int) < GC_HEAP_STAT_SIZE - 1) + stat->usage_sizes[size / sizeof(int)] += 1; + else + stat->usage_sizes[GC_HEAP_STAT_SIZE - 1] += 1; + } + + cur = (hmu_t *)((char *)cur + size); + } +} + +void +gc_print_stat(void *heap_ptr, int verbose) +{ + gc_stat_t stat; + int i; + + bh_assert(heap_ptr != NULL); + gc_heap_t *heap = (gc_heap_t *)(heap_ptr); + + gc_heap_stat(heap, &stat); + + os_printf("# stat %s %p use %d free %d \n", "instance", heap, stat.usage, + stat.free); + os_printf("# stat %s %p wo_usage %d vo_usage %d \n", "instance", heap, + stat.wo_usage, stat.vo_usage); + os_printf("# stat %s %p wo_free %d vo_free %d \n", "instance", heap, + stat.wo_free, stat.vo_free); +#if WASM_ENABLE_GC == 0 + os_printf("# stat free size %" PRIu32 " high %" PRIu32 "\n", + heap->total_free_size, heap->highmark_size); +#else + os_printf("# stat gc %" PRIu32 " free size %" PRIu32 " high %" PRIu32 "\n", + heap->total_gc_count, heap->total_free_size, heap->highmark_size); +#endif + if (verbose) { + os_printf("usage sizes: \n"); + for (i = 0; i < GC_HEAP_STAT_SIZE; i++) + if (stat.usage_sizes[i]) + os_printf(" %d: %d; ", i * 4, stat.usage_sizes[i]); + os_printf(" \n"); + os_printf("free sizes: \n"); + for (i = 0; i < GC_HEAP_STAT_SIZE; i++) + if (stat.free_sizes[i]) + os_printf(" %d: %d; ", i * 4, stat.free_sizes[i]); + } +} + +void * +gc_heap_stats(void *heap_arg, uint32 *stats, int size) +{ + int i; + gc_heap_t *heap = (gc_heap_t *)heap_arg; + + if (!gci_is_heap_valid(heap)) { + for (i = 0; i < size; i++) + stats[i] = 0; + return NULL; + } + + for (i = 0; i < size; i++) { + switch (i) { + case GC_STAT_TOTAL: + stats[i] = heap->current_size; + break; + case GC_STAT_FREE: + stats[i] = heap->total_free_size; + break; + case GC_STAT_HIGHMARK: + stats[i] = heap->highmark_size; + break; +#if WASM_ENABLE_GC != 0 + case GC_STAT_COUNT: + stats[i] = heap->total_gc_count; + break; + case GC_STAT_TIME: + stats[i] = heap->total_gc_time; + break; +#endif + default: + break; + } + } + + return heap; +} + +void +gc_traverse_tree(hmu_tree_node_t *node, gc_size_t *stats, int *n) +{ + if (!node) + return; + + if (*n > 0) + gc_traverse_tree(node->right, stats, n); + + if (*n > 0) { + (*n)--; + stats[*n] = node->size; + } + + if (*n > 0) + gc_traverse_tree(node->left, stats, n); +} + +void +gc_show_stat(void *heap) +{ + + uint32 stats[GC_STAT_MAX]; + + heap = gc_heap_stats(heap, stats, GC_STAT_MAX); + + os_printf("\n[GC stats %p] %" PRIu32 " %" PRIu32 " %" PRIu32 " %" PRIu32 + " %" PRIu32 "\n", + heap, stats[0], stats[1], stats[2], stats[3], stats[4]); +} + +#if WASM_ENABLE_GC != 0 +void +gc_show_fragment(void *heap_arg) +{ + uint32 stats[3]; + int n = 3; + gc_heap_t *heap = (gc_heap_t *)heap_arg; + + memset(stats, 0, n * sizeof(int)); + gct_vm_mutex_lock(&heap->lock); + gc_traverse_tree(heap->kfc_tree_root, (gc_size_t *)stats, &n); + gct_vm_mutex_unlock(&heap->lock); + os_printf("\n[GC %p top sizes] %" PRIu32 " %" PRIu32 " %" PRIu32 "\n", heap, + stats[0], stats[1], stats[2]); +} + +#if WASM_ENABLE_GC_PERF_PROFILING != 0 +void +gc_dump_perf_profiling(gc_handle_t *handle) +{ + gc_heap_t *gc_heap_handle = (void *)handle; + if (gc_heap_handle) { + os_printf("\nGC performance summary\n"); + os_printf(" Total GC time (ms): %u\n", + gc_heap_handle->total_gc_time); + os_printf(" Max GC time (ms): %u\n", gc_heap_handle->max_gc_time); + } + else { + os_printf("Failed to dump GC performance\n"); + } +} +#endif +#endif diff --git a/wamr/src/mem_alloc.c b/wamr/src/mem_alloc.c new file mode 100644 index 0000000..c6ae9bf --- /dev/null +++ b/wamr/src/mem_alloc.c @@ -0,0 +1,252 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "mem_alloc.h" // mem_alloc.h +#include + +#if DEFAULT_MEM_ALLOCATOR == MEM_ALLOCATOR_EMS + +#include "ems_gc.h" // ems/ems_gc.h + +mem_allocator_t +mem_allocator_create(void *mem, uint32_t size) +{ + return gc_init_with_pool((char *)mem, size); +} + +mem_allocator_t +mem_allocator_create_with_struct_and_pool(void *struct_buf, + uint32_t struct_buf_size, + void *pool_buf, + uint32_t pool_buf_size) +{ + return gc_init_with_struct_and_pool((char *)struct_buf, struct_buf_size, + pool_buf, pool_buf_size); +} + +int +mem_allocator_destroy(mem_allocator_t allocator) +{ + return gc_destroy_with_pool((gc_handle_t)allocator); +} + +uint32 +mem_allocator_get_heap_struct_size() +{ + return gc_get_heap_struct_size(); +} + +void * +mem_allocator_malloc(mem_allocator_t allocator, uint32_t size) +{ + return gc_alloc_vo((gc_handle_t)allocator, size); +} + +void * +mem_allocator_realloc(mem_allocator_t allocator, void *ptr, uint32_t size) +{ + return gc_realloc_vo((gc_handle_t)allocator, ptr, size); +} + +void +mem_allocator_free(mem_allocator_t allocator, void *ptr) +{ + if (ptr) + gc_free_vo((gc_handle_t)allocator, ptr); +} + +#if WASM_ENABLE_GC != 0 +void * +mem_allocator_malloc_with_gc(mem_allocator_t allocator, uint32_t size) +{ + return gc_alloc_wo((gc_handle_t)allocator, size); +} + +#if WASM_GC_MANUALLY != 0 +void +mem_allocator_free_with_gc(mem_allocator_t allocator, void *ptr) +{ + if (ptr) + gc_free_wo((gc_handle_t)allocator, ptr); +} +#endif + +#if WASM_ENABLE_THREAD_MGR == 0 +void +mem_allocator_enable_gc_reclaim(mem_allocator_t allocator, void *exec_env) +{ + gc_enable_gc_reclaim((gc_handle_t)allocator, exec_env); +} +#else +void +mem_allocator_enable_gc_reclaim(mem_allocator_t allocator, void *cluster) +{ + gc_enable_gc_reclaim((gc_handle_t)allocator, cluster); +} +#endif + +int +mem_allocator_add_root(mem_allocator_t allocator, WASMObjectRef obj) +{ + return gc_add_root((gc_handle_t)allocator, (gc_object_t)obj); +} +#endif + +int +mem_allocator_migrate(mem_allocator_t allocator, char *pool_buf_new, + uint32 pool_buf_size) +{ + return gc_migrate((gc_handle_t)allocator, pool_buf_new, pool_buf_size); +} + +bool +mem_allocator_is_heap_corrupted(mem_allocator_t allocator) +{ + return gc_is_heap_corrupted((gc_handle_t)allocator); +} + +bool +mem_allocator_get_alloc_info(mem_allocator_t allocator, void *mem_alloc_info) +{ + gc_heap_stats((gc_handle_t)allocator, mem_alloc_info, 3); + return true; +} + +#if WASM_ENABLE_GC != 0 +bool +mem_allocator_set_gc_finalizer(mem_allocator_t allocator, void *obj, + gc_finalizer_t cb, void *data) +{ + return gc_set_finalizer((gc_handle_t)allocator, (gc_object_t)obj, cb, data); +} + +void +mem_allocator_unset_gc_finalizer(mem_allocator_t allocator, void *obj) +{ + gc_unset_finalizer((gc_handle_t)allocator, (gc_object_t)obj); +} + +#if WASM_ENABLE_GC_PERF_PROFILING != 0 +void +mem_allocator_dump_perf_profiling(mem_allocator_t allocator) +{ + gc_dump_perf_profiling((gc_handle_t)allocator); +} +#endif + +#endif + +#else /* else of DEFAULT_MEM_ALLOCATOR */ + +#include "tlsf.h" // tlsf/tlsf.h + +typedef struct mem_allocator_tlsf { + tlsf_t tlsf; + korp_mutex lock; +} mem_allocator_tlsf; + +mem_allocator_t +mem_allocator_create(void *mem, uint32_t size) +{ + mem_allocator_tlsf *allocator_tlsf; + tlsf_t tlsf; + char *mem_aligned = (char *)(((uintptr_t)mem + 3) & ~3); + + if (size < 1024) { + printf("Create mem allocator failed: pool size must be " + "at least 1024 bytes.\n"); + return NULL; + } + + size -= mem_aligned - (char *)mem; + mem = (void *)mem_aligned; + + tlsf = tlsf_create_with_pool(mem, size); + if (!tlsf) { + printf("Create mem allocator failed: tlsf_create_with_pool failed.\n"); + return NULL; + } + + allocator_tlsf = tlsf_malloc(tlsf, sizeof(mem_allocator_tlsf)); + if (!allocator_tlsf) { + printf("Create mem allocator failed: tlsf_malloc failed.\n"); + tlsf_destroy(tlsf); + return NULL; + } + + allocator_tlsf->tlsf = tlsf; + + if (os_mutex_init(&allocator_tlsf->lock)) { + printf("Create mem allocator failed: tlsf_malloc failed.\n"); + tlsf_free(tlsf, allocator_tlsf); + tlsf_destroy(tlsf); + return NULL; + } + + return allocator_tlsf; +} + +void +mem_allocator_destroy(mem_allocator_t allocator) +{ + mem_allocator_tlsf *allocator_tlsf = (mem_allocator_tlsf *)allocator; + tlsf_t tlsf = allocator_tlsf->tlsf; + + os_mutex_destroy(&allocator_tlsf->lock); + tlsf_free(tlsf, allocator_tlsf); + tlsf_destroy(tlsf); +} + +void * +mem_allocator_malloc(mem_allocator_t allocator, uint32_t size) +{ + void *ret; + mem_allocator_tlsf *allocator_tlsf = (mem_allocator_tlsf *)allocator; + + if (size == 0) + /* tlsf doesn't allow to allocate 0 byte */ + size = 1; + + os_mutex_lock(&allocator_tlsf->lock); + ret = tlsf_malloc(allocator_tlsf->tlsf, size); + os_mutex_unlock(&allocator_tlsf->lock); + return ret; +} + +void * +mem_allocator_realloc(mem_allocator_t allocator, void *ptr, uint32_t size) +{ + void *ret; + mem_allocator_tlsf *allocator_tlsf = (mem_allocator_tlsf *)allocator; + + if (size == 0) + /* tlsf doesn't allow to allocate 0 byte */ + size = 1; + + os_mutex_lock(&allocator_tlsf->lock); + ret = tlsf_realloc(allocator_tlsf->tlsf, ptr, size); + os_mutex_unlock(&allocator_tlsf->lock); + return ret; +} + +void +mem_allocator_free(mem_allocator_t allocator, void *ptr) +{ + if (ptr) { + mem_allocator_tlsf *allocator_tlsf = (mem_allocator_tlsf *)allocator; + os_mutex_lock(&allocator_tlsf->lock); + tlsf_free(allocator_tlsf->tlsf, ptr); + os_mutex_unlock(&allocator_tlsf->lock); + } +} + +int +mem_allocator_migrate(mem_allocator_t allocator, mem_allocator_t allocator_old) +{ + return tlsf_migrate((mem_allocator_tlsf *)allocator, + (mem_allocator_tlsf *)allocator_old); +} + +#endif /* end of DEFAULT_MEM_ALLOCATOR */ diff --git a/wamr/src/posix_thread.c b/wamr/src/posix_thread.c new file mode 100644 index 0000000..0f2c047 --- /dev/null +++ b/wamr/src/posix_thread.c @@ -0,0 +1,784 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _GNU_SOURCE +#if !defined(__RTTHREAD__) +#define _GNU_SOURCE +#endif +#endif +#include "platform_api_vmcore.h" // platform_api_vmcore.h +#include "platform_api_extension.h" // platform_api_extension.h + +#if defined(__APPLE__) || defined(__MACH__) +#include +#endif + +typedef struct { + thread_start_routine_t start; + void *arg; +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_signal_handler signal_handler; +#endif +} thread_wrapper_arg; + +#ifdef OS_ENABLE_HW_BOUND_CHECK +/* The signal handler passed to os_thread_signal_init() */ +static os_thread_local_attribute os_signal_handler signal_handler; +#endif + +static void * +os_thread_wrapper(void *arg) +{ + thread_wrapper_arg *targ = arg; + thread_start_routine_t start_func = targ->start; + void *thread_arg = targ->arg; +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_signal_handler handler = targ->signal_handler; +#endif + +#if 0 + os_printf("THREAD CREATED %jx\n", (uintmax_t)(uintptr_t)pthread_self()); +#endif + BH_FREE(targ); +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (os_thread_signal_init(handler) != 0) + return NULL; +#endif +#ifdef OS_ENABLE_WAKEUP_BLOCKING_OP + os_end_blocking_op(); +#endif +#if BH_DEBUG != 0 +#if defined __APPLE__ + pthread_setname_np("wamr"); +#else + pthread_setname_np(pthread_self(), "wamr"); +#endif +#endif + start_func(thread_arg); +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_thread_signal_destroy(); +#endif + return NULL; +} + +int +os_thread_create_with_prio(korp_tid *tid, thread_start_routine_t start, + void *arg, unsigned int stack_size, int prio) +{ + pthread_attr_t tattr; + thread_wrapper_arg *targ; + + assert(stack_size > 0); + assert(tid); + assert(start); + + pthread_attr_init(&tattr); + pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_JOINABLE); + if (pthread_attr_setstacksize(&tattr, stack_size) != 0) { + os_printf("Invalid thread stack size %u. " + "Min stack size on Linux = %u\n", + stack_size, (unsigned int)PTHREAD_STACK_MIN); + pthread_attr_destroy(&tattr); + return BHT_ERROR; + } + + targ = (thread_wrapper_arg *)BH_MALLOC(sizeof(*targ)); + if (!targ) { + pthread_attr_destroy(&tattr); + return BHT_ERROR; + } + + targ->start = start; + targ->arg = arg; +#ifdef OS_ENABLE_HW_BOUND_CHECK + targ->signal_handler = signal_handler; +#endif + + if (pthread_create(tid, &tattr, os_thread_wrapper, targ) != 0) { + pthread_attr_destroy(&tattr); + BH_FREE(targ); + return BHT_ERROR; + } + + pthread_attr_destroy(&tattr); + return BHT_OK; +} + +int +os_thread_create(korp_tid *tid, thread_start_routine_t start, void *arg, + unsigned int stack_size) +{ + return os_thread_create_with_prio(tid, start, arg, stack_size, + BH_THREAD_DEFAULT_PRIORITY); +} + +korp_tid +os_self_thread() +{ + return (korp_tid)pthread_self(); +} + +int +os_mutex_init(korp_mutex *mutex) +{ + return pthread_mutex_init(mutex, NULL) == 0 ? BHT_OK : BHT_ERROR; +} + +int +os_recursive_mutex_init(korp_mutex *mutex) +{ + int ret; + + pthread_mutexattr_t mattr; + + assert(mutex); + ret = pthread_mutexattr_init(&mattr); + if (ret) + return BHT_ERROR; + + pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_RECURSIVE); + ret = pthread_mutex_init(mutex, &mattr); + pthread_mutexattr_destroy(&mattr); + + return ret == 0 ? BHT_OK : BHT_ERROR; +} + +int +os_mutex_destroy(korp_mutex *mutex) +{ + int ret; + + assert(mutex); + ret = pthread_mutex_destroy(mutex); + + return ret == 0 ? BHT_OK : BHT_ERROR; +} + +int +os_mutex_lock(korp_mutex *mutex) +{ + int ret; + + assert(mutex); + ret = pthread_mutex_lock(mutex); + + return ret == 0 ? BHT_OK : BHT_ERROR; +} + +int +os_mutex_unlock(korp_mutex *mutex) +{ + int ret; + + assert(mutex); + ret = pthread_mutex_unlock(mutex); + + return ret == 0 ? BHT_OK : BHT_ERROR; +} + +int +os_cond_init(korp_cond *cond) +{ + assert(cond); + + if (pthread_cond_init(cond, NULL) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_cond_destroy(korp_cond *cond) +{ + assert(cond); + + if (pthread_cond_destroy(cond) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_cond_wait(korp_cond *cond, korp_mutex *mutex) +{ + assert(cond); + assert(mutex); + + if (pthread_cond_wait(cond, mutex) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +korp_sem * +os_sem_open(const char *name, int oflags, int mode, int val) +{ + return sem_open(name, oflags, mode, val); +} + +int +os_sem_close(korp_sem *sem) +{ + return sem_close(sem); +} + +int +os_sem_wait(korp_sem *sem) +{ + return sem_wait(sem); +} + +int +os_sem_trywait(korp_sem *sem) +{ + return sem_trywait(sem); +} + +int +os_sem_post(korp_sem *sem) +{ + return sem_post(sem); +} + +int +os_sem_getvalue(korp_sem *sem, int *sval) +{ +#if defined(__APPLE__) + /* + * macOS doesn't have working sem_getvalue. + * It's marked as deprecated in the system header. + * Mock it up here to avoid compile-time deprecation warnings. + */ + errno = ENOSYS; + return -1; +#else + return sem_getvalue(sem, sval); +#endif +} + +int +os_sem_unlink(const char *name) +{ + return sem_unlink(name); +} + +static void +msec_nsec_to_abstime(struct timespec *ts, uint64 usec) +{ + struct timeval tv; + time_t tv_sec_new; + long int tv_nsec_new; + + gettimeofday(&tv, NULL); + + tv_sec_new = (time_t)(tv.tv_sec + usec / 1000000); + if (tv_sec_new >= tv.tv_sec) { + ts->tv_sec = tv_sec_new; + } + else { + /* integer overflow */ + ts->tv_sec = BH_TIME_T_MAX; + os_printf("Warning: os_cond_reltimedwait exceeds limit, " + "set to max timeout instead\n"); + } + + tv_nsec_new = (long int)(tv.tv_usec * 1000 + (usec % 1000000) * 1000); + if (tv.tv_usec * 1000 >= tv.tv_usec && tv_nsec_new >= tv.tv_usec * 1000) { + ts->tv_nsec = tv_nsec_new; + } + else { + /* integer overflow */ + ts->tv_nsec = LONG_MAX; + os_printf("Warning: os_cond_reltimedwait exceeds limit, " + "set to max timeout instead\n"); + } + + if (ts->tv_nsec >= 1000000000L && ts->tv_sec < BH_TIME_T_MAX) { + ts->tv_sec++; + ts->tv_nsec -= 1000000000L; + } +} + +int +os_cond_reltimedwait(korp_cond *cond, korp_mutex *mutex, uint64 useconds) +{ + int ret; + struct timespec abstime; + + if (useconds == BHT_WAIT_FOREVER) + ret = pthread_cond_wait(cond, mutex); + else { + msec_nsec_to_abstime(&abstime, useconds); + ret = pthread_cond_timedwait(cond, mutex, &abstime); + } + + if (ret != BHT_OK && ret != ETIMEDOUT) + return BHT_ERROR; + + return ret; +} + +int +os_cond_signal(korp_cond *cond) +{ + assert(cond); + + if (pthread_cond_signal(cond) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_cond_broadcast(korp_cond *cond) +{ + assert(cond); + + if (pthread_cond_broadcast(cond) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_rwlock_init(korp_rwlock *lock) +{ + assert(lock); + + if (pthread_rwlock_init(lock, NULL) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_rwlock_rdlock(korp_rwlock *lock) +{ + assert(lock); + + if (pthread_rwlock_rdlock(lock) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_rwlock_wrlock(korp_rwlock *lock) +{ + assert(lock); + + if (pthread_rwlock_wrlock(lock) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_rwlock_unlock(korp_rwlock *lock) +{ + assert(lock); + + if (pthread_rwlock_unlock(lock) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_rwlock_destroy(korp_rwlock *lock) +{ + assert(lock); + + if (pthread_rwlock_destroy(lock) != BHT_OK) + return BHT_ERROR; + + return BHT_OK; +} + +int +os_thread_join(korp_tid thread, void **value_ptr) +{ + return pthread_join(thread, value_ptr); +} + +int +os_thread_detach(korp_tid thread) +{ + return pthread_detach(thread); +} + +void +os_thread_exit(void *retval) +{ +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_thread_signal_destroy(); +#endif + return pthread_exit(retval); +} + +#if defined(os_thread_local_attribute) +static os_thread_local_attribute uint8 *thread_stack_boundary = NULL; +#endif + +uint8 * +os_thread_get_stack_boundary() +{ + pthread_t self; +#ifdef __linux__ + pthread_attr_t attr; + size_t guard_size; +#endif + uint8 *addr = NULL; + size_t stack_size, max_stack_size; + int page_size; + +#if defined(os_thread_local_attribute) + if (thread_stack_boundary) + return thread_stack_boundary; +#endif + + page_size = getpagesize(); + self = pthread_self(); + max_stack_size = + (size_t)(APP_THREAD_STACK_SIZE_MAX + page_size - 1) & ~(page_size - 1); + + if (max_stack_size < APP_THREAD_STACK_SIZE_DEFAULT) + max_stack_size = APP_THREAD_STACK_SIZE_DEFAULT; + +#ifdef __linux__ + if (pthread_getattr_np(self, &attr) == 0) { + pthread_attr_getstack(&attr, (void **)&addr, &stack_size); + pthread_attr_getguardsize(&attr, &guard_size); + pthread_attr_destroy(&attr); + if (stack_size > max_stack_size) + addr = addr + stack_size - max_stack_size; + addr += guard_size; + } + (void)stack_size; +#elif defined(__APPLE__) || defined(__NuttX__) || defined(__RTTHREAD__) + if ((addr = (uint8 *)pthread_get_stackaddr_np(self))) { + stack_size = pthread_get_stacksize_np(self); + + /** + * Check whether stack_addr is the base or end of the stack, + * change it to the base if it is the end of stack. + */ + if (addr <= (uint8 *)&stack_size) + addr = addr + stack_size; + + if (stack_size > max_stack_size) + stack_size = max_stack_size; + + addr -= stack_size; + } +#endif + +#if defined(os_thread_local_attribute) + thread_stack_boundary = addr; +#endif + return addr; +} + +void +os_thread_jit_write_protect_np(bool enabled) +{ +#if (defined(__APPLE__) || defined(__MACH__)) && defined(__arm64__) \ + && defined(TARGET_OS_OSX) && TARGET_OS_OSX != 0 + pthread_jit_write_protect_np(enabled); +#endif +} + +#ifdef OS_ENABLE_HW_BOUND_CHECK + +#define SIG_ALT_STACK_SIZE (32 * 1024) + +/** + * Whether thread signal environment is initialized: + * the signal handler is registered, the stack pages are touched, + * the stack guard pages are set and signal alternate stack are set. + */ +static os_thread_local_attribute bool thread_signal_inited = false; + +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 +/* The signal alternate stack base addr */ +static os_thread_local_attribute uint8 *sigalt_stack_base_addr; +/* The previous signal alternate stack */ +static os_thread_local_attribute stack_t prev_sigalt_stack; + +/* + * ASAN is not designed to work with custom stack unwind or other low-level + * things. Ignore a function that does some low-level magic. (e.g. walking + * through the thread's stack bypassing the frame boundaries) + */ +#if defined(__clang__) +#pragma clang optimize off +__attribute__((no_sanitize_address)) +#elif defined(__GNUC__) +#pragma GCC push_options +#pragma GCC optimize("O0") +__attribute__((no_sanitize_address)) +#endif +static uint32 +touch_pages(uint8 *stack_min_addr, uint32 page_size) +{ + uint8 sum = 0; + while (1) { + volatile uint8 *touch_addr = (volatile uint8 *)os_alloca(page_size / 2); + if (touch_addr < stack_min_addr + page_size) { + sum += *(stack_min_addr + page_size - 1); + break; + } + *touch_addr = 0; + sum += *touch_addr; + } + return sum; +} +#if defined(__clang__) +#pragma clang optimize on +#elif defined(__GNUC__) +#pragma GCC pop_options +#endif + +static bool +init_stack_guard_pages() +{ + uint32 page_size = os_getpagesize(); + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; + uint8 *stack_min_addr = os_thread_get_stack_boundary(); + + if (stack_min_addr == NULL) + return false; + + /* Touch each stack page to ensure that it has been mapped: the OS + may lazily grow the stack mapping as a guard page is hit. */ + (void)touch_pages(stack_min_addr, page_size); + /* First time to call aot function, protect guard pages */ + if (os_mprotect(stack_min_addr, page_size * guard_page_count, + MMAP_PROT_NONE) + != 0) { + return false; + } + return true; +} + +static void +destroy_stack_guard_pages() +{ + uint32 page_size = os_getpagesize(); + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; + uint8 *stack_min_addr = os_thread_get_stack_boundary(); + + os_mprotect(stack_min_addr, page_size * guard_page_count, + MMAP_PROT_READ | MMAP_PROT_WRITE); +} +#endif /* end of WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 */ + +/* + * ASAN is not designed to work with custom stack unwind or other low-level + * things. Ignore a function that does some low-level magic. (e.g. walking + * through the thread's stack bypassing the frame boundaries) + */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((no_sanitize_address)) +#endif +static void +mask_signals(int how) +{ + sigset_t set; + + sigemptyset(&set); + sigaddset(&set, SIGSEGV); + sigaddset(&set, SIGBUS); + pthread_sigmask(how, &set, NULL); +} + +static struct sigaction prev_sig_act_SIGSEGV; +static struct sigaction prev_sig_act_SIGBUS; + +/* + * ASAN is not designed to work with custom stack unwind or other low-level + * things. Ignore a function that does some low-level magic. (e.g. walking + * through the thread's stack bypassing the frame boundaries) + */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((no_sanitize_address)) +#endif +static void +signal_callback(int sig_num, siginfo_t *sig_info, void *sig_ucontext) +{ + void *sig_addr = sig_info->si_addr; + struct sigaction *prev_sig_act = NULL; + + mask_signals(SIG_BLOCK); + + /* Try to handle signal with the registered signal handler */ + if (signal_handler && (sig_num == SIGSEGV || sig_num == SIGBUS)) { + signal_handler(sig_addr); + } + + if (sig_num == SIGSEGV) + prev_sig_act = &prev_sig_act_SIGSEGV; + else if (sig_num == SIGBUS) + prev_sig_act = &prev_sig_act_SIGBUS; + + /* Forward the signal to next handler if found */ + if (prev_sig_act && (prev_sig_act->sa_flags & SA_SIGINFO)) { + prev_sig_act->sa_sigaction(sig_num, sig_info, sig_ucontext); + } + else if (prev_sig_act + && prev_sig_act->sa_handler + /* Filter out SIG_DFL and SIG_IGN here, they will + run into the else branch below */ + && (void *)prev_sig_act->sa_handler != SIG_DFL + && (void *)prev_sig_act->sa_handler != SIG_IGN) { + prev_sig_act->sa_handler(sig_num); + } + /* Output signal info and then crash if signal is unhandled */ + else { + switch (sig_num) { + case SIGSEGV: + os_printf("unhandled SIGSEGV, si_addr: %p\n", sig_addr); + break; + case SIGBUS: + os_printf("unhandled SIGBUS, si_addr: %p\n", sig_addr); + break; + default: + os_printf("unhandle signal %d, si_addr: %p\n", sig_num, + sig_addr); + break; + } + + abort(); + } +} + +int +os_thread_signal_init(os_signal_handler handler) +{ + struct sigaction sig_act; +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + stack_t sigalt_stack_info; + uint32 map_size = SIG_ALT_STACK_SIZE; + uint8 *map_addr; +#endif + + if (thread_signal_inited) + return 0; + +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + if (!init_stack_guard_pages()) { + os_printf("Failed to init stack guard pages\n"); + return -1; + } + + /* Initialize memory for signal alternate stack of current thread */ + if (!(map_addr = os_mmap(NULL, map_size, MMAP_PROT_READ | MMAP_PROT_WRITE, + MMAP_MAP_NONE, os_get_invalid_handle()))) { + os_printf("Failed to mmap memory for alternate stack\n"); + goto fail1; + } + + /* Initialize signal alternate stack */ + memset(map_addr, 0, map_size); + sigalt_stack_info.ss_sp = map_addr; + sigalt_stack_info.ss_size = map_size; + sigalt_stack_info.ss_flags = 0; + memset(&prev_sigalt_stack, 0, sizeof(stack_t)); + /* Set signal alternate stack and save the previous one */ + if (sigaltstack(&sigalt_stack_info, &prev_sigalt_stack) != 0) { + os_printf("Failed to init signal alternate stack\n"); + goto fail2; + } +#endif + + memset(&prev_sig_act_SIGSEGV, 0, sizeof(struct sigaction)); + memset(&prev_sig_act_SIGBUS, 0, sizeof(struct sigaction)); + + /* Install signal handler */ + sig_act.sa_sigaction = signal_callback; + sig_act.sa_flags = SA_SIGINFO | SA_NODEFER; +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + sig_act.sa_flags |= SA_ONSTACK; +#endif + sigemptyset(&sig_act.sa_mask); + if (sigaction(SIGSEGV, &sig_act, &prev_sig_act_SIGSEGV) != 0 + || sigaction(SIGBUS, &sig_act, &prev_sig_act_SIGBUS) != 0) { + os_printf("Failed to register signal handler\n"); + goto fail3; + } + +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + sigalt_stack_base_addr = map_addr; +#endif + +#if defined(os_thread_local_attribute) + // calculate and cache the new stack boundary. + // see https://github.com/bytecodealliance/wasm-micro-runtime/issues/3966 + (void)os_thread_get_stack_boundary(); +#endif + + signal_handler = handler; + thread_signal_inited = true; + return 0; + +fail3: +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + memset(&sigalt_stack_info, 0, sizeof(stack_t)); + sigalt_stack_info.ss_flags = SS_DISABLE; + sigalt_stack_info.ss_size = map_size; + sigaltstack(&sigalt_stack_info, NULL); +fail2: + os_munmap(map_addr, map_size); +fail1: + destroy_stack_guard_pages(); +#endif + return -1; +} + +void +os_thread_signal_destroy() +{ + if (!thread_signal_inited) + return; + +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + /* Restore the previous signal alternate stack */ + sigaltstack(&prev_sigalt_stack, NULL); + + os_munmap(sigalt_stack_base_addr, SIG_ALT_STACK_SIZE); + + destroy_stack_guard_pages(); +#endif + + thread_signal_inited = false; +} + +bool +os_thread_signal_inited() +{ + return thread_signal_inited; +} + +void +os_signal_unmask() +{ + mask_signals(SIG_UNBLOCK); +} + +void +os_sigreturn() +{ +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 +#if defined(__APPLE__) +#define UC_RESET_ALT_STACK 0x80000000 + extern int __sigreturn(void *, int); + + /* It's necessary to call __sigreturn to restore the sigaltstack state + after exiting the signal handler. */ + __sigreturn(NULL, UC_RESET_ALT_STACK); +#endif +#endif +} +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ diff --git a/wamr/src/wasm_c_api.c b/wamr/src/wasm_c_api.c new file mode 100644 index 0000000..a13c30a --- /dev/null +++ b/wamr/src/wasm_c_api.c @@ -0,0 +1,5387 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_log.h" // bh_log.h +#include "wasm_c_api_internal.h" // wasm_c_api_internal.h + +#include "bh_assert.h" // bh_assert.h +#include "wasm_export.h" // wasm_export.h +#include "wasm_memory.h" // wasm_memory.h +#if WASM_ENABLE_INTERP != 0 +#include "wasm_runtime.h" // wasm_runtime.h +#endif +#if WASM_ENABLE_AOT != 0 + +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT == 0 + + +#endif /*WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT == 0*/ +#endif /*WASM_ENABLE_AOT != 0*/ + + + + +/* + * Thread Model: + * - Only one wasm_engine_t in one process + * - One wasm_store_t is only accessed by one thread. wasm_store_t can't be + * shared in threads + * - wasm_module_t can be shared in threads + * - wasm_instance_t can not be shared in threads + */ + +#define ASSERT_NOT_IMPLEMENTED() bh_assert(!"not implemented") +#define UNREACHABLE() bh_assert(!"unreachable") + +typedef struct wasm_module_ex_t { + struct WASMModuleCommon *module_comm_rt; + wasm_byte_vec_t *binary; + /* If true, binary in wasm_module_ex_t contains a copy of the WASM binary */ + bool is_binary_cloned; + korp_mutex lock; + uint32 ref_count; +#if WASM_ENABLE_WASM_CACHE != 0 + char hash[SHA256_DIGEST_LENGTH]; +#endif +} wasm_module_ex_t; + +#ifndef os_thread_local_attribute +typedef struct thread_local_stores { + korp_tid tid; + unsigned stores_num; +} thread_local_stores; +#endif + +static void +wasm_module_delete_internal(wasm_module_t *); + +static void +wasm_instance_delete_internal(wasm_instance_t *); + +/* temporarily put stubs here */ +static wasm_store_t * +wasm_store_copy(const wasm_store_t *src) +{ + (void)src; + LOG_WARNING("in the stub of %s", __FUNCTION__); + return NULL; +} + +wasm_module_t * +wasm_module_copy(const wasm_module_t *src) +{ + (void)src; + LOG_WARNING("in the stub of %s", __FUNCTION__); + return NULL; +} + +wasm_instance_t * +wasm_instance_copy(const wasm_instance_t *src) +{ + (void)src; + LOG_WARNING("in the stub of %s", __FUNCTION__); + return NULL; +} + +/* ---------------------------------------------------------------------- */ +static inline void * +malloc_internal(uint64 size) +{ + void *mem = NULL; + + if (size < UINT32_MAX && (mem = wasm_runtime_malloc((uint32)size))) { + memset(mem, 0, size); + } + + return mem; +} + +/* clang-format off */ +#define RETURN_OBJ(obj, obj_del_func) \ + return obj; \ +failed: \ + obj_del_func(obj); \ + return NULL; + +#define RETURN_VOID(obj, obj_del_func) \ + return; \ +failed: \ + obj_del_func(obj); \ + return; +/* clang-format on */ + +/* Vectors */ +#define INIT_VEC(vector_p, init_func, ...) \ + do { \ + if (!(vector_p = malloc_internal(sizeof(*(vector_p))))) { \ + goto failed; \ + } \ + \ + init_func(vector_p, ##__VA_ARGS__); \ + if (vector_p->size && !vector_p->data) { \ + LOG_DEBUG("%s failed", #init_func); \ + goto failed; \ + } \ + } while (false) + +#define DEINIT_VEC(vector_p, deinit_func) \ + if ((vector_p)) { \ + deinit_func(vector_p); \ + wasm_runtime_free(vector_p); \ + vector_p = NULL; \ + } + +#define WASM_DEFINE_VEC(name) \ + void wasm_##name##_vec_new_empty(own wasm_##name##_vec_t *out) \ + { \ + wasm_##name##_vec_new_uninitialized(out, 0); \ + } \ + void wasm_##name##_vec_new_uninitialized(own wasm_##name##_vec_t *out, \ + size_t size) \ + { \ + wasm_##name##_vec_new(out, size, NULL); \ + } + +/* vectors with no ownership management of elements */ +#define WASM_DEFINE_VEC_PLAIN(name) \ + WASM_DEFINE_VEC(name) \ + void wasm_##name##_vec_new(own wasm_##name##_vec_t *out, size_t size, \ + own wasm_##name##_t const data[]) \ + { \ + if (!out) { \ + return; \ + } \ + \ + memset(out, 0, sizeof(wasm_##name##_vec_t)); \ + \ + if (!size) { \ + return; \ + } \ + \ + if (!bh_vector_init((Vector *)out, size, sizeof(wasm_##name##_t), \ + true)) { \ + LOG_DEBUG("bh_vector_init failed"); \ + goto failed; \ + } \ + \ + if (data) { \ + uint32 size_in_bytes = 0; \ + size_in_bytes = (uint32)(size * sizeof(wasm_##name##_t)); \ + bh_memcpy_s(out->data, size_in_bytes, data, size_in_bytes); \ + out->num_elems = size; \ + } \ + \ + RETURN_VOID(out, wasm_##name##_vec_delete) \ + } \ + void wasm_##name##_vec_copy(wasm_##name##_vec_t *out, \ + const wasm_##name##_vec_t *src) \ + { \ + if (!src) { \ + return; \ + } \ + wasm_##name##_vec_new(out, src->size, src->data); \ + } \ + void wasm_##name##_vec_delete(wasm_##name##_vec_t *v) \ + { \ + if (v) { \ + bh_vector_destroy((Vector *)v); \ + } \ + } + +/* vectors that own their elements */ +#define WASM_DEFINE_VEC_OWN(name, elem_destroy_func) \ + WASM_DEFINE_VEC(name) \ + void wasm_##name##_vec_new(own wasm_##name##_vec_t *out, size_t size, \ + own wasm_##name##_t *const data[]) \ + { \ + if (!out) { \ + return; \ + } \ + \ + memset(out, 0, sizeof(wasm_##name##_vec_t)); \ + \ + if (!size) { \ + return; \ + } \ + \ + if (!bh_vector_init((Vector *)out, size, sizeof(wasm_##name##_t *), \ + true)) { \ + LOG_DEBUG("bh_vector_init failed"); \ + goto failed; \ + } \ + \ + if (data) { \ + uint32 size_in_bytes = 0; \ + size_in_bytes = (uint32)(size * sizeof(wasm_##name##_t *)); \ + bh_memcpy_s(out->data, size_in_bytes, data, size_in_bytes); \ + out->num_elems = size; \ + } \ + \ + RETURN_VOID(out, wasm_##name##_vec_delete) \ + } \ + void wasm_##name##_vec_copy(own wasm_##name##_vec_t *out, \ + const wasm_##name##_vec_t *src) \ + { \ + size_t i = 0; \ + \ + if (!out) { \ + return; \ + } \ + memset(out, 0, sizeof(Vector)); \ + \ + if (!src || !src->size) { \ + return; \ + } \ + \ + if (!bh_vector_init((Vector *)out, src->size, \ + sizeof(wasm_##name##_t *), true)) { \ + LOG_DEBUG("bh_vector_init failed"); \ + goto failed; \ + } \ + \ + for (i = 0; i != src->num_elems; ++i) { \ + if (!(out->data[i] = wasm_##name##_copy(src->data[i]))) { \ + LOG_DEBUG("wasm_%s_copy failed", #name); \ + goto failed; \ + } \ + } \ + out->num_elems = src->num_elems; \ + \ + RETURN_VOID(out, wasm_##name##_vec_delete) \ + } \ + void wasm_##name##_vec_delete(wasm_##name##_vec_t *v) \ + { \ + size_t i = 0; \ + if (!v) { \ + return; \ + } \ + for (i = 0; i != v->num_elems && v->data; ++i) { \ + elem_destroy_func(*(v->data + i)); \ + } \ + bh_vector_destroy((Vector *)v); \ + } + +WASM_DEFINE_VEC_PLAIN(byte) +WASM_DEFINE_VEC_PLAIN(val) + +WASM_DEFINE_VEC_OWN(exporttype, wasm_exporttype_delete) +WASM_DEFINE_VEC_OWN(extern, wasm_extern_delete) +WASM_DEFINE_VEC_OWN(frame, wasm_frame_delete) +WASM_DEFINE_VEC_OWN(functype, wasm_functype_delete) +WASM_DEFINE_VEC_OWN(importtype, wasm_importtype_delete) +WASM_DEFINE_VEC_OWN(instance, wasm_instance_delete_internal) +WASM_DEFINE_VEC_OWN(module, wasm_module_delete_internal) +WASM_DEFINE_VEC_OWN(store, wasm_store_delete) +WASM_DEFINE_VEC_OWN(valtype, wasm_valtype_delete) + +#ifndef NDEBUG +#if WASM_ENABLE_MEMORY_PROFILING != 0 +#define WASM_C_DUMP_PROC_MEM() LOG_PROC_MEM() +#else +#define WASM_C_DUMP_PROC_MEM() (void)0 +#endif +#else +#define WASM_C_DUMP_PROC_MEM() (void)0 +#endif + +/* Runtime Environment */ +own wasm_config_t * +wasm_config_new(void) +{ + /* since wasm_runtime_malloc is not ready */ + wasm_config_t *config = os_malloc(sizeof(wasm_config_t)); + if (!config) + return NULL; + + memset(config, 0, sizeof(wasm_config_t)); + config->mem_alloc_type = Alloc_With_System_Allocator; + + return config; +} + +void +wasm_config_delete(own wasm_config_t *config) +{ + if (config) + os_free(config); +} + +wasm_config_t * +wasm_config_set_mem_alloc_opt(wasm_config_t *config, + mem_alloc_type_t mem_alloc_type, + MemAllocOption *mem_alloc_option) +{ + if (!config) + return NULL; + + config->mem_alloc_type = mem_alloc_type; + if (mem_alloc_option) + memcpy(&config->mem_alloc_option, mem_alloc_option, + sizeof(MemAllocOption)); + return config; +} + +wasm_config_t * +wasm_config_set_linux_perf_opt(wasm_config_t *config, bool enable) +{ + if (!config) + return NULL; + + config->enable_linux_perf = enable; + return config; +} + +wasm_config_t * +wasm_config_set_segue_flags(wasm_config_t *config, uint32 segue_flags) +{ + if (!config) + return NULL; + + config->segue_flags = segue_flags; + return config; +} + +static void +wasm_engine_delete_internal(wasm_engine_t *engine) +{ + if (engine) { + /* clean all created wasm_module_t and their locks */ + unsigned i; + + for (i = 0; i < engine->modules.num_elems; i++) { + wasm_module_ex_t *module; + if (bh_vector_get(&engine->modules, i, &module)) { + os_mutex_destroy(&module->lock); + wasm_runtime_free(module); + } + } + + bh_vector_destroy(&engine->modules); + +#ifndef os_thread_local_attribute + bh_vector_destroy(&engine->stores_by_tid); +#endif + + wasm_runtime_free(engine); + } + + wasm_runtime_destroy(); +} + +static wasm_engine_t * +wasm_engine_new_internal(wasm_config_t *config) +{ + wasm_engine_t *engine = NULL; + /* init runtime */ + RuntimeInitArgs init_args = { 0 }; +#if WASM_ENABLE_JIT != 0 + LLVMJITOptions *jit_options = wasm_runtime_get_llvm_jit_options(); +#endif + +#ifndef NDEBUG + bh_log_set_verbose_level(BH_LOG_LEVEL_VERBOSE); +#else + bh_log_set_verbose_level(BH_LOG_LEVEL_WARNING); +#endif + + WASM_C_DUMP_PROC_MEM(); + + /* wasm_config_t->MemAllocOption -> RuntimeInitArgs->MemAllocOption */ + init_args.mem_alloc_type = config->mem_alloc_type; + memcpy(&init_args.mem_alloc_option, &config->mem_alloc_option, + sizeof(MemAllocOption)); + init_args.enable_linux_perf = config->enable_linux_perf; + init_args.segue_flags = config->segue_flags; + +#if WASM_ENABLE_JIT != 0 + jit_options->quick_invoke_c_api_import = true; +#endif + + if (!wasm_runtime_full_init(&init_args)) { + LOG_DEBUG("wasm_runtime_full_init failed"); + goto failed; + } + + /* create wasm_engine_t */ + if (!(engine = malloc_internal(sizeof(wasm_engine_t)))) { + goto failed; + } + + if (!bh_vector_init(&engine->modules, DEFAULT_VECTOR_INIT_SIZE, + sizeof(wasm_module_ex_t *), true)) + goto failed; + +#ifndef os_thread_local_attribute + if (!bh_vector_init(&engine->stores_by_tid, DEFAULT_VECTOR_INIT_SIZE, + sizeof(thread_local_stores), true)) + goto failed; +#endif + + engine->ref_count = 1; + + WASM_C_DUMP_PROC_MEM(); + + RETURN_OBJ(engine, wasm_engine_delete_internal) +} + +/* global engine instance */ +static wasm_engine_t *singleton_engine; +#ifdef os_thread_local_attribute +/* categorize wasm_store_t as threads*/ +static os_thread_local_attribute unsigned thread_local_stores_num = 0; +#endif +#if defined(OS_THREAD_MUTEX_INITIALIZER) +/** + * lock for the singleton_engine + * Note: if the platform has mutex initializer, we use a global lock to + * lock the operations of the singleton_engine, otherwise when there are + * operations happening simultaneously in multiple threads, developer + * must create the lock by himself, and use it to lock the operations + */ +static korp_mutex engine_lock = OS_THREAD_MUTEX_INITIALIZER; +#endif + +own wasm_engine_t * +wasm_engine_new() +{ + wasm_config_t config = { 0 }; + wasm_config_set_mem_alloc_opt(&config, Alloc_With_System_Allocator, NULL); + wasm_engine_t *engine = wasm_engine_new_with_config(&config); + return engine; +} + +own wasm_engine_t * +wasm_engine_new_with_config(wasm_config_t *config) +{ +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&engine_lock); +#endif + + if (!singleton_engine) + singleton_engine = wasm_engine_new_internal(config); + else + singleton_engine->ref_count++; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&engine_lock); +#endif + + return singleton_engine; +} + +own wasm_engine_t * +wasm_engine_new_with_args(mem_alloc_type_t type, const MemAllocOption *opts) +{ + wasm_config_t config = { 0 }; + config.mem_alloc_type = type; + memcpy(&config.mem_alloc_option, opts, sizeof(MemAllocOption)); + return wasm_engine_new_with_config(&config); +} + +void +wasm_engine_delete(wasm_engine_t *engine) +{ + if (!engine) + return; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&engine_lock); +#endif + + if (!singleton_engine) { +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&engine_lock); +#endif + return; + } + + bh_assert(engine == singleton_engine); + bh_assert(singleton_engine->ref_count > 0); + + singleton_engine->ref_count--; + if (singleton_engine->ref_count == 0) { + wasm_engine_delete_internal(engine); + singleton_engine = NULL; + } + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&engine_lock); +#endif +} + +#ifndef os_thread_local_attribute +static bool +search_thread_local_store_num(Vector *stores_by_tid, korp_tid tid, + thread_local_stores *out_ts, unsigned *out_i) +{ + unsigned i; + + for (i = 0; i < stores_by_tid->num_elems; i++) { + bool ret = bh_vector_get(stores_by_tid, i, out_ts); + bh_assert(ret); + (void)ret; + + if (out_ts->tid == tid) { + *out_i = i; + return true; + } + } + + return false; +} +#endif + +static unsigned +retrieve_thread_local_store_num(Vector *stores_by_tid, korp_tid tid) +{ +#ifndef os_thread_local_attribute + unsigned i = 0; + thread_local_stores ts = { 0 }; + unsigned ret = 0; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&engine_lock); +#endif + + if (search_thread_local_store_num(stores_by_tid, tid, &ts, &i)) + ret = ts.stores_num; + else + ret = 0; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&engine_lock); +#endif + + return ret; +#else + (void)stores_by_tid; + (void)tid; + + return thread_local_stores_num; +#endif +} + +static bool +increase_thread_local_store_num(Vector *stores_by_tid, korp_tid tid) +{ +#ifndef os_thread_local_attribute + unsigned i = 0; + thread_local_stores ts = { 0 }; + bool ret = false; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&engine_lock); +#endif + + if (search_thread_local_store_num(stores_by_tid, tid, &ts, &i)) { + /* just in case if integer overflow */ + if (ts.stores_num + 1 < ts.stores_num) { + ret = false; + } + else { + ts.stores_num++; + ret = bh_vector_set(stores_by_tid, i, &ts); + bh_assert(ret); + } + } + else { + ts.tid = tid; + ts.stores_num = 1; + ret = bh_vector_append(stores_by_tid, &ts); + } + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&engine_lock); +#endif + return ret; +#else + (void)stores_by_tid; + (void)tid; + + /* just in case if integer overflow */ + if (thread_local_stores_num + 1 < thread_local_stores_num) + return false; + + thread_local_stores_num++; + return true; +#endif +} + +static bool +decrease_thread_local_store_num(Vector *stores_by_tid, korp_tid tid) +{ +#ifndef os_thread_local_attribute + unsigned i = 0; + thread_local_stores ts = { 0 }; + bool ret = false; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&engine_lock); +#endif + + ret = search_thread_local_store_num(stores_by_tid, tid, &ts, &i); + bh_assert(ret); + + /* just in case if integer overflow */ + if (ts.stores_num - 1 > ts.stores_num) { + ret = false; + } + else { + ts.stores_num--; + ret = bh_vector_set(stores_by_tid, i, &ts); + bh_assert(ret); + } + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&engine_lock); +#endif + + return ret; +#else + (void)stores_by_tid; + (void)tid; + + /* just in case if integer overflow */ + if (thread_local_stores_num - 1 > thread_local_stores_num) + return false; + + thread_local_stores_num--; + return true; +#endif +} + +wasm_store_t * +wasm_store_new(wasm_engine_t *engine) +{ + wasm_store_t *store = NULL; + + WASM_C_DUMP_PROC_MEM(); + + if (!engine || singleton_engine != engine) + return NULL; + + if (!retrieve_thread_local_store_num(&engine->stores_by_tid, + os_self_thread())) { + if (!wasm_runtime_init_thread_env()) { + LOG_ERROR("init thread environment failed"); + return NULL; + } + + if (!increase_thread_local_store_num(&engine->stores_by_tid, + os_self_thread())) { + wasm_runtime_destroy_thread_env(); + return NULL; + } + + if (!(store = malloc_internal(sizeof(wasm_store_t)))) { + decrease_thread_local_store_num(&singleton_engine->stores_by_tid, + os_self_thread()); + wasm_runtime_destroy_thread_env(); + return NULL; + } + } + else { + if (!increase_thread_local_store_num(&engine->stores_by_tid, + os_self_thread())) + return NULL; + + if (!(store = malloc_internal(sizeof(wasm_store_t)))) { + decrease_thread_local_store_num(&singleton_engine->stores_by_tid, + os_self_thread()); + return NULL; + } + } + + /* new a vector, and new its data */ + INIT_VEC(store->modules, wasm_module_vec_new_uninitialized, + DEFAULT_VECTOR_INIT_LENGTH); + INIT_VEC(store->instances, wasm_instance_vec_new_uninitialized, + DEFAULT_VECTOR_INIT_LENGTH); + + if (!(store->foreigns = malloc_internal(sizeof(Vector))) + || !(bh_vector_init(store->foreigns, 24, sizeof(wasm_foreign_t *), + true))) { + goto failed; + } + + WASM_C_DUMP_PROC_MEM(); + + return store; +failed: + wasm_store_delete(store); + return NULL; +} + +void +wasm_store_delete(wasm_store_t *store) +{ + if (!store) { + return; + } + + DEINIT_VEC(store->instances, wasm_instance_vec_delete); + DEINIT_VEC(store->modules, wasm_module_vec_delete); + if (store->foreigns) { + bh_vector_destroy(store->foreigns); + wasm_runtime_free(store->foreigns); + } + + wasm_runtime_free(store); + + if (decrease_thread_local_store_num(&singleton_engine->stores_by_tid, + os_self_thread())) { + if (!retrieve_thread_local_store_num(&singleton_engine->stores_by_tid, + os_self_thread())) { + wasm_runtime_destroy_thread_env(); + } + } +} + +/* Type Representations */ +static inline wasm_valkind_t +val_type_rt_2_valkind(uint8 val_type_rt) +{ + switch (val_type_rt) { +#define WAMR_VAL_TYPE_2_WASM_VAL_KIND(name) \ + case VALUE_TYPE_##name: \ + return WASM_##name; + + WAMR_VAL_TYPE_2_WASM_VAL_KIND(I32) + WAMR_VAL_TYPE_2_WASM_VAL_KIND(I64) + WAMR_VAL_TYPE_2_WASM_VAL_KIND(F32) + WAMR_VAL_TYPE_2_WASM_VAL_KIND(F64) + WAMR_VAL_TYPE_2_WASM_VAL_KIND(V128) + WAMR_VAL_TYPE_2_WASM_VAL_KIND(FUNCREF) +#undef WAMR_VAL_TYPE_2_WASM_VAL_KIND + + default: + return WASM_EXTERNREF; + } +} + +static wasm_valtype_t * +wasm_valtype_new_internal(uint8 val_type_rt) +{ + return wasm_valtype_new(val_type_rt_2_valkind(val_type_rt)); +} + +wasm_valtype_t * +wasm_valtype_new(wasm_valkind_t kind) +{ + wasm_valtype_t *val_type; + + if (kind > WASM_V128 && WASM_FUNCREF != kind +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + && WASM_EXTERNREF != kind +#endif + ) { + return NULL; + } + + if (!(val_type = malloc_internal(sizeof(wasm_valtype_t)))) { + return NULL; + } + + val_type->kind = kind; + + return val_type; +} + +void +wasm_valtype_delete(wasm_valtype_t *val_type) +{ + if (val_type) { + wasm_runtime_free(val_type); + } +} + +wasm_valtype_t * +wasm_valtype_copy(const wasm_valtype_t *src) +{ + return src ? wasm_valtype_new(src->kind) : NULL; +} + +wasm_valkind_t +wasm_valtype_kind(const wasm_valtype_t *val_type) +{ + return val_type ? val_type->kind : WASM_EXTERNREF; +} + +static wasm_functype_t * +wasm_functype_new_internal(WASMFuncType *type_rt) +{ + wasm_functype_t *type = NULL; + wasm_valtype_t *param_type = NULL, *result_type = NULL; + uint32 i = 0; + + if (!type_rt) { + return NULL; + } + + if (!(type = malloc_internal(sizeof(wasm_functype_t)))) { + return NULL; + } + + type->extern_kind = WASM_EXTERN_FUNC; + + /* WASMFuncType->types[0 : type_rt->param_count) -> type->params */ + INIT_VEC(type->params, wasm_valtype_vec_new_uninitialized, + type_rt->param_count); + for (i = 0; i < type_rt->param_count; ++i) { + if (!(param_type = wasm_valtype_new_internal(*(type_rt->types + i)))) { + goto failed; + } + + if (!bh_vector_append((Vector *)type->params, ¶m_type)) { + LOG_DEBUG("bh_vector_append failed"); + goto failed; + } + } + + /* WASMFuncType->types[type_rt->param_count : type_rt->result_count) -> + * type->results */ + INIT_VEC(type->results, wasm_valtype_vec_new_uninitialized, + type_rt->result_count); + for (i = 0; i < type_rt->result_count; ++i) { + if (!(result_type = wasm_valtype_new_internal( + *(type_rt->types + type_rt->param_count + i)))) { + goto failed; + } + + if (!bh_vector_append((Vector *)type->results, &result_type)) { + LOG_DEBUG("bh_vector_append failed"); + goto failed; + } + } + + return type; + +failed: + wasm_valtype_delete(param_type); + wasm_valtype_delete(result_type); + wasm_functype_delete(type); + return NULL; +} + +wasm_functype_t * +wasm_functype_new(own wasm_valtype_vec_t *params, + own wasm_valtype_vec_t *results) +{ + wasm_functype_t *type = NULL; + + if (!(type = malloc_internal(sizeof(wasm_functype_t)))) { + goto failed; + } + + type->extern_kind = WASM_EXTERN_FUNC; + + /* take ownership */ + if (!(type->params = malloc_internal(sizeof(wasm_valtype_vec_t)))) { + goto failed; + } + if (params) { + bh_memcpy_s(type->params, sizeof(wasm_valtype_vec_t), params, + sizeof(wasm_valtype_vec_t)); + } + + if (!(type->results = malloc_internal(sizeof(wasm_valtype_vec_t)))) { + goto failed; + } + if (results) { + bh_memcpy_s(type->results, sizeof(wasm_valtype_vec_t), results, + sizeof(wasm_valtype_vec_t)); + } + + return type; + +failed: + wasm_functype_delete(type); + return NULL; +} + +wasm_functype_t * +wasm_functype_copy(const wasm_functype_t *src) +{ + wasm_functype_t *functype; + wasm_valtype_vec_t params = { 0 }, results = { 0 }; + + if (!src) { + return NULL; + } + + wasm_valtype_vec_copy(¶ms, src->params); + if (src->params->size && !params.data) { + goto failed; + } + + wasm_valtype_vec_copy(&results, src->results); + if (src->results->size && !results.data) { + goto failed; + } + + if (!(functype = wasm_functype_new(¶ms, &results))) { + goto failed; + } + + return functype; + +failed: + wasm_valtype_vec_delete(¶ms); + wasm_valtype_vec_delete(&results); + return NULL; +} + +void +wasm_functype_delete(wasm_functype_t *func_type) +{ + if (!func_type) { + return; + } + + DEINIT_VEC(func_type->params, wasm_valtype_vec_delete); + DEINIT_VEC(func_type->results, wasm_valtype_vec_delete); + + wasm_runtime_free(func_type); +} + +const wasm_valtype_vec_t * +wasm_functype_params(const wasm_functype_t *func_type) +{ + if (!func_type) { + return NULL; + } + + return func_type->params; +} + +const wasm_valtype_vec_t * +wasm_functype_results(const wasm_functype_t *func_type) +{ + if (!func_type) { + return NULL; + } + + return func_type->results; +} + +static bool +cmp_val_kind_with_val_type(wasm_valkind_t v_k, uint8 v_t) +{ + return (v_k == WASM_I32 && v_t == VALUE_TYPE_I32) + || (v_k == WASM_I64 && v_t == VALUE_TYPE_I64) + || (v_k == WASM_F32 && v_t == VALUE_TYPE_F32) + || (v_k == WASM_F64 && v_t == VALUE_TYPE_F64) + || (v_k == WASM_V128 && v_t == VALUE_TYPE_V128) + || (v_k == WASM_EXTERNREF && v_t == VALUE_TYPE_EXTERNREF) + || (v_k == WASM_FUNCREF && v_t == VALUE_TYPE_FUNCREF); +} + +/* + *to compare a function type of wasm-c-api with a function type of wasm_runtime + */ +static bool +wasm_functype_same_internal(const wasm_functype_t *type, + const WASMFuncType *type_intl) +{ + uint32 i = 0; + + if (!type || !type_intl || type->params->num_elems != type_intl->param_count + || type->results->num_elems != type_intl->result_count) + return false; + + for (i = 0; i < type->params->num_elems; i++) { + wasm_valtype_t *v_t = type->params->data[i]; + if (!cmp_val_kind_with_val_type(wasm_valtype_kind(v_t), + type_intl->types[i])) + return false; + } + + for (i = 0; i < type->results->num_elems; i++) { + wasm_valtype_t *v_t = type->results->data[i]; + if (!cmp_val_kind_with_val_type( + wasm_valtype_kind(v_t), + type_intl->types[i + type->params->num_elems])) + return false; + } + + return true; +} + +wasm_globaltype_t * +wasm_globaltype_new(own wasm_valtype_t *val_type, wasm_mutability_t mut) +{ + wasm_globaltype_t *global_type = NULL; + + if (!val_type) { + return NULL; + } + + if (!(global_type = malloc_internal(sizeof(wasm_globaltype_t)))) { + return NULL; + } + + global_type->extern_kind = WASM_EXTERN_GLOBAL; + global_type->val_type = val_type; + global_type->mutability = mut; + + return global_type; +} + +wasm_globaltype_t * +wasm_globaltype_new_internal(uint8 val_type_rt, bool is_mutable) +{ + wasm_globaltype_t *globaltype; + wasm_valtype_t *val_type; + + if (!(val_type = wasm_valtype_new(val_type_rt_2_valkind(val_type_rt)))) { + return NULL; + } + + if (!(globaltype = wasm_globaltype_new( + val_type, is_mutable ? WASM_VAR : WASM_CONST))) { + wasm_valtype_delete(val_type); + } + + return globaltype; +} + +void +wasm_globaltype_delete(wasm_globaltype_t *global_type) +{ + if (!global_type) { + return; + } + + if (global_type->val_type) { + wasm_valtype_delete(global_type->val_type); + global_type->val_type = NULL; + } + + wasm_runtime_free(global_type); +} + +wasm_globaltype_t * +wasm_globaltype_copy(const wasm_globaltype_t *src) +{ + wasm_globaltype_t *global_type; + wasm_valtype_t *val_type; + + if (!src) { + return NULL; + } + + if (!(val_type = wasm_valtype_copy(src->val_type))) { + return NULL; + } + + if (!(global_type = wasm_globaltype_new(val_type, src->mutability))) { + wasm_valtype_delete(val_type); + } + + return global_type; +} + +const wasm_valtype_t * +wasm_globaltype_content(const wasm_globaltype_t *global_type) +{ + if (!global_type) { + return NULL; + } + + return global_type->val_type; +} + +wasm_mutability_t +wasm_globaltype_mutability(const wasm_globaltype_t *global_type) +{ + if (!global_type) { + return false; + } + + return global_type->mutability; +} + +static wasm_tabletype_t * +wasm_tabletype_new_internal(uint8 val_type_rt, uint32 init_size, + uint32 max_size) +{ + wasm_tabletype_t *table_type; + wasm_limits_t limits = { init_size, max_size }; + wasm_valtype_t *val_type; + + if (!(val_type = wasm_valtype_new_internal(val_type_rt))) { + return NULL; + } + + if (!(table_type = wasm_tabletype_new(val_type, &limits))) { + wasm_valtype_delete(val_type); + } + + return table_type; +} + +wasm_tabletype_t * +wasm_tabletype_new(own wasm_valtype_t *val_type, const wasm_limits_t *limits) +{ + wasm_tabletype_t *table_type = NULL; + + if (!val_type || !limits) { + return NULL; + } + + if (wasm_valtype_kind(val_type) != WASM_FUNCREF +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + && wasm_valtype_kind(val_type) != WASM_EXTERNREF +#endif + ) { + return NULL; + } + + if (!(table_type = malloc_internal(sizeof(wasm_tabletype_t)))) { + return NULL; + } + + table_type->extern_kind = WASM_EXTERN_TABLE; + table_type->val_type = val_type; + table_type->limits.min = limits->min; + table_type->limits.max = limits->max; + + return table_type; +} + +wasm_tabletype_t * +wasm_tabletype_copy(const wasm_tabletype_t *src) +{ + wasm_tabletype_t *table_type; + wasm_valtype_t *val_type; + + if (!src) { + return NULL; + } + + if (!(val_type = wasm_valtype_copy(src->val_type))) { + return NULL; + } + + if (!(table_type = wasm_tabletype_new(val_type, &src->limits))) { + wasm_valtype_delete(val_type); + } + + return table_type; +} + +void +wasm_tabletype_delete(wasm_tabletype_t *table_type) +{ + if (!table_type) { + return; + } + + if (table_type->val_type) { + wasm_valtype_delete(table_type->val_type); + table_type->val_type = NULL; + } + + wasm_runtime_free(table_type); +} + +const wasm_valtype_t * +wasm_tabletype_element(const wasm_tabletype_t *table_type) +{ + if (!table_type) { + return NULL; + } + + return table_type->val_type; +} + +const wasm_limits_t * +wasm_tabletype_limits(const wasm_tabletype_t *table_type) +{ + if (!table_type) { + return NULL; + } + + return &(table_type->limits); +} + +static wasm_memorytype_t * +wasm_memorytype_new_internal(uint32 min_pages, uint32 max_pages) +{ + wasm_limits_t limits = { min_pages, max_pages }; + return wasm_memorytype_new(&limits); +} + +wasm_memorytype_t * +wasm_memorytype_new(const wasm_limits_t *limits) +{ + wasm_memorytype_t *memory_type = NULL; + + if (!limits) { + return NULL; + } + + if (!(memory_type = malloc_internal(sizeof(wasm_memorytype_t)))) { + return NULL; + } + + memory_type->extern_kind = WASM_EXTERN_MEMORY; + memory_type->limits.min = limits->min; + memory_type->limits.max = limits->max; + + return memory_type; +} + +wasm_memorytype_t * +wasm_memorytype_copy(const wasm_memorytype_t *src) +{ + if (!src) { + return NULL; + } + + return wasm_memorytype_new(&src->limits); +} + +void +wasm_memorytype_delete(wasm_memorytype_t *memory_type) +{ + if (memory_type) { + wasm_runtime_free(memory_type); + } +} + +const wasm_limits_t * +wasm_memorytype_limits(const wasm_memorytype_t *memory_type) +{ + if (!memory_type) { + return NULL; + } + + return &(memory_type->limits); +} + +wasm_externkind_t +wasm_externtype_kind(const wasm_externtype_t *extern_type) +{ + if (!extern_type) { + return WASM_EXTERN_FUNC; + } + + return extern_type->extern_kind; +} + +#define BASIC_FOUR_TYPE_LIST(V) \ + V(functype) \ + V(globaltype) \ + V(memorytype) \ + V(tabletype) + +#define WASM_EXTERNTYPE_AS_OTHERTYPE(name) \ + wasm_##name##_t *wasm_externtype_as_##name(wasm_externtype_t *extern_type) \ + { \ + return (wasm_##name##_t *)extern_type; \ + } + +BASIC_FOUR_TYPE_LIST(WASM_EXTERNTYPE_AS_OTHERTYPE) +#undef WASM_EXTERNTYPE_AS_OTHERTYPE + +#define WASM_OTHERTYPE_AS_EXTERNTYPE(name) \ + wasm_externtype_t *wasm_##name##_as_externtype(wasm_##name##_t *other) \ + { \ + return (wasm_externtype_t *)other; \ + } + +BASIC_FOUR_TYPE_LIST(WASM_OTHERTYPE_AS_EXTERNTYPE) +#undef WASM_OTHERTYPE_AS_EXTERNTYPE + +#define WASM_EXTERNTYPE_AS_OTHERTYPE_CONST(name) \ + const wasm_##name##_t *wasm_externtype_as_##name##_const( \ + const wasm_externtype_t *extern_type) \ + { \ + return (const wasm_##name##_t *)extern_type; \ + } + +BASIC_FOUR_TYPE_LIST(WASM_EXTERNTYPE_AS_OTHERTYPE_CONST) +#undef WASM_EXTERNTYPE_AS_OTHERTYPE_CONST + +#define WASM_OTHERTYPE_AS_EXTERNTYPE_CONST(name) \ + const wasm_externtype_t *wasm_##name##_as_externtype_const( \ + const wasm_##name##_t *other) \ + { \ + return (const wasm_externtype_t *)other; \ + } + +BASIC_FOUR_TYPE_LIST(WASM_OTHERTYPE_AS_EXTERNTYPE_CONST) +#undef WASM_OTHERTYPE_AS_EXTERNTYPE_CONST + +wasm_externtype_t * +wasm_externtype_copy(const wasm_externtype_t *src) +{ + wasm_externtype_t *extern_type = NULL; + + if (!src) { + return NULL; + } + + switch (src->extern_kind) { +#define COPY_EXTERNTYPE(NAME, name) \ + case WASM_EXTERN_##NAME: \ + { \ + extern_type = wasm_##name##_as_externtype( \ + wasm_##name##_copy(wasm_externtype_as_##name##_const(src))); \ + break; \ + } + COPY_EXTERNTYPE(FUNC, functype) + COPY_EXTERNTYPE(GLOBAL, globaltype) + COPY_EXTERNTYPE(MEMORY, memorytype) + COPY_EXTERNTYPE(TABLE, tabletype) +#undef COPY_EXTERNTYPE + default: + LOG_WARNING("%s meets unsupported kind %u", __FUNCTION__, + src->extern_kind); + break; + } + return extern_type; +} + +void +wasm_externtype_delete(wasm_externtype_t *extern_type) +{ + if (!extern_type) { + return; + } + + switch (wasm_externtype_kind(extern_type)) { + case WASM_EXTERN_FUNC: + wasm_functype_delete(wasm_externtype_as_functype(extern_type)); + break; + case WASM_EXTERN_GLOBAL: + wasm_globaltype_delete(wasm_externtype_as_globaltype(extern_type)); + break; + case WASM_EXTERN_MEMORY: + wasm_memorytype_delete(wasm_externtype_as_memorytype(extern_type)); + break; + case WASM_EXTERN_TABLE: + wasm_tabletype_delete(wasm_externtype_as_tabletype(extern_type)); + break; + default: + LOG_WARNING("%s meets unsupported type %u", __FUNCTION__, + wasm_externtype_kind(extern_type)); + break; + } +} + +own wasm_importtype_t * +wasm_importtype_new(own wasm_byte_vec_t *module_name, + own wasm_byte_vec_t *field_name, + own wasm_externtype_t *extern_type) +{ + wasm_importtype_t *import_type = NULL; + + if (!module_name || !field_name || !extern_type) { + return NULL; + } + + if (!(import_type = malloc_internal(sizeof(wasm_importtype_t)))) { + return NULL; + } + + /* take ownership */ + if (!(import_type->module_name = + malloc_internal(sizeof(wasm_byte_vec_t)))) { + goto failed; + } + bh_memcpy_s(import_type->module_name, sizeof(wasm_byte_vec_t), module_name, + sizeof(wasm_byte_vec_t)); + + if (!(import_type->name = malloc_internal(sizeof(wasm_byte_vec_t)))) { + goto failed; + } + bh_memcpy_s(import_type->name, sizeof(wasm_byte_vec_t), field_name, + sizeof(wasm_byte_vec_t)); + + import_type->extern_type = extern_type; + + return import_type; +failed: + wasm_importtype_delete(import_type); + return NULL; +} + +void +wasm_importtype_delete(own wasm_importtype_t *import_type) +{ + if (!import_type) { + return; + } + + DEINIT_VEC(import_type->module_name, wasm_byte_vec_delete); + DEINIT_VEC(import_type->name, wasm_byte_vec_delete); + wasm_externtype_delete(import_type->extern_type); + import_type->extern_type = NULL; + wasm_runtime_free(import_type); +} + +own wasm_importtype_t * +wasm_importtype_copy(const wasm_importtype_t *src) +{ + wasm_byte_vec_t module_name = { 0 }, name = { 0 }; + wasm_externtype_t *extern_type = NULL; + wasm_importtype_t *import_type = NULL; + + if (!src) { + return NULL; + } + + wasm_byte_vec_copy(&module_name, src->module_name); + if (src->module_name->size && !module_name.data) { + goto failed; + } + + wasm_byte_vec_copy(&name, src->name); + if (src->name->size && !name.data) { + goto failed; + } + + if (!(extern_type = wasm_externtype_copy(src->extern_type))) { + goto failed; + } + + if (!(import_type = + wasm_importtype_new(&module_name, &name, extern_type))) { + goto failed; + } + + return import_type; + +failed: + wasm_byte_vec_delete(&module_name); + wasm_byte_vec_delete(&name); + wasm_externtype_delete(extern_type); + wasm_importtype_delete(import_type); + return NULL; +} + +const wasm_byte_vec_t * +wasm_importtype_module(const wasm_importtype_t *import_type) +{ + if (!import_type) { + return NULL; + } + + return import_type->module_name; +} + +const wasm_byte_vec_t * +wasm_importtype_name(const wasm_importtype_t *import_type) +{ + if (!import_type) { + return NULL; + } + + return import_type->name; +} + +const wasm_externtype_t * +wasm_importtype_type(const wasm_importtype_t *import_type) +{ + if (!import_type) { + return NULL; + } + + return import_type->extern_type; +} + +bool +wasm_importtype_is_linked(const wasm_importtype_t *import_type) +{ + if (!import_type) + return false; + + const wasm_name_t *module_name = wasm_importtype_module(import_type); + const wasm_name_t *field_name = wasm_importtype_name(import_type); + + switch (wasm_externtype_kind(wasm_importtype_type(import_type))) { + case WASM_EXTERN_FUNC: + return wasm_runtime_is_import_func_linked(module_name->data, + field_name->data); + case WASM_EXTERN_GLOBAL: + return wasm_runtime_is_import_global_linked(module_name->data, + field_name->data); + case WASM_EXTERN_MEMORY: + case WASM_EXTERN_TABLE: + default: + break; + } + return false; +} + +own wasm_exporttype_t * +wasm_exporttype_new(own wasm_byte_vec_t *name, + own wasm_externtype_t *extern_type) +{ + wasm_exporttype_t *export_type = NULL; + + if (!name || !extern_type) { + return NULL; + } + + if (!(export_type = malloc_internal(sizeof(wasm_exporttype_t)))) { + return NULL; + } + + if (!(export_type->name = malloc_internal(sizeof(wasm_byte_vec_t)))) { + wasm_exporttype_delete(export_type); + return NULL; + } + bh_memcpy_s(export_type->name, sizeof(wasm_byte_vec_t), name, + sizeof(wasm_byte_vec_t)); + + export_type->extern_type = extern_type; + + return export_type; +} + +wasm_exporttype_t * +wasm_exporttype_copy(const wasm_exporttype_t *src) +{ + wasm_exporttype_t *export_type; + wasm_byte_vec_t name = { 0 }; + wasm_externtype_t *extern_type = NULL; + + if (!src) { + return NULL; + } + + wasm_byte_vec_copy(&name, src->name); + if (src->name->size && !name.data) { + goto failed; + } + + if (!(extern_type = wasm_externtype_copy(src->extern_type))) { + goto failed; + } + + if (!(export_type = wasm_exporttype_new(&name, extern_type))) { + goto failed; + } + + return export_type; +failed: + wasm_byte_vec_delete(&name); + wasm_externtype_delete(extern_type); + return NULL; +} + +void +wasm_exporttype_delete(wasm_exporttype_t *export_type) +{ + if (!export_type) { + return; + } + + DEINIT_VEC(export_type->name, wasm_byte_vec_delete); + + wasm_externtype_delete(export_type->extern_type); + + wasm_runtime_free(export_type); +} + +const wasm_byte_vec_t * +wasm_exporttype_name(const wasm_exporttype_t *export_type) +{ + if (!export_type) { + return NULL; + } + return export_type->name; +} + +const wasm_externtype_t * +wasm_exporttype_type(const wasm_exporttype_t *export_type) +{ + if (!export_type) { + return NULL; + } + return export_type->extern_type; +} + +/* Runtime Objects */ +void +wasm_val_delete(wasm_val_t *v) +{ + if (v) + wasm_runtime_free(v); +} + +void +wasm_val_copy(wasm_val_t *out, const wasm_val_t *src) +{ + if (!out || !src) { + return; + } + + bh_memcpy_s(out, sizeof(wasm_val_t), src, sizeof(wasm_val_t)); +} + +bool +rt_val_to_wasm_val(const uint8 *data, uint8 val_type_rt, wasm_val_t *out) +{ + bool ret = true; + switch (val_type_rt) { + case VALUE_TYPE_I32: + out->kind = WASM_I32; + out->of.i32 = *((int32 *)data); + break; + case VALUE_TYPE_F32: + out->kind = WASM_F32; + out->of.f32 = *((float32 *)data); + break; + case VALUE_TYPE_I64: + out->kind = WASM_I64; + out->of.i64 = *((int64 *)data); + break; + case VALUE_TYPE_F64: + out->kind = WASM_F64; + out->of.f64 = *((float64 *)data); + break; + case VALUE_TYPE_V128: + bh_assert(0); + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + out->kind = WASM_EXTERNREF; + if (NULL_REF == *(uint32 *)data) { + out->of.ref = NULL; + } + else { + ret = wasm_externref_ref2obj(*(uint32 *)data, + (void **)&out->of.ref); + } + break; +#endif + default: + LOG_WARNING("unexpected value type %d", val_type_rt); + ret = false; + } + return ret; +} + +bool +wasm_val_to_rt_val(WASMModuleInstanceCommon *inst_comm_rt, uint8 val_type_rt, + const wasm_val_t *v, uint8 *data) +{ + bool ret = true; + switch (val_type_rt) { + case VALUE_TYPE_I32: + bh_assert(WASM_I32 == v->kind); + *((int32 *)data) = v->of.i32; + break; + case VALUE_TYPE_F32: + bh_assert(WASM_F32 == v->kind); + *((float32 *)data) = v->of.f32; + break; + case VALUE_TYPE_I64: + bh_assert(WASM_I64 == v->kind); + *((int64 *)data) = v->of.i64; + break; + case VALUE_TYPE_F64: + bh_assert(WASM_F64 == v->kind); + *((float64 *)data) = v->of.f64; + break; + case VALUE_TYPE_V128: + bh_assert(0); + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + bh_assert(WASM_EXTERNREF == v->kind); + ret = + wasm_externref_obj2ref(inst_comm_rt, v->of.ref, (uint32 *)data); + break; +#endif + default: + LOG_WARNING("unexpected value type %d", val_type_rt); + ret = false; + break; + } + + (void)inst_comm_rt; + return ret; +} + +wasm_ref_t * +wasm_ref_new_internal(wasm_store_t *store, enum wasm_reference_kind kind, + uint32 ref_idx_rt, WASMModuleInstanceCommon *inst_comm_rt) +{ + wasm_ref_t *ref; + + if (!store) { + return NULL; + } + + if (!(ref = malloc_internal(sizeof(wasm_ref_t)))) { + return NULL; + } + + ref->store = store; + ref->kind = kind; + ref->ref_idx_rt = ref_idx_rt; + ref->inst_comm_rt = inst_comm_rt; + + /* workaround */ + if (WASM_REF_foreign == kind) { + wasm_foreign_t *foreign; + + if (!(bh_vector_get(ref->store->foreigns, ref->ref_idx_rt, &foreign)) + || !foreign) { + wasm_runtime_free(ref); + return NULL; + } + + foreign->ref_cnt++; + } + /* others doesn't include ref counters */ + + return ref; +} + +own wasm_ref_t * +wasm_ref_copy(const wasm_ref_t *src) +{ + if (!src) + return NULL; + + /* host_info are different in wasm_ref_t(s) */ + return wasm_ref_new_internal(src->store, src->kind, src->ref_idx_rt, + src->inst_comm_rt); +} + +#define DELETE_HOST_INFO(obj) \ + if (obj->host_info.info) { \ + if (obj->host_info.finalizer) { \ + obj->host_info.finalizer(obj->host_info.info); \ + } \ + } + +void +wasm_ref_delete(own wasm_ref_t *ref) +{ + if (!ref || !ref->store) + return; + + DELETE_HOST_INFO(ref); + + if (WASM_REF_foreign == ref->kind) { + wasm_foreign_t *foreign = NULL; + + if (bh_vector_get(ref->store->foreigns, ref->ref_idx_rt, &foreign) + && foreign) { + wasm_foreign_delete(foreign); + } + } + + wasm_runtime_free(ref); +} + +#define WASM_DEFINE_REF_BASE(name) \ + bool wasm_##name##_same(const wasm_##name##_t *o1, \ + const wasm_##name##_t *o2) \ + { \ + return (!o1 && !o2) ? true \ + : (!o1 || !o2) ? false \ + : (o1->kind != o2->kind) \ + ? false \ + : o1->name##_idx_rt == o2->name##_idx_rt; \ + } \ + \ + void *wasm_##name##_get_host_info(const wasm_##name##_t *obj) \ + { \ + return obj ? obj->host_info.info : NULL; \ + } \ + \ + void wasm_##name##_set_host_info(wasm_##name##_t *obj, void *host_info) \ + { \ + if (obj) { \ + obj->host_info.info = host_info; \ + obj->host_info.finalizer = NULL; \ + } \ + } \ + \ + void wasm_##name##_set_host_info_with_finalizer( \ + wasm_##name##_t *obj, void *host_info, void (*finalizer)(void *)) \ + { \ + if (obj) { \ + obj->host_info.info = host_info; \ + obj->host_info.finalizer = finalizer; \ + } \ + } + +#define WASM_DEFINE_REF(name) \ + WASM_DEFINE_REF_BASE(name) \ + \ + wasm_ref_t *wasm_##name##_as_ref(wasm_##name##_t *name) \ + { \ + if (!name) { \ + return NULL; \ + } \ + \ + return wasm_ref_new_internal(name->store, WASM_REF_##name, \ + name->name##_idx_rt, name->inst_comm_rt); \ + } \ + \ + const wasm_ref_t *wasm_##name##_as_ref_const(const wasm_##name##_t *name) \ + { \ + if (!name) { \ + return NULL; \ + } \ + \ + return wasm_ref_new_internal(name->store, WASM_REF_##name, \ + name->name##_idx_rt, name->inst_comm_rt); \ + } \ + \ + wasm_##name##_t *wasm_ref_as_##name(wasm_ref_t *ref) \ + { \ + if (!ref || WASM_REF_##name != ref->kind) { \ + return NULL; \ + } \ + \ + return wasm_##name##_new_internal(ref->store, ref->ref_idx_rt, \ + ref->inst_comm_rt); \ + } \ + \ + const wasm_##name##_t *wasm_ref_as_##name##_const(const wasm_ref_t *ref) \ + { \ + if (!ref || WASM_REF_##name != ref->kind) { \ + return NULL; \ + } \ + \ + return wasm_##name##_new_internal(ref->store, ref->ref_idx_rt, \ + ref->inst_comm_rt); \ + } + +WASM_DEFINE_REF_BASE(ref) +WASM_DEFINE_REF(foreign) +WASM_DEFINE_REF(func) +WASM_DEFINE_REF(global) +WASM_DEFINE_REF(memory) +WASM_DEFINE_REF(table) + +static wasm_frame_t * +wasm_frame_new(wasm_instance_t *instance, size_t module_offset, + uint32 func_index, size_t func_offset) +{ + wasm_frame_t *frame; + + if (!(frame = malloc_internal(sizeof(wasm_frame_t)))) { + return NULL; + } + + frame->instance = instance; + frame->module_offset = (uint32)module_offset; + frame->func_index = func_index; + frame->func_offset = (uint32)func_offset; + return frame; +} + +own wasm_frame_t * +wasm_frame_copy(const wasm_frame_t *src) +{ + if (!src) { + return NULL; + } + + return wasm_frame_new(src->instance, src->module_offset, src->func_index, + src->func_offset); +} + +void +wasm_frame_delete(own wasm_frame_t *frame) +{ + if (frame) { + wasm_runtime_free(frame); + } +} + +struct wasm_instance_t * +wasm_frame_instance(const wasm_frame_t *frame) +{ + return frame ? frame->instance : NULL; +} + +size_t +wasm_frame_module_offset(const wasm_frame_t *frame) +{ + return frame ? frame->module_offset : 0; +} + +uint32_t +wasm_frame_func_index(const wasm_frame_t *frame) +{ + return frame ? frame->func_index : 0; +} + +size_t +wasm_frame_func_offset(const wasm_frame_t *frame) +{ + return frame ? frame->func_offset : 0; +} + +void +wasm_frame_vec_clone_internal(Vector *src, Vector *out) +{ + if (src->num_elems == 0) { + bh_vector_destroy(out); + return; + } + + if (!bh_vector_destroy(out) + || !bh_vector_init(out, src->num_elems, sizeof(WASMCApiFrame), false)) { + return; + } + + bh_memcpy_s(out->data, (uint32)(src->num_elems * sizeof(WASMCApiFrame)), + src->data, (uint32)(src->num_elems * sizeof(WASMCApiFrame))); + out->num_elems = src->num_elems; +} + +static wasm_trap_t * +wasm_trap_new_internal(wasm_store_t *store, + WASMModuleInstanceCommon *inst_comm_rt, + const char *error_info, Vector *cluster_frames) +{ + wasm_trap_t *trap; +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + wasm_instance_vec_t *instances; + wasm_instance_t *frame_instance = NULL; + uint32 i; +#endif + + if (!singleton_engine) + return NULL; + + if (!(trap = malloc_internal(sizeof(wasm_trap_t)))) { + return NULL; + } + + /* fill in message */ + if (error_info && strlen(error_info) > 0) { + if (!(trap->message = malloc_internal(sizeof(wasm_byte_vec_t)))) { + goto failed; + } + + wasm_name_new_from_string_nt(trap->message, error_info); + if (!trap->message->data) { + goto failed; + } + } + + /* fill in frames */ +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + trap->frames = cluster_frames + ? cluster_frames + : ((WASMModuleInstance *)inst_comm_rt)->frames; + + if (trap->frames) { + /* fill in instances */ + instances = store->instances; + bh_assert(instances != NULL); + + for (i = 0; i < instances->num_elems; i++) { + if (instances->data[i]->inst_comm_rt == inst_comm_rt) { + frame_instance = instances->data[i]; + break; + } + } + + for (i = 0; i < trap->frames->num_elems; i++) { + (((wasm_frame_t *)trap->frames->data) + i)->instance = + frame_instance; + } + } +#else + (void)store; + (void)inst_comm_rt; +#endif /* WASM_ENABLE_DUMP_CALL_STACK != 0 */ + + return trap; +failed: + wasm_trap_delete(trap); + return NULL; +} + +wasm_trap_t * +wasm_trap_new(wasm_store_t *store, const wasm_message_t *message) +{ + wasm_trap_t *trap; + + if (!store) { + return NULL; + } + + if (!(trap = malloc_internal(sizeof(wasm_trap_t)))) { + return NULL; + } + + if (message) { + INIT_VEC(trap->message, wasm_byte_vec_new, message->size, + message->data); + } + + return trap; +failed: + wasm_trap_delete(trap); + return NULL; +} + +void +wasm_trap_delete(wasm_trap_t *trap) +{ + if (!trap) { + return; + } + + DEINIT_VEC(trap->message, wasm_byte_vec_delete); + /* reuse frames of WASMModuleInstance, do not free it here */ + + wasm_runtime_free(trap); +} + +void +wasm_trap_message(const wasm_trap_t *trap, own wasm_message_t *out) +{ + if (!trap || !out) { + return; + } + + wasm_byte_vec_copy(out, trap->message); +} + +own wasm_frame_t * +wasm_trap_origin(const wasm_trap_t *trap) +{ + wasm_frame_t *latest_frame; + + if (!trap || !trap->frames || !trap->frames->num_elems) { + return NULL; + } + + /* first frame is the latest frame */ + latest_frame = (wasm_frame_t *)trap->frames->data; + return wasm_frame_copy(latest_frame); +} + +void +wasm_trap_trace(const wasm_trap_t *trap, own wasm_frame_vec_t *out) +{ + uint32 i; + + if (!trap || !out) { + return; + } + + if (!trap->frames || !trap->frames->num_elems) { + wasm_frame_vec_new_empty(out); + return; + } + + wasm_frame_vec_new_uninitialized(out, trap->frames->num_elems); + if (out->size == 0 || !out->data) { + return; + } + + for (i = 0; i < trap->frames->num_elems; i++) { + wasm_frame_t *frame = ((wasm_frame_t *)trap->frames->data) + i; + if (!(out->data[i] = + wasm_frame_new(frame->instance, frame->module_offset, + frame->func_index, frame->func_offset))) { + goto failed; + } + out->num_elems++; + } + + return; +failed: + for (i = 0; i < out->num_elems; i++) { + if (out->data[i]) { + wasm_runtime_free(out->data[i]); + } + } + + wasm_runtime_free(out->data); +} + +wasm_foreign_t * +wasm_foreign_new_internal(wasm_store_t *store, uint32 foreign_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt) +{ + wasm_foreign_t *foreign = NULL; + + if (!store || !store->foreigns) + return NULL; + + if (!(bh_vector_get(store->foreigns, foreign_idx_rt, &foreign)) + || !foreign) { + return NULL; + } + + foreign->ref_cnt++; + (void)inst_comm_rt; + return foreign; +} + +own wasm_foreign_t * +wasm_foreign_new(wasm_store_t *store) +{ + wasm_foreign_t *foreign; + + if (!store) + return NULL; + + if (!(foreign = malloc_internal(sizeof(wasm_foreign_t)))) + return NULL; + + foreign->store = store; + foreign->kind = WASM_REF_foreign; + foreign->foreign_idx_rt = (uint32)bh_vector_size(store->foreigns); + if (!(bh_vector_append(store->foreigns, &foreign))) { + wasm_runtime_free(foreign); + return NULL; + } + + return foreign; +} + +void +wasm_foreign_delete(wasm_foreign_t *foreign) +{ + if (!foreign) + return; + + if (foreign->ref_cnt < 1) { + return; + } + + foreign->ref_cnt--; + if (!foreign->ref_cnt) { + wasm_runtime_free(foreign); + } +} + +static inline wasm_module_t * +module_ext_to_module(wasm_module_ex_t *module_ex) +{ + return (wasm_module_t *)module_ex; +} + +static inline wasm_module_ex_t * +module_to_module_ext(wasm_module_t *module) +{ + return (wasm_module_ex_t *)module; +} + +#if WASM_ENABLE_INTERP != 0 +#define MODULE_INTERP(module_comm) ((WASMModule *)(*module_comm)) +#endif + +#if WASM_ENABLE_AOT != 0 +#define MODULE_AOT(module_comm) ((AOTModule *)(*module_comm)) +#endif + +#if WASM_ENABLE_WASM_CACHE != 0 +static wasm_module_ex_t * +check_loaded_module(Vector *modules, char *binary_hash) +{ + unsigned i; + wasm_module_ex_t *module = NULL; + + for (i = 0; i < modules->num_elems; i++) { + bh_vector_get(modules, i, &module); + if (!module) { + LOG_ERROR("Unexpected failure at %d\n", __LINE__); + return NULL; + } + + if (!module->ref_count) + /* deleted */ + continue; + + if (memcmp(module->hash, binary_hash, SHA256_DIGEST_LENGTH) == 0) + return module; + } + return NULL; +} + +static wasm_module_ex_t * +try_reuse_loaded_module(wasm_store_t *store, char *binary_hash) +{ + wasm_module_ex_t *cached = NULL; + wasm_module_ex_t *ret = NULL; + + cached = check_loaded_module(&singleton_engine->modules, binary_hash); + if (!cached) + goto quit; + + os_mutex_lock(&cached->lock); + if (!cached->ref_count) + goto unlock; + + if (!bh_vector_append((Vector *)store->modules, &cached)) + goto unlock; + + cached->ref_count += 1; + ret = cached; + +unlock: + os_mutex_unlock(&cached->lock); +quit: + return ret; +} +#endif /* WASM_ENABLE_WASM_CACHE != 0 */ + +wasm_module_t * +wasm_module_new_ex(wasm_store_t *store, wasm_byte_vec_t *binary, LoadArgs *args) +{ + char error_buf[128] = { 0 }; + wasm_module_ex_t *module_ex = NULL; +#if WASM_ENABLE_WASM_CACHE != 0 + char binary_hash[SHA256_DIGEST_LENGTH] = { 0 }; +#endif + + bh_assert(singleton_engine); + + if (!store || !binary || binary->size == 0 || binary->size > UINT32_MAX) + goto quit; + + /* whether the combination of compilation flags are compatible with the + * package type */ + { + PackageType pkg_type; + pkg_type = + get_package_type((uint8 *)binary->data, (uint32)binary->size); + bool result = false; +#if WASM_ENABLE_INTERP != 0 + result = (pkg_type == Wasm_Module_Bytecode); +#endif + +#if WASM_ENABLE_AOT != 0 + result = result || (pkg_type == Wasm_Module_AoT); +#endif + if (!result) { + LOG_VERBOSE("current building isn't compatible with the module," + "may need recompile"); + goto quit; + } + } + +#if WASM_ENABLE_WASM_CACHE != 0 + /* if cached */ + SHA256((void *)binary->data, binary->num_elems, (uint8_t *)binary_hash); + module_ex = try_reuse_loaded_module(store, binary_hash); + if (module_ex) + return module_ext_to_module(module_ex); +#endif + + WASM_C_DUMP_PROC_MEM(); + + module_ex = malloc_internal(sizeof(wasm_module_ex_t)); + if (!module_ex) + goto quit; + + module_ex->is_binary_cloned = args->clone_wasm_binary; + if (args->clone_wasm_binary) { + module_ex->binary = malloc_internal(sizeof(wasm_byte_vec_t)); + if (!module_ex->binary) + goto free_module; + + wasm_byte_vec_copy(module_ex->binary, binary); + if (!module_ex->binary->data) + goto free_binary; + } + else { + module_ex->binary = binary; + } + + args->wasm_binary_freeable = !args->clone_wasm_binary; + module_ex->module_comm_rt = wasm_runtime_load_ex( + (uint8 *)module_ex->binary->data, (uint32)module_ex->binary->size, args, + error_buf, (uint32)sizeof(error_buf)); + if (!(module_ex->module_comm_rt)) { + LOG_ERROR("%s", error_buf); + goto free_vec; + } + + /* append it to a watching list in store */ + if (!bh_vector_append((Vector *)store->modules, &module_ex)) + goto unload; + + if (os_mutex_init(&module_ex->lock) != BHT_OK) + goto remove_last; + + if (!bh_vector_append(&singleton_engine->modules, &module_ex)) + goto destroy_lock; + +#if WASM_ENABLE_WASM_CACHE != 0 + bh_memcpy_s(module_ex->hash, sizeof(module_ex->hash), binary_hash, + sizeof(binary_hash)); +#endif + + module_ex->ref_count = 1; + + WASM_C_DUMP_PROC_MEM(); + + return module_ext_to_module(module_ex); + +destroy_lock: + os_mutex_destroy(&module_ex->lock); +remove_last: + bh_vector_remove((Vector *)store->modules, + (uint32)(store->modules->num_elems - 1), NULL); +unload: + wasm_runtime_unload(module_ex->module_comm_rt); +free_vec: + if (args->clone_wasm_binary) + wasm_byte_vec_delete(module_ex->binary); +free_binary: + if (args->clone_wasm_binary) + wasm_runtime_free(module_ex->binary); +free_module: + wasm_runtime_free(module_ex); +quit: + LOG_ERROR("%s failed", __FUNCTION__); + return NULL; +} + +wasm_module_t * +wasm_module_new(wasm_store_t *store, const wasm_byte_vec_t *binary) +{ + LoadArgs args = { 0 }; + args.name = ""; + args.clone_wasm_binary = true; + return wasm_module_new_ex(store, (wasm_byte_vec_t *)binary, &args); +} + +bool +wasm_module_validate(wasm_store_t *store, const wasm_byte_vec_t *binary) +{ + wasm_byte_vec_t local_binary = { 0 }; + struct WASMModuleCommon *module_rt; + char error_buf[128] = { 0 }; + bool ret; + + bh_assert(singleton_engine); + + if (!store || !binary || binary->size > UINT32_MAX) { + LOG_ERROR("%s failed", __FUNCTION__); + return false; + } + + /* make a copy of binary */ + wasm_byte_vec_copy(&local_binary, binary); + + if (binary->size && !local_binary.data) + return false; + + module_rt = wasm_runtime_load((uint8 *)local_binary.data, + (uint32)local_binary.size, error_buf, 128); + wasm_byte_vec_delete(&local_binary); + if (module_rt) { + wasm_runtime_unload(module_rt); + ret = true; + } + else { + ret = false; + LOG_VERBOSE("%s", error_buf); + } + + return ret; +} + +static void +wasm_module_delete_internal(wasm_module_t *module) +{ + wasm_module_ex_t *module_ex; + + if (!module) { + return; + } + + module_ex = module_to_module_ext(module); + + os_mutex_lock(&module_ex->lock); + + /* N -> N-1 -> 0 -> UINT32_MAX */ + module_ex->ref_count--; + if (module_ex->ref_count > 0) { + os_mutex_unlock(&module_ex->lock); + return; + } + + if (module_ex->is_binary_cloned) + DEINIT_VEC(module_ex->binary, wasm_byte_vec_delete); + + if (module_ex->module_comm_rt) { + wasm_runtime_unload(module_ex->module_comm_rt); + module_ex->module_comm_rt = NULL; + } + +#if WASM_ENABLE_WASM_CACHE != 0 + memset(module_ex->hash, 0, sizeof(module_ex->hash)); +#endif + + os_mutex_unlock(&module_ex->lock); +} + +void +wasm_module_delete(wasm_module_t *module) +{ + /* the module will be released when releasing the store */ + (void)module; +} + +void +wasm_module_imports(const wasm_module_t *module, own wasm_importtype_vec_t *out) +{ + uint32 i, import_func_count = 0, import_memory_count = 0, + import_global_count = 0, import_table_count = 0, import_count = 0; + wasm_byte_vec_t module_name = { 0 }, name = { 0 }; + wasm_externtype_t *extern_type = NULL; + wasm_importtype_t *import_type = NULL; + + if (!module || !out) { + return; + } + + if (((const wasm_module_ex_t *)(module))->ref_count == 0) + return; + +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + import_func_count = MODULE_INTERP(module)->import_function_count; + import_global_count = MODULE_INTERP(module)->import_global_count; + import_memory_count = MODULE_INTERP(module)->import_memory_count; + import_table_count = MODULE_INTERP(module)->import_table_count; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + import_func_count = MODULE_AOT(module)->import_func_count; + import_global_count = MODULE_AOT(module)->import_global_count; + import_memory_count = MODULE_AOT(module)->import_memory_count; + import_table_count = MODULE_AOT(module)->import_table_count; + } +#endif + + import_count = import_func_count + import_global_count + import_table_count + + import_memory_count; + + wasm_importtype_vec_new_uninitialized(out, import_count); + /* + * a wrong combination of module filetype and compilation flags + * also leads to below branch + */ + if (!out->data) { + return; + } + + for (i = 0; i != import_count; ++i) { + char *module_name_rt = NULL, *field_name_rt = NULL; + + memset(&module_name, 0, sizeof(wasm_val_vec_t)); + memset(&name, 0, sizeof(wasm_val_vec_t)); + extern_type = NULL; + import_type = NULL; + + if (i < import_func_count) { + wasm_functype_t *type = NULL; + WASMFuncType *type_rt = NULL; + +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + WASMImport *import = + MODULE_INTERP(module)->import_functions + i; + module_name_rt = import->u.names.module_name; + field_name_rt = import->u.names.field_name; + type_rt = import->u.function.func_type; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + AOTImportFunc *import = MODULE_AOT(module)->import_funcs + i; + module_name_rt = import->module_name; + field_name_rt = import->func_name; + type_rt = import->func_type; + } +#endif + + if (!module_name_rt || !field_name_rt || !type_rt) { + continue; + } + + if (!(type = wasm_functype_new_internal(type_rt))) { + goto failed; + } + + extern_type = wasm_functype_as_externtype(type); + } + else if (i < import_func_count + import_global_count) { + wasm_globaltype_t *type = NULL; + uint8 val_type_rt = 0; + bool mutability_rt = 0; + +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + WASMImport *import = MODULE_INTERP(module)->import_globals + + (i - import_func_count); + module_name_rt = import->u.names.module_name; + field_name_rt = import->u.names.field_name; + val_type_rt = import->u.global.type.val_type; + mutability_rt = import->u.global.type.is_mutable; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + AOTImportGlobal *import = MODULE_AOT(module)->import_globals + + (i - import_func_count); + module_name_rt = import->module_name; + field_name_rt = import->global_name; + val_type_rt = import->type.val_type; + mutability_rt = import->type.is_mutable; + } +#endif + + if (!module_name_rt || !field_name_rt) { + continue; + } + + if (!(type = wasm_globaltype_new_internal(val_type_rt, + mutability_rt))) { + goto failed; + } + + extern_type = wasm_globaltype_as_externtype(type); + } + else if (i < import_func_count + import_global_count + + import_memory_count) { + wasm_memorytype_t *type = NULL; + uint32 min_page = 0, max_page = 0; + +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + WASMImport *import = + MODULE_INTERP(module)->import_memories + + (i - import_func_count - import_global_count); + module_name_rt = import->u.names.module_name; + field_name_rt = import->u.names.field_name; + min_page = import->u.memory.mem_type.init_page_count; + max_page = import->u.memory.mem_type.max_page_count; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + AOTImportMemory *import = + MODULE_AOT(module)->import_memories + + (i - import_func_count - import_global_count); + module_name_rt = import->module_name; + field_name_rt = import->memory_name; + min_page = import->mem_type.init_page_count; + max_page = import->mem_type.max_page_count; + } +#endif + + if (!module_name_rt || !field_name_rt) { + continue; + } + + if (!(type = wasm_memorytype_new_internal(min_page, max_page))) { + goto failed; + } + + extern_type = wasm_memorytype_as_externtype(type); + } + else { + wasm_tabletype_t *type = NULL; + uint8 elem_type_rt = 0; + uint32 min_size = 0, max_size = 0; + +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + WASMImport *import = + MODULE_INTERP(module)->import_tables + + (i - import_func_count - import_global_count + - import_memory_count); + module_name_rt = import->u.names.module_name; + field_name_rt = import->u.names.field_name; + elem_type_rt = import->u.table.table_type.elem_type; + min_size = import->u.table.table_type.init_size; + max_size = import->u.table.table_type.max_size; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + AOTImportTable *import = + MODULE_AOT(module)->import_tables + + (i - import_func_count - import_global_count + - import_memory_count); + module_name_rt = import->module_name; + field_name_rt = import->table_name; + elem_type_rt = import->table_type.elem_type; + min_size = import->table_type.init_size; + max_size = import->table_type.max_size; + } +#endif + + if (!module_name_rt || !field_name_rt) { + continue; + } + + if (!(type = wasm_tabletype_new_internal(elem_type_rt, min_size, + max_size))) { + goto failed; + } + + extern_type = wasm_tabletype_as_externtype(type); + } + + bh_assert(extern_type); + + wasm_name_new_from_string_nt(&module_name, module_name_rt); + if (strlen(module_name_rt) && !module_name.data) { + goto failed; + } + + wasm_name_new_from_string_nt(&name, field_name_rt); + if (strlen(field_name_rt) && !name.data) { + goto failed; + } + + if (!(import_type = + wasm_importtype_new(&module_name, &name, extern_type))) { + goto failed; + } + + if (!bh_vector_append((Vector *)out, &import_type)) { + goto failed_importtype_new; + } + + continue; + + failed: + wasm_byte_vec_delete(&module_name); + wasm_byte_vec_delete(&name); + wasm_externtype_delete(extern_type); + failed_importtype_new: + wasm_importtype_delete(import_type); + } +} + +void +wasm_module_exports(const wasm_module_t *module, wasm_exporttype_vec_t *out) +{ + uint32 i, export_count = 0; + wasm_byte_vec_t name = { 0 }; + wasm_externtype_t *extern_type = NULL; + wasm_exporttype_t *export_type = NULL; + + if (!module || !out) { + return; + } + + if (((const wasm_module_ex_t *)(module))->ref_count == 0) + return; + +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + export_count = MODULE_INTERP(module)->export_count; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + export_count = MODULE_AOT(module)->export_count; + } +#endif + + wasm_exporttype_vec_new_uninitialized(out, export_count); + /* + * a wrong combination of module filetype and compilation flags + * also leads to below branch + */ + if (!out->data) { + return; + } + + for (i = 0; i != export_count; i++) { + WASMExport *export = NULL; +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + export = MODULE_INTERP(module)->exports + i; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + export = MODULE_AOT(module)->exports + i; + } +#endif + + if (!export) { + continue; + } + + /* byte* -> wasm_byte_vec_t */ + wasm_name_new_from_string_nt(&name, export->name); + if (strlen(export->name) && !name.data) { + goto failed; + } + + /* WASMExport -> (WASMFuncType, (uint8, bool)) -> (wasm_functype_t, + * wasm_globaltype_t) -> wasm_externtype_t*/ + switch (export->kind) { + case EXPORT_KIND_FUNC: + { + wasm_functype_t *type = NULL; + WASMFuncType *type_rt; + + if (!wasm_runtime_get_export_func_type(*module, export, + &type_rt)) { + goto failed; + } + + if (!(type = wasm_functype_new_internal(type_rt))) { + goto failed; + } + + extern_type = wasm_functype_as_externtype(type); + break; + } + case EXPORT_KIND_GLOBAL: + { + wasm_globaltype_t *type = NULL; + uint8 val_type_rt = 0; + bool mutability_rt = 0; + + if (!wasm_runtime_get_export_global_type( + *module, export, &val_type_rt, &mutability_rt)) { + goto failed; + } + + if (!(type = wasm_globaltype_new_internal(val_type_rt, + mutability_rt))) { + goto failed; + } + + extern_type = wasm_globaltype_as_externtype(type); + break; + } + case EXPORT_KIND_MEMORY: + { + wasm_memorytype_t *type = NULL; + uint32 min_page = 0, max_page = 0; + + if (!wasm_runtime_get_export_memory_type( + *module, export, &min_page, &max_page)) { + goto failed; + } + + if (!(type = + wasm_memorytype_new_internal(min_page, max_page))) { + goto failed; + } + + extern_type = wasm_memorytype_as_externtype(type); + break; + } + case EXPORT_KIND_TABLE: + { + wasm_tabletype_t *type = NULL; + uint8 elem_type_rt = 0; +#if WASM_ENABLE_GC != 0 + WASMRefType *elem_ref_type_rt; +#endif + uint32 min_size = 0, max_size = 0; + + if (!wasm_runtime_get_export_table_type(*module, export, + &elem_type_rt, +#if WASM_ENABLE_GC != 0 + &elem_ref_type_rt, +#endif + &min_size, &max_size)) { + goto failed; + } +#if WASM_ENABLE_GC != 0 + (void)elem_ref_type_rt; /* TODO */ +#endif + + if (!(type = wasm_tabletype_new_internal(elem_type_rt, min_size, + max_size))) { + goto failed; + } + + extern_type = wasm_tabletype_as_externtype(type); + break; + } + default: + { + LOG_WARNING("%s meets unsupported type %u", __FUNCTION__, + export->kind); + break; + } + } + + if (!(export_type = wasm_exporttype_new(&name, extern_type))) { + goto failed; + } + + if (!(bh_vector_append((Vector *)out, &export_type))) { + goto failed_exporttype_new; + } + } + + return; + +failed: + wasm_byte_vec_delete(&name); + wasm_externtype_delete(extern_type); +failed_exporttype_new: + wasm_exporttype_delete(export_type); + wasm_exporttype_vec_delete(out); +} + +#if WASM_ENABLE_JIT == 0 || WASM_ENABLE_LAZY_JIT != 0 +void +wasm_module_serialize(wasm_module_t *module, own wasm_byte_vec_t *out) +{ + (void)module; + (void)out; + LOG_ERROR("only supported serialization in JIT with eager compilation"); +} + +own wasm_module_t * +wasm_module_deserialize(wasm_store_t *module, const wasm_byte_vec_t *binary) +{ + (void)module; + (void)binary; + LOG_ERROR("only supported deserialization in JIT with eager compilation"); + return NULL; +} +#else + +extern uint8 * +aot_emit_aot_file_buf(AOTCompContext *comp_ctx, AOTCompData *comp_data, + uint32 *p_aot_file_size); +void +wasm_module_serialize(wasm_module_t *module, own wasm_byte_vec_t *out) +{ + wasm_module_ex_t *module_ex; + AOTCompContext *comp_ctx; + AOTCompData *comp_data; + uint8 *aot_file_buf = NULL; + uint32 aot_file_size = 0; + + if (!module || !out) + return; + + if (((const wasm_module_ex_t *)(module))->ref_count == 0) + return; + + module_ex = module_to_module_ext(module); + comp_ctx = ((WASMModule *)(module_ex->module_comm_rt))->comp_ctx; + comp_data = ((WASMModule *)(module_ex->module_comm_rt))->comp_data; + bh_assert(comp_ctx != NULL && comp_data != NULL); + + aot_file_buf = aot_emit_aot_file_buf(comp_ctx, comp_data, &aot_file_size); + if (!aot_file_buf) + return; + + wasm_byte_vec_new(out, aot_file_size, (wasm_byte_t *)aot_file_buf); + wasm_runtime_free(aot_file_buf); + return; +} + +own wasm_module_t * +wasm_module_deserialize(wasm_store_t *store, const wasm_byte_vec_t *binary) +{ + return wasm_module_new(store, binary); +} +#endif + +wasm_module_t * +wasm_module_obtain(wasm_store_t *store, wasm_shared_module_t *shared_module) +{ + wasm_module_ex_t *module_ex = NULL; + + if (!store || !shared_module) + return NULL; + + module_ex = (wasm_module_ex_t *)shared_module; + + os_mutex_lock(&module_ex->lock); + + /* deleting the module... */ + if (module_ex->ref_count == 0) { + LOG_WARNING("wasm_module_obtain re-enter a module under deleting."); + os_mutex_unlock(&module_ex->lock); + return NULL; + } + + /* add it to a watching list in store */ + if (!bh_vector_append((Vector *)store->modules, &module_ex)) { + os_mutex_unlock(&module_ex->lock); + return NULL; + } + + module_ex->ref_count++; + os_mutex_unlock(&module_ex->lock); + + return (wasm_module_t *)shared_module; +} + +wasm_shared_module_t * +wasm_module_share(wasm_module_t *module) +{ + wasm_module_ex_t *module_ex = NULL; + + if (!module) + return NULL; + + module_ex = (wasm_module_ex_t *)module; + + os_mutex_lock(&module_ex->lock); + + /* deleting the module... */ + if (module_ex->ref_count == 0) { + LOG_WARNING("wasm_module_share re-enter a module under deleting."); + os_mutex_unlock(&module_ex->lock); + return NULL; + } + + module_ex->ref_count++; + + os_mutex_unlock(&module_ex->lock); + + return (wasm_shared_module_t *)module; +} + +void +wasm_shared_module_delete(own wasm_shared_module_t *shared_module) +{ + wasm_module_delete_internal((wasm_module_t *)shared_module); +} + +bool +wasm_module_set_name(wasm_module_t *module, const char *name) +{ + char error_buf[256] = { 0 }; + wasm_module_ex_t *module_ex = NULL; + + if (!module) + return false; + + module_ex = module_to_module_ext(module); + bool ret = wasm_runtime_set_module_name(module_ex->module_comm_rt, name, + error_buf, sizeof(error_buf) - 1); + if (!ret) + LOG_WARNING("set module name failed: %s", error_buf); + return ret; +} + +const char * +wasm_module_get_name(wasm_module_t *module) +{ + wasm_module_ex_t *module_ex = NULL; + if (!module) + return ""; + + module_ex = module_to_module_ext(module); + return wasm_runtime_get_module_name(module_ex->module_comm_rt); +} + +bool +wasm_module_is_underlying_binary_freeable(const wasm_module_t *module) +{ + if (((wasm_module_ex_t *)module)->is_binary_cloned) + return true; + + return wasm_runtime_is_underlying_binary_freeable(*module); +} + +static wasm_func_t * +wasm_func_new_basic(wasm_store_t *store, const wasm_functype_t *type, + wasm_func_callback_t func_callback) +{ + wasm_func_t *func = NULL; + + if (!type) { + goto failed; + } + + if (!(func = malloc_internal(sizeof(wasm_func_t)))) { + goto failed; + } + + func->store = store; + func->kind = WASM_EXTERN_FUNC; + func->func_idx_rt = (uint16)-1; + func->with_env = false; + func->u.cb = func_callback; + + if (!(func->type = wasm_functype_copy(type))) { + goto failed; + } + /* func type's param_count and result_count were checked in + loader and are no larger than UINT16_MAX */ + func->param_count = (uint16)func->type->params->num_elems; + func->result_count = (uint16)func->type->results->num_elems; + + RETURN_OBJ(func, wasm_func_delete) +} + +static wasm_func_t * +wasm_func_new_with_env_basic(wasm_store_t *store, const wasm_functype_t *type, + wasm_func_callback_with_env_t callback, void *env, + void (*finalizer)(void *)) +{ + wasm_func_t *func = NULL; + + if (!type) { + goto failed; + } + + if (!(func = malloc_internal(sizeof(wasm_func_t)))) { + goto failed; + } + + func->store = store; + func->kind = WASM_EXTERN_FUNC; + func->func_idx_rt = (uint16)-1; + func->with_env = true; + func->u.cb_env.cb = callback; + func->u.cb_env.env = env; + func->u.cb_env.finalizer = finalizer; + + if (!(func->type = wasm_functype_copy(type))) { + goto failed; + } + /* func type's param_count and result_count were checked in + loader and are no larger than UINT16_MAX */ + func->param_count = (uint16)func->type->params->num_elems; + func->result_count = (uint16)func->type->results->num_elems; + + RETURN_OBJ(func, wasm_func_delete) +} + +wasm_func_t * +wasm_func_new(wasm_store_t *store, const wasm_functype_t *type, + wasm_func_callback_t callback) +{ + bh_assert(singleton_engine); + if (!callback) { + return NULL; + } + return wasm_func_new_basic(store, type, callback); +} + +wasm_func_t * +wasm_func_new_with_env(wasm_store_t *store, const wasm_functype_t *type, + wasm_func_callback_with_env_t callback, void *env, + void (*finalizer)(void *)) +{ + bh_assert(singleton_engine); + if (!callback) { + return NULL; + } + return wasm_func_new_with_env_basic(store, type, callback, env, finalizer); +} + +wasm_func_t * +wasm_func_new_internal(wasm_store_t *store, uint16 func_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt) +{ + wasm_func_t *func = NULL; + WASMFuncType *type_rt = NULL; + + bh_assert(singleton_engine); + + if (!inst_comm_rt) { + return NULL; + } + + func = malloc_internal(sizeof(wasm_func_t)); + if (!func) { + goto failed; + } + + func->kind = WASM_EXTERN_FUNC; + +#if WASM_ENABLE_INTERP != 0 + if (inst_comm_rt->module_type == Wasm_Module_Bytecode) { + bh_assert(func_idx_rt + < ((WASMModuleInstance *)inst_comm_rt)->e->function_count); + WASMFunctionInstance *func_interp = + ((WASMModuleInstance *)inst_comm_rt)->e->functions + func_idx_rt; + type_rt = func_interp->is_import_func + ? func_interp->u.func_import->func_type + : func_interp->u.func->func_type; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (inst_comm_rt->module_type == Wasm_Module_AoT) { + /* use same index to trace the function type in AOTFuncType **func_types + */ + AOTModule *module_aot = + (AOTModule *)((AOTModuleInstance *)inst_comm_rt)->module; + if (func_idx_rt < module_aot->import_func_count) { + type_rt = (module_aot->import_funcs + func_idx_rt)->func_type; + } + else { + type_rt = + (AOTFuncType *)module_aot + ->types[module_aot->func_type_indexes + [func_idx_rt - module_aot->import_func_count]]; + } + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * also leads to below branch + */ + if (!type_rt) { + goto failed; + } + + func->type = wasm_functype_new_internal(type_rt); + if (!func->type) { + goto failed; + } + /* func type's param_count and result_count were checked in + loader and are no larger than UINT16_MAX */ + func->param_count = (uint16)func->type->params->num_elems; + func->result_count = (uint16)func->type->results->num_elems; + + /* will add name information when processing "exports" */ + func->store = store; + func->module_name = NULL; + func->name = NULL; + func->func_idx_rt = func_idx_rt; + func->inst_comm_rt = inst_comm_rt; + return func; + +failed: + LOG_DEBUG("%s failed", __FUNCTION__); + wasm_func_delete(func); + return NULL; +} + +static wasm_func_t * +wasm_func_new_empty(wasm_store_t *store) +{ + wasm_func_t *func = NULL; + + if (!(func = malloc_internal(sizeof(wasm_func_t)))) + goto failed; + + func->store = store; + func->kind = WASM_EXTERN_FUNC; + + RETURN_OBJ(func, wasm_func_delete) +} + +void +wasm_func_delete(wasm_func_t *func) +{ + if (!func) { + return; + } + + if (func->type) { + wasm_functype_delete(func->type); + func->type = NULL; + } + + if (func->with_env) { + if (func->u.cb_env.finalizer) { + func->u.cb_env.finalizer(func->u.cb_env.env); + func->u.cb_env.finalizer = NULL; + func->u.cb_env.env = NULL; + } + } + + DELETE_HOST_INFO(func) + + wasm_runtime_free(func); +} + +own wasm_func_t * +wasm_func_copy(const wasm_func_t *func) +{ + wasm_func_t *cloned = NULL; + + if (!func) { + return NULL; + } + + if (!(cloned = func->with_env ? wasm_func_new_with_env_basic( + func->store, func->type, func->u.cb_env.cb, + func->u.cb_env.env, func->u.cb_env.finalizer) + : wasm_func_new_basic(func->store, func->type, + func->u.cb))) { + goto failed; + } + + cloned->func_idx_rt = func->func_idx_rt; + cloned->inst_comm_rt = func->inst_comm_rt; + + RETURN_OBJ(cloned, wasm_func_delete) +} + +own wasm_functype_t * +wasm_func_type(const wasm_func_t *func) +{ + if (!func) { + return NULL; + } + return wasm_functype_copy(func->type); +} + +static bool +params_to_argv(const wasm_val_vec_t *params, + const wasm_valtype_vec_t *param_defs, uint32 *argv, + uint32 *ptr_argc) +{ + uint32 *argv_org = argv; + const wasm_val_t *param, *param_end; + + bh_assert(params && params->num_elems && params->size && params->data); + + param = params->data; + param_end = param + param_defs->num_elems; + + for (; param < param_end; param++) { + switch (param->kind) { + case WASM_I32: + case WASM_F32: + *(int32 *)argv = param->of.i32; + argv += 1; + break; + case WASM_I64: + case WASM_F64: + *(int64 *)argv = param->of.i64; + argv += 2; + break; + case WASM_V128: + bh_assert(0); + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case WASM_EXTERNREF: + *(uintptr_t *)argv = (uintptr_t)param->of.ref; + argv += sizeof(uintptr_t) / sizeof(uint32); + break; +#endif + default: + LOG_WARNING("unexpected parameter val type %d", param->kind); + return false; + } + } + + *ptr_argc = (uint32)(argv - argv_org); + return true; +} + +static bool +argv_to_results(const uint32 *argv, const wasm_valtype_vec_t *result_defs, + wasm_val_vec_t *results) +{ + wasm_valtype_t **result_def, **result_def_end; + wasm_val_t *result; + + bh_assert(results && results->size && results->data); + + result_def = result_defs->data; + result_def_end = result_def + result_defs->num_elems; + result = results->data; + + for (; result_def < result_def_end; result_def++, result++) { + result->kind = result_def[0]->kind; + switch (result->kind) { + case WASM_I32: + case WASM_F32: + result->of.i32 = *(int32 *)argv; + argv += 1; + break; + case WASM_I64: + case WASM_F64: + result->of.i64 = *(int64 *)argv; + argv += 2; + break; + case WASM_V128: + bh_assert(0); + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case WASM_EXTERNREF: + case WASM_FUNCREF: + result->of.ref = (struct wasm_ref_t *)(*(uintptr_t *)argv); + argv += sizeof(uintptr_t) / sizeof(uint32); + break; +#endif + default: + LOG_WARNING("%s meets unsupported type: %d", __FUNCTION__, + result->kind); + return false; + } + } + + return true; +} + +wasm_trap_t * +wasm_func_call(const wasm_func_t *func, const wasm_val_vec_t *params, + wasm_val_vec_t *results) +{ + /* parameters count as if all are uint32 */ + /* a int64 or float64 parameter means 2 */ + uint32 argc = 0; + /* a parameter list and a return value list */ + uint32 argv_buf[32] = { 0 }, *argv = argv_buf; + WASMFunctionInstanceCommon *func_comm_rt = NULL; + WASMExecEnv *exec_env = NULL; + size_t param_count, result_count, alloc_count; + Vector *cluster_frames = NULL; + + bh_assert(func && func->type); + + if (!func->inst_comm_rt) { + wasm_name_t message = { 0 }; + wasm_trap_t *trap; + + wasm_name_new_from_string_nt(&message, + "failed to call unlinked function"); + trap = wasm_trap_new(func->store, &message); + wasm_byte_vec_delete(&message); + + return trap; + } + + if (func->inst_comm_rt->module_type == Wasm_Module_Bytecode) { +#if WASM_ENABLE_INTERP != 0 + func_comm_rt = ((WASMModuleInstance *)func->inst_comm_rt)->e->functions + + func->func_idx_rt; +#endif + } + else if (func->inst_comm_rt->module_type == Wasm_Module_AoT) { +#if WASM_ENABLE_AOT != 0 + if (!(func_comm_rt = func->func_comm_rt)) { + AOTModuleInstance *inst_aot = + (AOTModuleInstance *)func->inst_comm_rt; + func_comm_rt = ((wasm_func_t *)func)->func_comm_rt = + aot_lookup_function_with_idx(inst_aot, func->func_idx_rt); + } +#endif + } + + /* + * a wrong combination of module filetype and compilation flags + * also leads to below branch + */ + if (!func_comm_rt) { + goto failed; + } + + param_count = wasm_func_param_arity(func); + result_count = wasm_func_result_arity(func); + + alloc_count = (param_count > result_count) ? param_count : result_count; + if (alloc_count > (size_t)sizeof(argv_buf) / sizeof(uint64)) { + if (!(argv = malloc_internal(sizeof(uint64) * alloc_count))) { + goto failed; + } + } + + /* copy parameters */ + if (param_count + && !params_to_argv(params, wasm_functype_params(func->type), argv, + &argc)) { + goto failed; + } + +#ifdef OS_ENABLE_HW_BOUND_CHECK + exec_env = wasm_runtime_get_exec_env_tls(); +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + if (!exec_env) { + exec_env = wasm_clusters_search_exec_env(func->inst_comm_rt); + } +#endif + if (!exec_env) { + exec_env = wasm_runtime_get_exec_env_singleton(func->inst_comm_rt); + } + if (!exec_env) { + goto failed; + } + + wasm_runtime_set_exception(func->inst_comm_rt, NULL); + if (!wasm_runtime_call_wasm(exec_env, func_comm_rt, argc, argv)) { + if (wasm_runtime_get_exception(func->inst_comm_rt)) { + LOG_DEBUG("%s", wasm_runtime_get_exception(func->inst_comm_rt)); + goto failed; + } + } + + /* copy results */ + if (result_count) { + if (!argv_to_results(argv, wasm_functype_results(func->type), + results)) { + wasm_runtime_set_exception(func->inst_comm_rt, + "argv_to_results failed"); + goto failed; + } + results->num_elems = result_count; + results->size = result_count; + } + + if (argv != argv_buf) + wasm_runtime_free(argv); + return NULL; + +failed: + if (argv != argv_buf) + wasm_runtime_free(argv); + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 && WASM_ENABLE_THREAD_MGR != 0 + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + cluster_frames = &cluster->exception_frames; + wasm_cluster_traverse_lock(exec_env); +#endif + + wasm_trap_t *trap = wasm_trap_new_internal( + func->store, func->inst_comm_rt, + wasm_runtime_get_exception(func->inst_comm_rt), cluster_frames); + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 && WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_traverse_unlock(exec_env); +#endif + return trap; +} + +size_t +wasm_func_param_arity(const wasm_func_t *func) +{ + return func->param_count; +} + +size_t +wasm_func_result_arity(const wasm_func_t *func) +{ + return func->result_count; +} + +wasm_global_t * +wasm_global_new(wasm_store_t *store, const wasm_globaltype_t *global_type, + const wasm_val_t *init) +{ + wasm_global_t *global = NULL; + + bh_assert(singleton_engine); + + if (!global_type || !init) { + goto failed; + } + + global = malloc_internal(sizeof(wasm_global_t)); + if (!global) { + goto failed; + } + + global->store = store; + global->kind = WASM_EXTERN_GLOBAL; + global->type = wasm_globaltype_copy(global_type); + if (!global->type) { + goto failed; + } + + global->init = malloc_internal(sizeof(wasm_val_t)); + if (!global->init) { + goto failed; + } + + wasm_val_copy(global->init, init); + /* TODO: how to check if above is failed */ + + return global; + +failed: + LOG_DEBUG("%s failed", __FUNCTION__); + wasm_global_delete(global); + return NULL; +} + +static wasm_global_t * +wasm_global_new_empty(wasm_store_t *store) +{ + wasm_global_t *global = NULL; + + global = malloc_internal(sizeof(wasm_global_t)); + if (!global) + goto failed; + + global->store = store; + global->kind = WASM_EXTERN_GLOBAL; + + return global; +failed: + LOG_DEBUG("%s failed", __FUNCTION__); + wasm_global_delete(global); + return NULL; +} + +/* almost same with wasm_global_new */ +wasm_global_t * +wasm_global_copy(const wasm_global_t *src) +{ + wasm_global_t *global = NULL; + + if (!src) { + return NULL; + } + + global = malloc_internal(sizeof(wasm_global_t)); + if (!global) { + goto failed; + } + + global->kind = WASM_EXTERN_GLOBAL; + global->type = wasm_globaltype_copy(src->type); + if (!global->type) { + goto failed; + } + + global->init = malloc_internal(sizeof(wasm_val_t)); + if (!global->init) { + goto failed; + } + + wasm_val_copy(global->init, src->init); + + global->global_idx_rt = src->global_idx_rt; + global->inst_comm_rt = src->inst_comm_rt; + + return global; + +failed: + LOG_DEBUG("%s failed", __FUNCTION__); + wasm_global_delete(global); + return NULL; +} + +void +wasm_global_delete(wasm_global_t *global) +{ + if (!global) { + return; + } + + if (global->init) { + wasm_val_delete(global->init); + global->init = NULL; + } + + if (global->type) { + wasm_globaltype_delete(global->type); + global->type = NULL; + } + + DELETE_HOST_INFO(global) + + wasm_runtime_free(global); +} + +#if WASM_ENABLE_INTERP != 0 +static bool +interp_global_set(const WASMModuleInstance *inst_interp, uint16 global_idx_rt, + const wasm_val_t *v) +{ + const WASMGlobalInstance *global_interp = + inst_interp->e->globals + global_idx_rt; + uint8 val_type_rt = global_interp->type; +#if WASM_ENABLE_MULTI_MODULE != 0 + uint8 *data = global_interp->import_global_inst + ? global_interp->import_module_inst->global_data + + global_interp->import_global_inst->data_offset + : inst_interp->global_data + global_interp->data_offset; +#else + uint8 *data = inst_interp->global_data + global_interp->data_offset; +#endif + + return wasm_val_to_rt_val((WASMModuleInstanceCommon *)inst_interp, + val_type_rt, v, data); +} + +static bool +interp_global_get(const WASMModuleInstance *inst_interp, uint16 global_idx_rt, + wasm_val_t *out) +{ + WASMGlobalInstance *global_interp = inst_interp->e->globals + global_idx_rt; + uint8 val_type_rt = global_interp->type; +#if WASM_ENABLE_MULTI_MODULE != 0 + uint8 *data = global_interp->import_global_inst + ? global_interp->import_module_inst->global_data + + global_interp->import_global_inst->data_offset + : inst_interp->global_data + global_interp->data_offset; +#else + uint8 *data = inst_interp->global_data + global_interp->data_offset; +#endif + + return rt_val_to_wasm_val(data, val_type_rt, out); +} +#endif + +#if WASM_ENABLE_AOT != 0 +static bool +aot_global_set(const AOTModuleInstance *inst_aot, uint16 global_idx_rt, + const wasm_val_t *v) +{ + AOTModule *module_aot = (AOTModule *)inst_aot->module; + uint8 val_type_rt = 0; + uint32 data_offset = 0; + void *data = NULL; + + if (global_idx_rt < module_aot->import_global_count) { + data_offset = module_aot->import_globals[global_idx_rt].data_offset; + val_type_rt = module_aot->import_globals[global_idx_rt].type.val_type; + } + else { + data_offset = + module_aot->globals[global_idx_rt - module_aot->import_global_count] + .data_offset; + val_type_rt = + module_aot->globals[global_idx_rt - module_aot->import_global_count] + .type.val_type; + } + + data = (void *)(inst_aot->global_data + data_offset); + return wasm_val_to_rt_val((WASMModuleInstanceCommon *)inst_aot, val_type_rt, + v, data); +} + +static bool +aot_global_get(const AOTModuleInstance *inst_aot, uint16 global_idx_rt, + wasm_val_t *out) +{ + AOTModule *module_aot = (AOTModule *)inst_aot->module; + uint8 val_type_rt = 0; + uint32 data_offset = 0; + uint8 *data = NULL; + + if (global_idx_rt < module_aot->import_global_count) { + data_offset = module_aot->import_globals[global_idx_rt].data_offset; + val_type_rt = module_aot->import_globals[global_idx_rt].type.val_type; + } + else { + data_offset = + module_aot->globals[global_idx_rt - module_aot->import_global_count] + .data_offset; + val_type_rt = + module_aot->globals[global_idx_rt - module_aot->import_global_count] + .type.val_type; + } + + data = inst_aot->global_data + data_offset; + return rt_val_to_wasm_val(data, val_type_rt, out); +} +#endif + +void +wasm_global_set(wasm_global_t *global, const wasm_val_t *v) +{ + if (!global || !v || !global->inst_comm_rt) { + return; + } + +#if WASM_ENABLE_INTERP != 0 + if (global->inst_comm_rt->module_type == Wasm_Module_Bytecode) { + (void)interp_global_set((WASMModuleInstance *)global->inst_comm_rt, + global->global_idx_rt, v); + return; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (global->inst_comm_rt->module_type == Wasm_Module_AoT) { + (void)aot_global_set((AOTModuleInstance *)global->inst_comm_rt, + global->global_idx_rt, v); + return; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + UNREACHABLE(); +} + +void +wasm_global_get(const wasm_global_t *global, wasm_val_t *out) +{ + if (!global || !out) { + return; + } + + if (!global->inst_comm_rt) { + return; + } + + memset(out, 0, sizeof(wasm_val_t)); + +#if WASM_ENABLE_INTERP != 0 + if (global->inst_comm_rt->module_type == Wasm_Module_Bytecode) { + (void)interp_global_get((WASMModuleInstance *)global->inst_comm_rt, + global->global_idx_rt, out); + return; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (global->inst_comm_rt->module_type == Wasm_Module_AoT) { + (void)aot_global_get((AOTModuleInstance *)global->inst_comm_rt, + global->global_idx_rt, out); + return; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + UNREACHABLE(); +} + +wasm_global_t * +wasm_global_new_internal(wasm_store_t *store, uint16 global_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt) +{ + wasm_global_t *global = NULL; + uint8 val_type_rt = 0; + bool is_mutable = 0; + bool init = false; + + bh_assert(singleton_engine); + + if (!inst_comm_rt) { + return NULL; + } + + global = malloc_internal(sizeof(wasm_global_t)); + if (!global) { + goto failed; + } + + global->store = store; + global->kind = WASM_EXTERN_GLOBAL; + +#if WASM_ENABLE_INTERP != 0 + if (inst_comm_rt->module_type == Wasm_Module_Bytecode) { + WASMGlobalInstance *global_interp = + ((WASMModuleInstance *)inst_comm_rt)->e->globals + global_idx_rt; + val_type_rt = global_interp->type; + is_mutable = global_interp->is_mutable; + init = true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (inst_comm_rt->module_type == Wasm_Module_AoT) { + AOTModuleInstance *inst_aot = (AOTModuleInstance *)inst_comm_rt; + AOTModule *module_aot = (AOTModule *)inst_aot->module; + + init = true; + + if (global_idx_rt < module_aot->import_global_count) { + AOTImportGlobal *global_import_aot = + module_aot->import_globals + global_idx_rt; + val_type_rt = global_import_aot->type.val_type; + is_mutable = global_import_aot->type.is_mutable; + } + else { + AOTGlobal *global_aot = + module_aot->globals + + (global_idx_rt - module_aot->import_global_count); + val_type_rt = global_aot->type.val_type; + is_mutable = global_aot->type.is_mutable; + } + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + if (!init) { + goto failed; + } + + global->type = wasm_globaltype_new_internal(val_type_rt, is_mutable); + if (!global->type) { + goto failed; + } + + global->init = malloc_internal(sizeof(wasm_val_t)); + if (!global->init) { + goto failed; + } + +#if WASM_ENABLE_INTERP != 0 + if (inst_comm_rt->module_type == Wasm_Module_Bytecode) { + interp_global_get((WASMModuleInstance *)inst_comm_rt, global_idx_rt, + global->init); + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (inst_comm_rt->module_type == Wasm_Module_AoT) { + aot_global_get((AOTModuleInstance *)inst_comm_rt, global_idx_rt, + global->init); + } +#endif + + global->inst_comm_rt = inst_comm_rt; + global->global_idx_rt = global_idx_rt; + + return global; + +failed: + LOG_DEBUG("%s failed", __FUNCTION__); + wasm_global_delete(global); + return NULL; +} + +wasm_globaltype_t * +wasm_global_type(const wasm_global_t *global) +{ + if (!global) { + return NULL; + } + return wasm_globaltype_copy(global->type); +} + +static wasm_table_t * +wasm_table_new_basic(wasm_store_t *store, const wasm_tabletype_t *type) +{ + wasm_table_t *table = NULL; + + if (!(table = malloc_internal(sizeof(wasm_table_t)))) { + goto failed; + } + + table->store = store; + table->kind = WASM_EXTERN_TABLE; + + if (!(table->type = wasm_tabletype_copy(type))) { + goto failed; + } + + RETURN_OBJ(table, wasm_table_delete); +} + +wasm_table_t * +wasm_table_new_internal(wasm_store_t *store, uint16 table_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt) +{ + wasm_table_t *table = NULL; + uint8 val_type_rt = 0; +#if WASM_ENABLE_GC != 0 + WASMRefType *val_ref_type_rt; +#endif + uint32 init_size = 0, max_size = 0; + + bh_assert(singleton_engine); + + if (!inst_comm_rt) { + return NULL; + } + + if (!(table = malloc_internal(sizeof(wasm_table_t)))) { + goto failed; + } + + table->store = store; + table->kind = WASM_EXTERN_TABLE; + + if (!wasm_runtime_get_table_inst_elem_type(inst_comm_rt, table_idx_rt, + &val_type_rt, +#if WASM_ENABLE_GC != 0 + &val_ref_type_rt, +#endif + &init_size, &max_size)) { + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + goto failed; + } +#if WASM_ENABLE_GC != 0 + (void)val_ref_type_rt; /* TODO */ +#endif + + if (!(table->type = + wasm_tabletype_new_internal(val_type_rt, init_size, max_size))) { + goto failed; + } + + table->inst_comm_rt = inst_comm_rt; + table->table_idx_rt = table_idx_rt; + + RETURN_OBJ(table, wasm_table_delete); +} + +/* will not actually apply this new table into the runtime */ +wasm_table_t * +wasm_table_new(wasm_store_t *store, const wasm_tabletype_t *table_type, + wasm_ref_t *init) +{ + wasm_table_t *table; + (void)init; + + bh_assert(singleton_engine); + + if ((table = wasm_table_new_basic(store, table_type))) { + table->store = store; + } + + return table; +} + +wasm_table_t * +wasm_table_copy(const wasm_table_t *src) +{ + wasm_table_t *table; + + if (!(table = wasm_table_new_basic(src->store, src->type))) { + return NULL; + } + + table->table_idx_rt = src->table_idx_rt; + table->inst_comm_rt = src->inst_comm_rt; + return table; +} + +void +wasm_table_delete(wasm_table_t *table) +{ + if (!table) { + return; + } + + if (table->type) { + wasm_tabletype_delete(table->type); + table->type = NULL; + } + + DELETE_HOST_INFO(table) + + wasm_runtime_free(table); +} + +wasm_tabletype_t * +wasm_table_type(const wasm_table_t *table) +{ + if (!table) { + return NULL; + } + return wasm_tabletype_copy(table->type); +} + +#if WASM_ENABLE_GC == 0 +own wasm_ref_t * +wasm_table_get(const wasm_table_t *table, wasm_table_size_t index) +{ + uint32 ref_idx = NULL_REF; + + if (!table || !table->inst_comm_rt) { + return NULL; + } + +#if WASM_ENABLE_INTERP != 0 + if (table->inst_comm_rt->module_type == Wasm_Module_Bytecode) { + WASMTableInstance *table_interp = + ((WASMModuleInstance *)table->inst_comm_rt) + ->tables[table->table_idx_rt]; + if (index >= table_interp->cur_size) { + return NULL; + } + ref_idx = (uint32)table_interp->elems[index]; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (table->inst_comm_rt->module_type == Wasm_Module_AoT) { + AOTModuleInstance *inst_aot = (AOTModuleInstance *)table->inst_comm_rt; + AOTTableInstance *table_aot = inst_aot->tables[table->table_idx_rt]; + if (index >= table_aot->cur_size) { + return NULL; + } + ref_idx = (uint32)table_aot->elems[index]; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * also leads to below branch + */ + if (ref_idx == NULL_REF) { + return NULL; + } + +#if WASM_ENABLE_REF_TYPES != 0 + if (table->type->val_type->kind == WASM_EXTERNREF) { + void *externref_obj; + if (!wasm_externref_ref2obj(ref_idx, &externref_obj)) { + return NULL; + } + + return externref_obj; + } + else +#endif + { + return wasm_ref_new_internal(table->store, WASM_REF_func, ref_idx, + table->inst_comm_rt); + } +} + +bool +wasm_table_set(wasm_table_t *table, wasm_table_size_t index, + own wasm_ref_t *ref) +{ + uint32 *p_ref_idx = NULL; + uint32 function_count = 0; + + if (!table || !table->inst_comm_rt) { + return false; + } + + if (ref +#if WASM_ENABLE_REF_TYPES != 0 + && !(WASM_REF_foreign == ref->kind + && WASM_EXTERNREF == table->type->val_type->kind) +#endif + && !(WASM_REF_func == ref->kind + && WASM_FUNCREF == table->type->val_type->kind)) { + return false; + } + +#if WASM_ENABLE_INTERP != 0 + if (table->inst_comm_rt->module_type == Wasm_Module_Bytecode) { + WASMTableInstance *table_interp = + ((WASMModuleInstance *)table->inst_comm_rt) + ->tables[table->table_idx_rt]; + + if (index >= table_interp->cur_size) { + return false; + } + + p_ref_idx = (uint32 *)(table_interp->elems + index); + function_count = + ((WASMModuleInstance *)table->inst_comm_rt)->e->function_count; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (table->inst_comm_rt->module_type == Wasm_Module_AoT) { + AOTModuleInstance *inst_aot = (AOTModuleInstance *)table->inst_comm_rt; + AOTModule *module_aot = (AOTModule *)inst_aot->module; + AOTTableInstance *table_aot = inst_aot->tables[table->table_idx_rt]; + + if (index >= table_aot->cur_size) { + return false; + } + + p_ref_idx = (uint32 *)(table_aot->elems + index); + function_count = module_aot->func_count; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + if (!p_ref_idx) { + return false; + } + +#if WASM_ENABLE_REF_TYPES != 0 + if (table->type->val_type->kind == WASM_EXTERNREF) { + return wasm_externref_obj2ref(table->inst_comm_rt, ref, p_ref_idx); + } + else +#endif + { + if (ref) { + if (NULL_REF != ref->ref_idx_rt) { + if (ref->ref_idx_rt >= function_count) { + return false; + } + } + *p_ref_idx = ref->ref_idx_rt; + wasm_ref_delete(ref); + } + else { + *p_ref_idx = NULL_REF; + } + } + + return true; +} +#else /* else of WASM_ENABLE_GC == 0 */ +own wasm_ref_t * +wasm_table_get(const wasm_table_t *table, wasm_table_size_t index) +{ + /* TODO */ + return NULL; +} + +bool +wasm_table_set(wasm_table_t *table, wasm_table_size_t index, + own wasm_ref_t *ref) +{ + /* TODO */ + return false; +} +#endif /* end of WASM_ENABLE_GC == 0 */ + +wasm_table_size_t +wasm_table_size(const wasm_table_t *table) +{ + if (!table || !table->inst_comm_rt) { + return 0; + } + +#if WASM_ENABLE_INTERP != 0 + if (table->inst_comm_rt->module_type == Wasm_Module_Bytecode) { + WASMTableInstance *table_interp = + ((WASMModuleInstance *)table->inst_comm_rt) + ->tables[table->table_idx_rt]; + return table_interp->cur_size; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (table->inst_comm_rt->module_type == Wasm_Module_AoT) { + AOTModuleInstance *inst_aot = (AOTModuleInstance *)table->inst_comm_rt; + AOTModule *module_aot = (AOTModule *)inst_aot->module; + + if (table->table_idx_rt < module_aot->import_table_count) { + AOTImportTable *table_aot = + module_aot->import_tables + table->table_idx_rt; + return table_aot->table_type.init_size; + } + else { + AOTTable *table_aot = + module_aot->tables + + (table->table_idx_rt - module_aot->import_table_count); + return table_aot->table_type.init_size; + } + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + return 0; +} + +bool +wasm_table_grow(wasm_table_t *table, wasm_table_size_t delta, + own wasm_ref_t *init) +{ + (void)table; + (void)delta; + (void)init; + LOG_WARNING("Calling wasm_table_grow() by host is not supported." + "Only allow growing a table via the opcode table.grow"); + return false; +} + +static wasm_memory_t * +wasm_memory_new_basic(wasm_store_t *store, const wasm_memorytype_t *type) +{ + wasm_memory_t *memory = NULL; + + if (!type) { + goto failed; + } + + if (!(memory = malloc_internal(sizeof(wasm_memory_t)))) { + goto failed; + } + + memory->store = store; + memory->kind = WASM_EXTERN_MEMORY; + memory->type = wasm_memorytype_copy(type); + + RETURN_OBJ(memory, wasm_memory_delete) +} + +wasm_memory_t * +wasm_memory_new(wasm_store_t *store, const wasm_memorytype_t *type) +{ + bh_assert(singleton_engine); + return wasm_memory_new_basic(store, type); +} + +wasm_memory_t * +wasm_memory_copy(const wasm_memory_t *src) +{ + wasm_memory_t *dst = NULL; + + if (!src) { + return NULL; + } + + if (!(dst = wasm_memory_new_basic(src->store, src->type))) { + goto failed; + } + + dst->memory_idx_rt = src->memory_idx_rt; + dst->inst_comm_rt = src->inst_comm_rt; + + RETURN_OBJ(dst, wasm_memory_delete) +} + +wasm_memory_t * +wasm_memory_new_internal(wasm_store_t *store, uint16 memory_idx_rt, + WASMModuleInstanceCommon *inst_comm_rt) +{ + wasm_memory_t *memory = NULL; + uint32 min_pages = 0, max_pages = 0; + bool init_flag = false; + + bh_assert(singleton_engine); + + if (!inst_comm_rt) { + return NULL; + } + + if (!(memory = malloc_internal(sizeof(wasm_memory_t)))) { + goto failed; + } + + memory->store = store; + memory->kind = WASM_EXTERN_MEMORY; + +#if WASM_ENABLE_INTERP != 0 + if (inst_comm_rt->module_type == Wasm_Module_Bytecode) { + WASMMemoryInstance *memory_interp = + ((WASMModuleInstance *)inst_comm_rt)->memories[memory_idx_rt]; + min_pages = memory_interp->cur_page_count; + max_pages = memory_interp->max_page_count; + init_flag = true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (inst_comm_rt->module_type == Wasm_Module_AoT) { + AOTModuleInstance *inst_aot = (AOTModuleInstance *)inst_comm_rt; + AOTModule *module_aot = (AOTModule *)inst_aot->module; + + if (memory_idx_rt < module_aot->import_memory_count) { + min_pages = module_aot->import_memories->mem_type.init_page_count; + max_pages = module_aot->import_memories->mem_type.max_page_count; + } + else { + min_pages = module_aot->memories->init_page_count; + max_pages = module_aot->memories->max_page_count; + } + init_flag = true; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + if (!init_flag) { + goto failed; + } + + if (!(memory->type = wasm_memorytype_new_internal(min_pages, max_pages))) { + goto failed; + } + + memory->inst_comm_rt = inst_comm_rt; + memory->memory_idx_rt = memory_idx_rt; + + RETURN_OBJ(memory, wasm_memory_delete); +} + +void +wasm_memory_delete(wasm_memory_t *memory) +{ + if (!memory) { + return; + } + + if (memory->type) { + wasm_memorytype_delete(memory->type); + memory->type = NULL; + } + + DELETE_HOST_INFO(memory) + + wasm_runtime_free(memory); +} + +wasm_memorytype_t * +wasm_memory_type(const wasm_memory_t *memory) +{ + if (!memory) { + return NULL; + } + + return wasm_memorytype_copy(memory->type); +} + +byte_t * +wasm_memory_data(wasm_memory_t *memory) +{ + WASMModuleInstanceCommon *module_inst_comm; + + if (!memory || !memory->inst_comm_rt) { + return NULL; + } + + module_inst_comm = memory->inst_comm_rt; +#if WASM_ENABLE_INTERP != 0 + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *module_inst = + (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst = + module_inst->memories[memory->memory_idx_rt]; + return (byte_t *)memory_inst->memory_data; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst_comm->module_type == Wasm_Module_AoT) { + AOTModuleInstance *module_inst = (AOTModuleInstance *)module_inst_comm; + AOTMemoryInstance *memory_inst = + ((AOTMemoryInstance **) + module_inst->memories)[memory->memory_idx_rt]; + return (byte_t *)memory_inst->memory_data; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + return NULL; +} + +size_t +wasm_memory_data_size(const wasm_memory_t *memory) +{ + WASMModuleInstanceCommon *module_inst_comm; + + if (!memory || !memory->inst_comm_rt) { + return 0; + } + + module_inst_comm = memory->inst_comm_rt; +#if WASM_ENABLE_INTERP != 0 + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *module_inst = + (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst = + module_inst->memories[memory->memory_idx_rt]; + return (size_t)memory_inst->cur_page_count + * memory_inst->num_bytes_per_page; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst_comm->module_type == Wasm_Module_AoT) { + AOTModuleInstance *module_inst = (AOTModuleInstance *)module_inst_comm; + AOTMemoryInstance *memory_inst = + ((AOTMemoryInstance **) + module_inst->memories)[memory->memory_idx_rt]; + return (size_t)memory_inst->cur_page_count + * memory_inst->num_bytes_per_page; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + return 0; +} + +wasm_memory_pages_t +wasm_memory_size(const wasm_memory_t *memory) +{ + WASMModuleInstanceCommon *module_inst_comm; + + if (!memory || !memory->inst_comm_rt) { + return 0; + } + + module_inst_comm = memory->inst_comm_rt; +#if WASM_ENABLE_INTERP != 0 + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *module_inst = + (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst = + module_inst->memories[memory->memory_idx_rt]; + return memory_inst->cur_page_count; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst_comm->module_type == Wasm_Module_AoT) { + AOTModuleInstance *module_inst = (AOTModuleInstance *)module_inst_comm; + AOTMemoryInstance *memory_inst = + ((AOTMemoryInstance **) + module_inst->memories)[memory->memory_idx_rt]; + return memory_inst->cur_page_count; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + return 0; +} + +bool +wasm_memory_grow(wasm_memory_t *memory, wasm_memory_pages_t delta) +{ + (void)memory; + (void)delta; + LOG_WARNING("Calling wasm_memory_grow() by host is not supported." + "Only allow growing a memory via the opcode memory.grow"); + return false; +} + +#if WASM_ENABLE_INTERP != 0 +static bool +interp_link_func(const wasm_instance_t *inst, const WASMModule *module_interp, + uint16 func_idx_rt, wasm_func_t *import) +{ + WASMImport *imported_func_interp = NULL; + + bh_assert(inst && module_interp && import); + bh_assert(func_idx_rt < module_interp->import_function_count); + bh_assert(WASM_EXTERN_FUNC == import->kind); + + imported_func_interp = module_interp->import_functions + func_idx_rt; + bh_assert(imported_func_interp); + bh_assert(imported_func_interp->kind == IMPORT_KIND_FUNC); + + /* it is a placeholder and let's skip it*/ + if (!import->type) + return true; + + /* type comparison */ + if (!wasm_functype_same_internal( + import->type, imported_func_interp->u.function.func_type)) + return false; + + imported_func_interp->u.function.call_conv_wasm_c_api = true; + /* only set func_ptr_linked to avoid unlink warning during instantiation, + func_ptr_linked, with_env and env will be stored in module instance's + c_api_func_imports later and used when calling import function */ + if (import->with_env) + imported_func_interp->u.function.func_ptr_linked = import->u.cb_env.cb; + else + imported_func_interp->u.function.func_ptr_linked = import->u.cb; + bh_assert(imported_func_interp->u.function.func_ptr_linked); + + import->func_idx_rt = func_idx_rt; + + (void)inst; + return true; +} + +static bool +interp_link_global(const WASMModule *module_interp, uint16 global_idx_rt, + wasm_global_t *import) +{ + WASMImport *imported_global_interp = NULL; + + bh_assert(module_interp && import); + bh_assert(global_idx_rt < module_interp->import_global_count); + bh_assert(WASM_EXTERN_GLOBAL == import->kind); + + imported_global_interp = module_interp->import_globals + global_idx_rt; + bh_assert(imported_global_interp); + bh_assert(imported_global_interp->kind == IMPORT_KIND_GLOBAL); + + /* it is a placeholder and let's skip it*/ + if (!import->type) + return true; + + /* type comparison */ + if (!cmp_val_kind_with_val_type( + wasm_valtype_kind(import->type->val_type), + imported_global_interp->u.global.type.val_type)) + return false; + + /* set init value */ + bh_assert(import->init); + switch (wasm_valtype_kind(import->type->val_type)) { + case WASM_I32: + imported_global_interp->u.global.global_data_linked.i32 = + import->init->of.i32; + break; + case WASM_I64: + imported_global_interp->u.global.global_data_linked.i64 = + import->init->of.i64; + break; + case WASM_F32: + imported_global_interp->u.global.global_data_linked.f32 = + import->init->of.f32; + break; + case WASM_F64: + imported_global_interp->u.global.global_data_linked.f64 = + import->init->of.f64; + break; + default: + return false; + } + + import->global_idx_rt = global_idx_rt; + imported_global_interp->u.global.is_linked = true; + return true; +} + +static bool +interp_process_export(wasm_store_t *store, + const WASMModuleInstance *inst_interp, + wasm_extern_vec_t *externals) +{ + WASMExport *exports = NULL; + WASMExport *export = NULL; + wasm_extern_t *external = NULL; + uint32 export_cnt = 0; + uint32 i = 0; + + bh_assert(store && inst_interp && inst_interp->module && externals); + + exports = inst_interp->module->exports; + export_cnt = inst_interp->module->export_count; + + for (i = 0; i < export_cnt; ++i) { + export = exports + i; + + switch (export->kind) { + case EXPORT_KIND_FUNC: + { + wasm_func_t *func; + if (!(func = wasm_func_new_internal( + store, export->index, + (WASMModuleInstanceCommon *)inst_interp))) { + goto failed; + } + + external = wasm_func_as_extern(func); + break; + } + case EXPORT_KIND_GLOBAL: + { + wasm_global_t *global; + if (!(global = wasm_global_new_internal( + store, export->index, + (WASMModuleInstanceCommon *)inst_interp))) { + goto failed; + } + + external = wasm_global_as_extern(global); + break; + } + case EXPORT_KIND_TABLE: + { + wasm_table_t *table; + if (!(table = wasm_table_new_internal( + store, export->index, + (WASMModuleInstanceCommon *)inst_interp))) { + goto failed; + } + + external = wasm_table_as_extern(table); + break; + } + case EXPORT_KIND_MEMORY: + { + wasm_memory_t *memory; + if (!(memory = wasm_memory_new_internal( + store, export->index, + (WASMModuleInstanceCommon *)inst_interp))) { + goto failed; + } + + external = wasm_memory_as_extern(memory); + break; + } + default: + LOG_WARNING("%s meets unsupported kind: %d", __FUNCTION__, + export->kind); + goto failed; + } + + if (!bh_vector_append((Vector *)externals, &external)) { + goto failed; + } + } + + return true; + +failed: + wasm_extern_delete(external); + return false; +} +#endif /* WASM_ENABLE_INTERP */ + +#if WASM_ENABLE_AOT != 0 +static bool +aot_link_func(const wasm_instance_t *inst, const AOTModule *module_aot, + uint32 import_func_idx_rt, wasm_func_t *import) +{ + AOTImportFunc *import_aot_func = NULL; + + bh_assert(inst && module_aot && import); + + import_aot_func = module_aot->import_funcs + import_func_idx_rt; + bh_assert(import_aot_func); + + /* it is a placeholder and let's skip it*/ + if (!import->type) + return true; + + /* type comparison */ + if (!wasm_functype_same_internal(import->type, import_aot_func->func_type)) + return false; + + import_aot_func->call_conv_wasm_c_api = true; + /* only set func_ptr_linked to avoid unlink warning during instantiation, + func_ptr_linked, with_env and env will be stored in module instance's + c_api_func_imports later and used when calling import function */ + if (import->with_env) + import_aot_func->func_ptr_linked = import->u.cb_env.cb; + else + import_aot_func->func_ptr_linked = import->u.cb; + bh_assert(import_aot_func->func_ptr_linked); + + import->func_idx_rt = import_func_idx_rt; + + return true; +} + +static bool +aot_link_global(const AOTModule *module_aot, uint16 global_idx_rt, + wasm_global_t *import) +{ + AOTImportGlobal *import_aot_global = NULL; + const wasm_valtype_t *val_type = NULL; + + bh_assert(module_aot && import); + + import_aot_global = module_aot->import_globals + global_idx_rt; + bh_assert(import_aot_global); + + /* it is a placeholder and let's skip it*/ + if (!import->type) + return true; + + val_type = wasm_globaltype_content(import->type); + bh_assert(val_type); + + if (!cmp_val_kind_with_val_type(wasm_valtype_kind(val_type), + import_aot_global->type.val_type)) + return false; + + bh_assert(import->init); + switch (wasm_valtype_kind(val_type)) { + case WASM_I32: + import_aot_global->global_data_linked.i32 = import->init->of.i32; + break; + case WASM_I64: + import_aot_global->global_data_linked.i64 = import->init->of.i64; + break; + case WASM_F32: + import_aot_global->global_data_linked.f32 = import->init->of.f32; + break; + case WASM_F64: + import_aot_global->global_data_linked.f64 = import->init->of.f64; + break; + default: + goto failed; + } + + import->global_idx_rt = global_idx_rt; + import_aot_global->is_linked = true; + return true; +failed: + LOG_DEBUG("%s failed", __FUNCTION__); + return false; +} + +static bool +aot_process_export(wasm_store_t *store, const AOTModuleInstance *inst_aot, + wasm_extern_vec_t *externals) +{ + uint32 i; + wasm_extern_t *external = NULL; + AOTModule *module_aot = NULL; + + bh_assert(store && inst_aot && externals); + + module_aot = (AOTModule *)inst_aot->module; + bh_assert(module_aot); + + for (i = 0; i < module_aot->export_count; ++i) { + AOTExport *export = module_aot->exports + i; + + switch (export->kind) { + case EXPORT_KIND_FUNC: + { + wasm_func_t *func = NULL; + if (!(func = wasm_func_new_internal( + store, export->index, + (WASMModuleInstanceCommon *)inst_aot))) { + goto failed; + } + + external = wasm_func_as_extern(func); + break; + } + case EXPORT_KIND_GLOBAL: + { + wasm_global_t *global = NULL; + if (!(global = wasm_global_new_internal( + store, export->index, + (WASMModuleInstanceCommon *)inst_aot))) { + goto failed; + } + + external = wasm_global_as_extern(global); + break; + } + case EXPORT_KIND_TABLE: + { + wasm_table_t *table; + if (!(table = wasm_table_new_internal( + store, export->index, + (WASMModuleInstanceCommon *)inst_aot))) { + goto failed; + } + + external = wasm_table_as_extern(table); + break; + } + case EXPORT_KIND_MEMORY: + { + wasm_memory_t *memory; + if (!(memory = wasm_memory_new_internal( + store, export->index, + (WASMModuleInstanceCommon *)inst_aot))) { + goto failed; + } + + external = wasm_memory_as_extern(memory); + break; + } + default: + LOG_WARNING("%s meets unsupported kind: %d", __FUNCTION__, + export->kind); + goto failed; + } + + if (!(external->name = malloc_internal(sizeof(wasm_byte_vec_t)))) { + goto failed; + } + + wasm_name_new_from_string_nt(external->name, export->name); + if (strlen(export->name) && !external->name->data) { + goto failed; + } + + if (!bh_vector_append((Vector *)externals, &external)) { + goto failed; + } + } + + return true; + +failed: + wasm_extern_delete(external); + return false; +} +#endif /* WASM_ENABLE_AOT */ + +static bool +do_link(const wasm_instance_t *inst, const wasm_module_t *module, + const wasm_extern_vec_t *imports) +{ + uint32 i, import_func_i, import_global_i; + + bh_assert(inst && module); + + /* we have run a module_type check before. */ + + for (i = 0, import_func_i = 0, import_global_i = 0; i < imports->num_elems; + i++) { + wasm_extern_t *import = imports->data[i]; + + if (!import) { + LOG_ERROR("imports[%d] is NULL and it is fatal\n", i); + goto failed; + } + + switch (wasm_extern_kind(import)) { + case WASM_EXTERN_FUNC: + { + bool ret = false; +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + ret = interp_link_func(inst, MODULE_INTERP(module), + import_func_i, + wasm_extern_as_func(import)); + } +#endif +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + ret = aot_link_func(inst, MODULE_AOT(module), import_func_i, + wasm_extern_as_func(import)); + } +#endif + if (!ret) { + LOG_WARNING("link function #%d failed", import_func_i); + goto failed; + } + + import_func_i++; + break; + } + case WASM_EXTERN_GLOBAL: + { + bool ret = false; +#if WASM_ENABLE_INTERP != 0 + if ((*module)->module_type == Wasm_Module_Bytecode) { + ret = interp_link_global(MODULE_INTERP(module), + import_global_i, + wasm_extern_as_global(import)); + } +#endif +#if WASM_ENABLE_AOT != 0 + if ((*module)->module_type == Wasm_Module_AoT) { + ret = aot_link_global(MODULE_AOT(module), import_global_i, + wasm_extern_as_global(import)); + } +#endif + if (!ret) { + LOG_WARNING("link global #%d failed", import_global_i); + goto failed; + } + + import_global_i++; + break; + } + case WASM_EXTERN_MEMORY: + case WASM_EXTERN_TABLE: + { + LOG_WARNING("doesn't support import memories and tables for " + "now, ignore them"); + break; + } + default: + { + UNREACHABLE(); + break; + } + } + } + + return true; +failed: + LOG_DEBUG("%s failed", __FUNCTION__); + return false; +} + +wasm_instance_t * +wasm_instance_new(wasm_store_t *store, const wasm_module_t *module, + const wasm_extern_vec_t *imports, own wasm_trap_t **trap) +{ + return wasm_instance_new_with_args(store, module, imports, trap, + KILOBYTE(32), KILOBYTE(32)); +} + +wasm_instance_t * +wasm_instance_new_with_args(wasm_store_t *store, const wasm_module_t *module, + const wasm_extern_vec_t *imports, + own wasm_trap_t **trap, const uint32 stack_size, + const uint32 heap_size) +{ + InstantiationArgs inst_args = { 0 }; + inst_args.default_stack_size = stack_size; + inst_args.host_managed_heap_size = heap_size; + return wasm_instance_new_with_args_ex(store, module, imports, trap, + &inst_args); +} + +wasm_instance_t * +wasm_instance_new_with_args_ex(wasm_store_t *store, const wasm_module_t *module, + const wasm_extern_vec_t *imports, + own wasm_trap_t **trap, + const InstantiationArgs *inst_args) +{ + char sub_error_buf[128] = { 0 }; + char error_buf[256] = { 0 }; + wasm_instance_t *instance = NULL; + CApiFuncImport *func_import = NULL, **p_func_imports = NULL; + uint32 i = 0, import_func_count = 0; + uint64 total_size; + bool build_exported = false; + + bh_assert(singleton_engine); + + if (!module) + return NULL; + + /* + * will do the check at the end of wasm_runtime_instantiate + */ + + WASM_C_DUMP_PROC_MEM(); + + instance = malloc_internal(sizeof(wasm_instance_t)); + if (!instance) { + snprintf(sub_error_buf, sizeof(sub_error_buf), + "Failed to malloc instance"); + goto failed; + } + + /* executes the instantiate-time linking if provided */ + if (imports) { + if (!do_link(instance, module, imports)) { + snprintf(sub_error_buf, sizeof(sub_error_buf), + "Failed to validate imports"); + goto failed; + } + } + /* + * will do the linking result check at the end of wasm_runtime_instantiate + */ + + instance->inst_comm_rt = wasm_runtime_instantiate_ex( + *module, inst_args, sub_error_buf, sizeof(sub_error_buf)); + if (!instance->inst_comm_rt) { + goto failed; + } + + if (!wasm_runtime_create_exec_env_singleton(instance->inst_comm_rt)) { + snprintf(sub_error_buf, sizeof(sub_error_buf), + "Failed to create exec env singleton"); + goto failed; + } + + /* create the c-api func import list */ +#if WASM_ENABLE_INTERP != 0 + if (instance->inst_comm_rt->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *wasm_module_inst = + (WASMModuleInstance *)instance->inst_comm_rt; + p_func_imports = &(wasm_module_inst->c_api_func_imports); + import_func_count = MODULE_INTERP(module)->import_function_count; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (instance->inst_comm_rt->module_type == Wasm_Module_AoT) { + AOTModuleInstance *aot_module_inst = + (AOTModuleInstance *)instance->inst_comm_rt; + p_func_imports = &(aot_module_inst->c_api_func_imports); + import_func_count = MODULE_AOT(module)->import_func_count; + } +#endif + bh_assert(p_func_imports); + + total_size = (uint64)sizeof(CApiFuncImport) * import_func_count; + if (total_size > 0 + && !(*p_func_imports = func_import = malloc_internal(total_size))) { + snprintf(sub_error_buf, sizeof(sub_error_buf), + "Failed to create wasm-c-api func imports"); + goto failed; + } + + /* fill in module_inst->c_api_func_imports */ + for (i = 0; imports && i < imports->num_elems; i++) { + wasm_func_t *func_host = NULL; + wasm_extern_t *in = imports->data[i]; + bh_assert(in); + + if (wasm_extern_kind(in) != WASM_EXTERN_FUNC) + continue; + + func_host = wasm_extern_as_func(in); + /* it is a placeholder and let's skip it*/ + if (!func_host->type) { + func_import++; + continue; + } + + func_import->with_env_arg = func_host->with_env; + if (func_host->with_env) { + func_import->func_ptr_linked = func_host->u.cb_env.cb; + func_import->env_arg = func_host->u.cb_env.env; + } + else { + func_import->func_ptr_linked = func_host->u.cb; + func_import->env_arg = NULL; + } + bh_assert(func_import->func_ptr_linked); + + func_import++; + } + + /* fill with inst */ + for (i = 0; imports && imports->data && i < imports->num_elems; ++i) { + wasm_extern_t *import = imports->data[i]; + bh_assert(import); + + switch (import->kind) { + case WASM_EXTERN_FUNC: + wasm_extern_as_func(import)->inst_comm_rt = + instance->inst_comm_rt; + break; + case WASM_EXTERN_GLOBAL: + wasm_extern_as_global(import)->inst_comm_rt = + instance->inst_comm_rt; + break; + case WASM_EXTERN_MEMORY: + wasm_extern_as_memory(import)->inst_comm_rt = + instance->inst_comm_rt; + break; + case WASM_EXTERN_TABLE: + wasm_extern_as_table(import)->inst_comm_rt = + instance->inst_comm_rt; + break; + default: + snprintf(sub_error_buf, sizeof(sub_error_buf), + "Unknown import kind"); + goto failed; + } + } + + /* build the exports list */ +#if WASM_ENABLE_INTERP != 0 + if (instance->inst_comm_rt->module_type == Wasm_Module_Bytecode) { + uint32 export_cnt = ((WASMModuleInstance *)instance->inst_comm_rt) + ->module->export_count; + + INIT_VEC(instance->exports, wasm_extern_vec_new_uninitialized, + export_cnt); + + if (!interp_process_export(store, + (WASMModuleInstance *)instance->inst_comm_rt, + instance->exports)) { + snprintf(sub_error_buf, sizeof(sub_error_buf), + "Interpreter failed to process exports"); + goto failed; + } + + build_exported = true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (instance->inst_comm_rt->module_type == Wasm_Module_AoT) { + uint32 export_cnt = + ((AOTModuleInstance *)instance->inst_comm_rt)->export_func_count + + ((AOTModuleInstance *)instance->inst_comm_rt)->export_global_count + + ((AOTModuleInstance *)instance->inst_comm_rt)->export_table_count + + ((AOTModuleInstance *)instance->inst_comm_rt) + ->export_memory_count; + + INIT_VEC(instance->exports, wasm_extern_vec_new_uninitialized, + export_cnt); + + if (!aot_process_export(store, + (AOTModuleInstance *)instance->inst_comm_rt, + instance->exports)) { + snprintf(sub_error_buf, sizeof(sub_error_buf), + "AOT failed to process exports"); + goto failed; + } + + build_exported = true; + } +#endif + + /* + * a wrong combination of module filetype and compilation flags + * leads to below branch + */ + if (!build_exported) { + snprintf(sub_error_buf, sizeof(sub_error_buf), + "Incorrect filetype and compilation flags"); + goto failed; + } + + /* add it to a watching list in store */ + if (!bh_vector_append((Vector *)store->instances, &instance)) { + snprintf(sub_error_buf, sizeof(sub_error_buf), + "Failed to add to store instances"); + goto failed; + } + + WASM_C_DUMP_PROC_MEM(); + + return instance; + +failed: + snprintf(error_buf, sizeof(error_buf), "%s failed: %s", __FUNCTION__, + sub_error_buf); + if (trap != NULL) { + wasm_message_t message = { 0 }; + wasm_name_new_from_string_nt(&message, error_buf); + *trap = wasm_trap_new(store, &message); + wasm_byte_vec_delete(&message); + } + LOG_DEBUG("%s", error_buf); + wasm_instance_delete_internal(instance); + return NULL; +} + +static void +wasm_instance_delete_internal(wasm_instance_t *instance) +{ + if (!instance) { + return; + } + + DEINIT_VEC(instance->exports, wasm_extern_vec_delete); + + if (instance->inst_comm_rt) { + wasm_runtime_deinstantiate(instance->inst_comm_rt); + instance->inst_comm_rt = NULL; + } + wasm_runtime_free(instance); +} + +void +wasm_instance_delete(wasm_instance_t *inst) +{ + DELETE_HOST_INFO(inst) + /* will release instance when releasing the store */ +} + +void +wasm_instance_exports(const wasm_instance_t *instance, + own wasm_extern_vec_t *out) +{ + if (!instance || !out) { + return; + } + wasm_extern_vec_copy(out, instance->exports); +} + +wasm_extern_t * +wasm_extern_copy(const wasm_extern_t *src) +{ + wasm_extern_t *dst = NULL; + + if (!src) { + return NULL; + } + + switch (wasm_extern_kind(src)) { + case WASM_EXTERN_FUNC: + dst = wasm_func_as_extern( + wasm_func_copy(wasm_extern_as_func_const(src))); + break; + case WASM_EXTERN_GLOBAL: + dst = wasm_global_as_extern( + wasm_global_copy(wasm_extern_as_global_const(src))); + break; + case WASM_EXTERN_MEMORY: + dst = wasm_memory_as_extern( + wasm_memory_copy(wasm_extern_as_memory_const(src))); + break; + case WASM_EXTERN_TABLE: + dst = wasm_table_as_extern( + wasm_table_copy(wasm_extern_as_table_const(src))); + break; + default: + LOG_WARNING("%s meets unsupported kind: %d", __FUNCTION__, + src->kind); + break; + } + + if (!dst) { + goto failed; + } + + return dst; + +failed: + LOG_DEBUG("%s failed", __FUNCTION__); + wasm_extern_delete(dst); + return NULL; +} + +void +wasm_extern_delete(wasm_extern_t *external) +{ + if (!external) { + return; + } + + if (external->name) { + wasm_byte_vec_delete(external->name); + wasm_runtime_free(external->name); + external->name = NULL; + } + + switch (wasm_extern_kind(external)) { + case WASM_EXTERN_FUNC: + wasm_func_delete(wasm_extern_as_func(external)); + break; + case WASM_EXTERN_GLOBAL: + wasm_global_delete(wasm_extern_as_global(external)); + break; + case WASM_EXTERN_MEMORY: + wasm_memory_delete(wasm_extern_as_memory(external)); + break; + case WASM_EXTERN_TABLE: + wasm_table_delete(wasm_extern_as_table(external)); + break; + default: + LOG_WARNING("%s meets unsupported kind: %d", __FUNCTION__, + external->kind); + break; + } +} + +wasm_externkind_t +wasm_extern_kind(const wasm_extern_t *external) +{ + if (!external) { + return WASM_EXTERNREF; + } + + return external->kind; +} + +own wasm_externtype_t * +wasm_extern_type(const wasm_extern_t *external) +{ + if (!external) { + return NULL; + } + + switch (wasm_extern_kind(external)) { + case WASM_EXTERN_FUNC: + return wasm_functype_as_externtype( + wasm_func_type(wasm_extern_as_func_const(external))); + case WASM_EXTERN_GLOBAL: + return wasm_globaltype_as_externtype( + wasm_global_type(wasm_extern_as_global_const(external))); + case WASM_EXTERN_MEMORY: + return wasm_memorytype_as_externtype( + wasm_memory_type(wasm_extern_as_memory_const(external))); + case WASM_EXTERN_TABLE: + return wasm_tabletype_as_externtype( + wasm_table_type(wasm_extern_as_table_const(external))); + default: + LOG_WARNING("%s meets unsupported kind: %d", __FUNCTION__, + external->kind); + break; + } + return NULL; +} + +#define BASIC_FOUR_LIST(V) \ + V(func) \ + V(global) \ + V(memory) \ + V(table) + +#define WASM_EXTERN_AS_OTHER(name) \ + wasm_##name##_t *wasm_extern_as_##name(wasm_extern_t *external) \ + { \ + return (wasm_##name##_t *)external; \ + } + +BASIC_FOUR_LIST(WASM_EXTERN_AS_OTHER) +#undef WASM_EXTERN_AS_OTHER + +#define WASM_OTHER_AS_EXTERN(name) \ + wasm_extern_t *wasm_##name##_as_extern(wasm_##name##_t *other) \ + { \ + return (wasm_extern_t *)other; \ + } + +BASIC_FOUR_LIST(WASM_OTHER_AS_EXTERN) +#undef WASM_OTHER_AS_EXTERN + +#define WASM_EXTERN_AS_OTHER_CONST(name) \ + const wasm_##name##_t *wasm_extern_as_##name##_const( \ + const wasm_extern_t *external) \ + { \ + return (const wasm_##name##_t *)external; \ + } + +BASIC_FOUR_LIST(WASM_EXTERN_AS_OTHER_CONST) +#undef WASM_EXTERN_AS_OTHER_CONST + +#define WASM_OTHER_AS_EXTERN_CONST(name) \ + const wasm_extern_t *wasm_##name##_as_extern_const( \ + const wasm_##name##_t *other) \ + { \ + return (const wasm_extern_t *)other; \ + } + +BASIC_FOUR_LIST(WASM_OTHER_AS_EXTERN_CONST) +#undef WASM_OTHER_AS_EXTERN_CONST + +wasm_extern_t * +wasm_extern_new_empty(wasm_store_t *store, wasm_externkind_t extern_kind) +{ + if (extern_kind == WASM_EXTERN_FUNC) + return wasm_func_as_extern(wasm_func_new_empty(store)); + + if (extern_kind == WASM_EXTERN_GLOBAL) + return wasm_global_as_extern(wasm_global_new_empty(store)); + + LOG_ERROR("Don't support linking table and memory for now"); + return NULL; +} + +double +wasm_instance_sum_wasm_exec_time(const wasm_instance_t *instance) +{ +#if WASM_ENABLE_PERF_PROFILING != 0 + return wasm_runtime_sum_wasm_exec_time(instance->inst_comm_rt); +#else + return -1.0; +#endif +} + +double +wasm_instance_get_wasm_func_exec_time(const wasm_instance_t *instance, + const char *name) +{ +#if WASM_ENABLE_PERF_PROFILING != 0 + return wasm_runtime_get_wasm_func_exec_time(instance->inst_comm_rt, name); +#else + return -1.0; +#endif +} diff --git a/wamr/src/wasm_exec_env.c b/wamr/src/wasm_exec_env.c new file mode 100644 index 0000000..1d4e4f3 --- /dev/null +++ b/wamr/src/wasm_exec_env.c @@ -0,0 +1,330 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_exec_env.h" // wasm_exec_env.h +#include "wasm_runtime_common.h" // wasm_runtime_common.h + +#if WASM_ENABLE_INTERP != 0 +#include "wasm_runtime.h" // ../interpreter/wasm_runtime.h +#endif + + + + +#if WASM_ENABLE_THREAD_MGR != 0 +#include "thread_manager.h" // ../libraries/thread-mgr/thread_manager.h +#if WASM_ENABLE_DEBUG_INTERP != 0 +#include "debug_engine.h" // ../libraries/debug-engine/debug_engine.h +#endif +#endif + +WASMExecEnv * +wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst, + uint32 stack_size) +{ + uint64 total_size = + offsetof(WASMExecEnv, wasm_stack_u.bottom) + (uint64)stack_size; + WASMExecEnv *exec_env; + + if (total_size >= UINT32_MAX + || !(exec_env = wasm_runtime_malloc((uint32)total_size))) + return NULL; + + memset(exec_env, 0, (uint32)total_size); + +#if WASM_ENABLE_AOT != 0 + if (!(exec_env->argv_buf = wasm_runtime_malloc(sizeof(uint32) * 64))) { + goto fail1; + } +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 + if (os_mutex_init(&exec_env->wait_lock) != 0) + goto fail2; + + if (os_cond_init(&exec_env->wait_cond) != 0) + goto fail3; + +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!(exec_env->current_status = wasm_cluster_create_exenv_status())) + goto fail4; +#endif +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!(exec_env->exce_check_guard_page = + os_mmap(NULL, os_getpagesize(), MMAP_PROT_NONE, MMAP_MAP_NONE, + os_get_invalid_handle()))) + goto fail5; +#endif + + exec_env->module_inst = module_inst; + exec_env->wasm_stack_size = stack_size; + exec_env->wasm_stack.bottom = exec_env->wasm_stack_u.bottom; + exec_env->wasm_stack.top_boundary = + exec_env->wasm_stack.bottom + stack_size; + exec_env->wasm_stack.top = exec_env->wasm_stack.bottom; + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstance *i = (AOTModuleInstance *)module_inst; + AOTModule *m = (AOTModule *)i->module; + exec_env->native_symbol = m->native_symbol_list; + } +#endif + +#if WASM_ENABLE_MEMORY_TRACING != 0 + wasm_runtime_dump_exec_env_mem_consumption(exec_env); +#endif + +#if WASM_ENABLE_INSTRUCTION_METERING != 0 + exec_env->instructions_to_execute = -1; +#endif + + return exec_env; + +#ifdef OS_ENABLE_HW_BOUND_CHECK +fail5: +#if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_DEBUG_INTERP != 0 + wasm_cluster_destroy_exenv_status(exec_env->current_status); +#endif +#endif +#if WASM_ENABLE_THREAD_MGR != 0 +#if WASM_ENABLE_DEBUG_INTERP != 0 +fail4: + os_cond_destroy(&exec_env->wait_cond); +#endif +fail3: + os_mutex_destroy(&exec_env->wait_lock); +fail2: +#endif +#if WASM_ENABLE_AOT != 0 + wasm_runtime_free(exec_env->argv_buf); +fail1: +#endif + wasm_runtime_free(exec_env); + return NULL; +} + +void +wasm_exec_env_destroy_internal(WASMExecEnv *exec_env) +{ +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_munmap(exec_env->exce_check_guard_page, os_getpagesize()); +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + os_mutex_destroy(&exec_env->wait_lock); + os_cond_destroy(&exec_env->wait_cond); +#if WASM_ENABLE_DEBUG_INTERP != 0 + wasm_cluster_destroy_exenv_status(exec_env->current_status); +#endif +#endif +#if WASM_ENABLE_AOT != 0 + wasm_runtime_free(exec_env->argv_buf); +#endif + wasm_runtime_free(exec_env); +} + +WASMExecEnv * +wasm_exec_env_create(struct WASMModuleInstanceCommon *module_inst, + uint32 stack_size) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + WASMCluster *cluster; +#endif + WASMExecEnv *exec_env = + wasm_exec_env_create_internal(module_inst, stack_size); +#if WASM_ENABLE_GC != 0 + void *gc_heap_handle = NULL; +#endif + + if (!exec_env) + return NULL; + +#if WASM_ENABLE_INTERP != 0 + /* Set the aux_stack_boundary and aux_stack_bottom */ + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModule *module = ((WASMModuleInstance *)module_inst)->module; + exec_env->aux_stack_bottom = (uintptr_t)module->aux_stack_bottom; + exec_env->aux_stack_boundary = + (uintptr_t)module->aux_stack_bottom - module->aux_stack_size; +#if WASM_ENABLE_GC != 0 + gc_heap_handle = + ((WASMModuleInstance *)module_inst)->e->common.gc_heap_pool; +#endif + } +#endif +#if WASM_ENABLE_AOT != 0 + /* Set the aux_stack_boundary and aux_stack_bottom */ + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModule *module = + (AOTModule *)((AOTModuleInstance *)module_inst)->module; + exec_env->aux_stack_bottom = (uintptr_t)module->aux_stack_bottom; + exec_env->aux_stack_boundary = + (uintptr_t)module->aux_stack_bottom - module->aux_stack_size; +#if WASM_ENABLE_GC != 0 + gc_heap_handle = + ((AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e) + ->common.gc_heap_handle; +#endif + } +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 + /* Create a new cluster for this exec_env */ + if (!(cluster = wasm_cluster_create(exec_env))) { + wasm_exec_env_destroy_internal(exec_env); + return NULL; + } +#if WASM_ENABLE_GC != 0 + mem_allocator_enable_gc_reclaim(gc_heap_handle, cluster); +#endif +#else +#if WASM_ENABLE_GC != 0 + mem_allocator_enable_gc_reclaim(gc_heap_handle, exec_env); +#endif +#endif /* end of WASM_ENABLE_THREAD_MGR */ + + return exec_env; +} + +void +wasm_exec_env_destroy(WASMExecEnv *exec_env) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + /* Wait for all sub-threads */ + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + if (cluster) { + wasm_cluster_wait_for_all_except_self(cluster, exec_env); +#if WASM_ENABLE_DEBUG_INTERP != 0 + /* Must fire exit event after other threads exits, otherwise + the stopped thread will be overridden by other threads */ + wasm_cluster_thread_exited(exec_env); +#endif + /* We have waited for other threads, this is the only alive thread, so + * we don't acquire cluster->lock because the cluster will be destroyed + * inside this function */ + wasm_cluster_del_exec_env(cluster, exec_env); + } +#endif /* end of WASM_ENABLE_THREAD_MGR */ + + wasm_exec_env_destroy_internal(exec_env); +} + +WASMModuleInstanceCommon * +wasm_exec_env_get_module_inst(WASMExecEnv *exec_env) +{ + return exec_env->module_inst; +} + +void +wasm_exec_env_set_module_inst(WASMExecEnv *exec_env, + WASMModuleInstanceCommon *const module_inst) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_traverse_lock(exec_env); +#endif + exec_env->module_inst = module_inst; +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_traverse_unlock(exec_env); +#endif +} + +void +wasm_exec_env_restore_module_inst( + WASMExecEnv *exec_env, WASMModuleInstanceCommon *const module_inst_common) +{ + WASMModuleInstanceCommon *old_module_inst_common = exec_env->module_inst; + WASMModuleInstance *old_module_inst = + (WASMModuleInstance *)old_module_inst_common; + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_common; + char cur_exception[EXCEPTION_BUF_LEN]; + +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_traverse_lock(exec_env); +#endif + exec_env->module_inst = module_inst_common; + /* + * propagate an exception if any. + */ + exception_lock(old_module_inst); + if (old_module_inst->cur_exception[0] != '\0') { + bh_memcpy_s(cur_exception, sizeof(cur_exception), + old_module_inst->cur_exception, + sizeof(old_module_inst->cur_exception)); + } + else { + cur_exception[0] = '\0'; + } + exception_unlock(old_module_inst); +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_traverse_unlock(exec_env); +#endif + if (cur_exception[0] != '\0') { + exception_lock(module_inst); + bh_memcpy_s(module_inst->cur_exception, + sizeof(module_inst->cur_exception), cur_exception, + sizeof(cur_exception)); + exception_unlock(module_inst); + } +} + +void +wasm_exec_env_set_thread_info(WASMExecEnv *exec_env) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + os_mutex_lock(&exec_env->wait_lock); +#endif + exec_env->handle = os_self_thread(); + if (exec_env->user_native_stack_boundary) + /* WASM_STACK_GUARD_SIZE isn't added for flexibility to developer, + he must ensure that enough guard bytes are kept. */ + exec_env->native_stack_boundary = exec_env->user_native_stack_boundary; + else { + uint8 *stack_boundary = os_thread_get_stack_boundary(); + exec_env->native_stack_boundary = + stack_boundary ? stack_boundary + WASM_STACK_GUARD_SIZE : NULL; + } + exec_env->native_stack_top_min = (void *)UINTPTR_MAX; +#if WASM_ENABLE_THREAD_MGR != 0 + os_mutex_unlock(&exec_env->wait_lock); +#endif +} + +#if WASM_ENABLE_THREAD_MGR != 0 +void * +wasm_exec_env_get_thread_arg(WASMExecEnv *exec_env) +{ + return exec_env->thread_arg; +} + +void +wasm_exec_env_set_thread_arg(WASMExecEnv *exec_env, void *thread_arg) +{ + exec_env->thread_arg = thread_arg; +} +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK +void +wasm_exec_env_push_jmpbuf(WASMExecEnv *exec_env, WASMJmpBuf *jmpbuf) +{ + jmpbuf->prev = exec_env->jmpbuf_stack_top; + exec_env->jmpbuf_stack_top = jmpbuf; +} + +WASMJmpBuf * +wasm_exec_env_pop_jmpbuf(WASMExecEnv *exec_env) +{ + WASMJmpBuf *stack_top = exec_env->jmpbuf_stack_top; + + if (stack_top) { + exec_env->jmpbuf_stack_top = stack_top->prev; + return stack_top; + } + + return NULL; +} +#endif diff --git a/wamr/src/wasm_interp_fast.c b/wamr/src/wasm_interp_fast.c new file mode 100644 index 0000000..3eb0494 --- /dev/null +++ b/wamr/src/wasm_interp_fast.c @@ -0,0 +1,7997 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_interp.h" // wasm_interp.h +#include "bh_log.h" // bh_log.h +#include "wasm_runtime.h" // wasm_runtime.h +#include "wasm_opcode.h" // wasm_opcode.h +#include "wasm_loader.h" // wasm_loader.h +#include "wasm_memory.h" // wasm_memory.h +#include "wasm_exec_env.h" // ../common/wasm_exec_env.h +#if WASM_ENABLE_GC != 0 + +#include "mem_alloc.h" // mem_alloc.h +#if WASM_ENABLE_STRINGREF != 0 +#include "string_object.h" // string_object.h +#endif +#endif + + +#if WASM_ENABLE_SIMDE != 0 +#include "simd128.h" // simde/wasm/simd128.h +#endif + +typedef int32 CellType_I32; +typedef int64 CellType_I64; +typedef float32 CellType_F32; +typedef float64 CellType_F64; + +#if WASM_ENABLE_THREAD_MGR == 0 +#define get_linear_mem_size() linear_mem_size +#else +/** + * Load memory data size in each time boundary check in + * multi-threading mode since it may be changed by other + * threads in memory.grow + */ +#define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) +#endif + +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ + /* If offset1 is in valid range, maddr must also \ + be in valid range, no need to check it again. */ \ + maddr = memory->memory_data + offset1; \ + else \ + goto out_of_bounds; \ + } while (0) + +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ + do { \ + uint64 offset1 = (uint32)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ + /* App heap space is not valid space for \ + bulk memory operation */ \ + maddr = memory->memory_data + offset1; \ + else \ + goto out_of_bounds; \ + } while (0) +#else +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + maddr = memory->memory_data + offset1; \ + } while (0) + +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ + do { \ + uint64 offset1 = (uint32)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + maddr = memory->memory_data + offset1; \ + } while (0) +#endif /* !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ + +#define CHECK_ATOMIC_MEMORY_ACCESS(align) \ + do { \ + if (((uintptr_t)maddr & (align - 1)) != 0) \ + goto unaligned_atomic; \ + } while (0) + +#if WASM_ENABLE_INSTRUCTION_METERING != 0 +#define CHECK_INSTRUCTION_LIMIT() \ + if (instructions_left == 0) { \ + wasm_set_exception(module, "instruction limit exceeded"); \ + goto got_exception; \ + } \ + else if (instructions_left > 0) \ + instructions_left--; + +#else +#define CHECK_INSTRUCTION_LIMIT() (void)0 +#endif + +static inline uint32 +rotl32(uint32 n, uint32 c) +{ + const uint32 mask = (31); + c = c % 32; + c &= mask; + return (n << c) | (n >> ((0 - c) & mask)); +} + +static inline uint32 +rotr32(uint32 n, uint32 c) +{ + const uint32 mask = (31); + c = c % 32; + c &= mask; + return (n >> c) | (n << ((0 - c) & mask)); +} + +static inline uint64 +rotl64(uint64 n, uint64 c) +{ + const uint64 mask = (63); + c = c % 64; + c &= mask; + return (n << c) | (n >> ((0 - c) & mask)); +} + +static inline uint64 +rotr64(uint64 n, uint64 c) +{ + const uint64 mask = (63); + c = c % 64; + c &= mask; + return (n >> c) | (n << ((0 - c) & mask)); +} + +static inline float32 +f32_min(float32 a, float32 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? a : b; + else + return a > b ? b : a; +} + +static inline float32 +f32_max(float32 a, float32 b) +{ + if (isnan(a) || isnan(b)) + return NAN; + else if (a == 0 && a == b) + return signbit(a) ? b : a; + else + return a > b ? a : b; +} + +static inline float64 +f64_min(float64 a, float64 b) +{ + if (isnan(a) || isnan(b)) + return (float64)NAN; + else if (a == 0 && a == b) + return signbit(a) ? a : b; + else + return a > b ? b : a; +} + +static inline float64 +f64_max(float64 a, float64 b) +{ + if (isnan(a) || isnan(b)) + return (float64)NAN; + else if (a == 0 && a == b) + return signbit(a) ? b : a; + else + return a > b ? a : b; +} + +static inline uint32 +clz32(uint32 type) +{ + uint32 num = 0; + if (type == 0) + return 32; + while (!(type & 0x80000000)) { + num++; + type <<= 1; + } + return num; +} + +static inline uint32 +clz64(uint64 type) +{ + uint32 num = 0; + if (type == 0) + return 64; + while (!(type & 0x8000000000000000LL)) { + num++; + type <<= 1; + } + return num; +} + +static inline uint32 +ctz32(uint32 type) +{ + uint32 num = 0; + if (type == 0) + return 32; + while (!(type & 1)) { + num++; + type >>= 1; + } + return num; +} + +static inline uint32 +ctz64(uint64 type) +{ + uint32 num = 0; + if (type == 0) + return 64; + while (!(type & 1)) { + num++; + type >>= 1; + } + return num; +} + +static inline uint32 +popcount32(uint32 u) +{ + uint32 ret = 0; + while (u) { + u = (u & (u - 1)); + ret++; + } + return ret; +} + +static inline uint32 +popcount64(uint64 u) +{ + uint32 ret = 0; + while (u) { + u = (u & (u - 1)); + ret++; + } + return ret; +} + +static float +local_copysignf(float x, float y) +{ + union { + float f; + uint32 i; + } ux = { x }, uy = { y }; + ux.i &= 0x7fffffff; + ux.i |= uy.i & 0x80000000; + return ux.f; +} + +static double +local_copysign(double x, double y) +{ + union { + double f; + uint64 i; + } ux = { x }, uy = { y }; + ux.i &= UINT64_MAX / 2; + ux.i |= uy.i & 1ULL << 63; + return ux.f; +} + +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 +#define LOAD_U32_WITH_2U16S(addr) (*(uint32 *)(addr)) +#define LOAD_PTR(addr) (*(void **)(addr)) +#else /* else of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +static inline uint32 +LOAD_U32_WITH_2U16S(void *addr) +{ + union { + uint32 val; + uint16 u16[2]; + } u; + + bh_assert(((uintptr_t)addr & 1) == 0); + u.u16[0] = ((uint16 *)addr)[0]; + u.u16[1] = ((uint16 *)addr)[1]; + return u.val; +} +#if UINTPTR_MAX == UINT32_MAX +#define LOAD_PTR(addr) ((void *)LOAD_U32_WITH_2U16S(addr)) +#elif UINTPTR_MAX == UINT64_MAX +static inline void * +LOAD_PTR(void *addr) +{ + uintptr_t addr1 = (uintptr_t)addr; + union { + void *val; + uint32 u32[2]; + uint16 u16[4]; + } u; + + bh_assert(((uintptr_t)addr & 1) == 0); + if ((addr1 & (uintptr_t)7) == 0) + return *(void **)addr; + + if ((addr1 & (uintptr_t)3) == 0) { + u.u32[0] = ((uint32 *)addr)[0]; + u.u32[1] = ((uint32 *)addr)[1]; + } + else { + u.u16[0] = ((uint16 *)addr)[0]; + u.u16[1] = ((uint16 *)addr)[1]; + u.u16[2] = ((uint16 *)addr)[2]; + u.u16[3] = ((uint16 *)addr)[3]; + } + return u.val; +} +#endif /* end of UINTPTR_MAX */ +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ + +#if WASM_ENABLE_GC != 0 +static void +init_frame_refs(uint8 *frame_ref, uint32 cell_num, WASMFunctionInstance *func) +{ + uint32 i, j; + + memset(frame_ref, 0, cell_num); + + for (i = 0, j = 0; i < func->param_count; i++) { + if (wasm_is_type_reftype(func->param_types[i]) + && !wasm_is_reftype_i31ref(func->param_types[i])) { + frame_ref[j++] = 1; +#if UINTPTR_MAX == UINT64_MAX + frame_ref[j++] = 1; +#endif + } + else { + j += wasm_value_type_cell_num(func->param_types[i]); + } + } + + for (i = 0; i < func->local_count; i++) { + if (wasm_is_type_reftype(func->local_types[i]) + && !wasm_is_reftype_i31ref(func->local_types[i])) { + frame_ref[j++] = 1; +#if UINTPTR_MAX == UINT64_MAX + frame_ref[j++] = 1; +#endif + } + else { + j += wasm_value_type_cell_num(func->local_types[i]); + } + } +} + +uint8 * +wasm_interp_get_frame_ref(WASMInterpFrame *frame) +{ + return frame->frame_ref; +} + +/* Return the corresponding ref slot of the given slot of local + variable or stack pointer. */ + +#define COMPUTE_FRAME_REF(ref, off) (ref + (unsigned)(off)) + +#define FRAME_REF(off) COMPUTE_FRAME_REF(frame_ref, off) + +#if UINTPTR_MAX == UINT64_MAX +#define SET_FRAME_REF(off) *FRAME_REF(off) = *FRAME_REF(off + 1) = 1 +#define CLEAR_FRAME_REF(off) \ + (unsigned)off >= local_cell_num \ + ? (*FRAME_REF(off) = *FRAME_REF(off + 1) = 0) \ + : (void)0 +#else +#define SET_FRAME_REF(off) *FRAME_REF(off) = 1 +#define CLEAR_FRAME_REF(off) \ + (unsigned)off >= local_cell_num ? (*FRAME_REF(off) = 0) : (void)0 +#endif + +#define FRAME_REF_FOR(frame, p) \ + COMPUTE_FRAME_REF(frame->frame_ref, p - frame->lp) + +#define CLEAR_FRAME_REF_FOR(p, n) \ + do { \ + int32 ref_i, ref_n = (int32)(n); \ + uint8 *ref = FRAME_REF(p - frame_lp); \ + for (ref_i = 0; ref_i < ref_n; ref_i++) \ + ref[ref_i] = 0; \ + } while (0) +#endif /* end of WASM_ENABLE_GC != 0 */ + +#define read_uint32(p) \ + (p += sizeof(uint32), LOAD_U32_WITH_2U16S(p - sizeof(uint32))) + +#define GET_LOCAL_INDEX_TYPE_AND_OFFSET() \ + do { \ + uint32 param_count = cur_func->param_count; \ + local_idx = read_uint32(frame_ip); \ + bh_assert(local_idx < param_count + cur_func->local_count); \ + local_offset = cur_func->local_offsets[local_idx]; \ + if (local_idx < param_count) \ + local_type = cur_func->param_types[local_idx]; \ + else \ + local_type = cur_func->local_types[local_idx - param_count]; \ + } while (0) + +#define GET_OFFSET() (frame_ip += 2, *(int16 *)(frame_ip - 2)) + +#define SET_OPERAND_I32(off, value) \ + do { \ + *(uint32 *)(frame_lp + *(int16 *)(frame_ip + off)) = value; \ + } while (0) +#define SET_OPERAND_F32(off, value) \ + do { \ + *(float32 *)(frame_lp + *(int16 *)(frame_ip + off)) = value; \ + } while (0) +#define SET_OPERAND_I64(off, value) \ + do { \ + uint32 *addr_tmp = frame_lp + *(int16 *)(frame_ip + off); \ + PUT_I64_TO_ADDR(addr_tmp, value); \ + } while (0) +#define SET_OPERAND_F64(off, value) \ + do { \ + uint32 *addr_tmp = frame_lp + *(int16 *)(frame_ip + off); \ + PUT_F64_TO_ADDR(addr_tmp, value); \ + } while (0) +#define SET_OPERAND_REF(off, value) \ + do { \ + uint32 *addr_tmp; \ + opnd_off = *(int16 *)(frame_ip + off); \ + addr_tmp = frame_lp + opnd_off; \ + PUT_REF_TO_ADDR(addr_tmp, value); \ + SET_FRAME_REF(ond_off); \ + } while (0) + +#define SET_OPERAND(op_type, off, value) SET_OPERAND_##op_type(off, value) + +#define GET_OPERAND_I32(type, off) \ + *(type *)(frame_lp + *(int16 *)(frame_ip + off)) +#define GET_OPERAND_F32(type, off) \ + *(type *)(frame_lp + *(int16 *)(frame_ip + off)) +#define GET_OPERAND_I64(type, off) \ + (type) GET_I64_FROM_ADDR(frame_lp + *(int16 *)(frame_ip + off)) +#define GET_OPERAND_F64(type, off) \ + (type) GET_F64_FROM_ADDR(frame_lp + *(int16 *)(frame_ip + off)) +#define GET_OPERAND_V128(off) \ + GET_V128_FROM_ADDR(frame_lp + *(int16 *)(frame_ip + off)) +#define GET_OPERAND_REF(type, off) \ + (type) GET_REF_FROM_ADDR(frame_lp + *(int16 *)(frame_ip + off)) + +#define GET_OPERAND(type, op_type, off) GET_OPERAND_##op_type(type, off) + +#define PUSH_I32(value) \ + do { \ + *(int32 *)(frame_lp + GET_OFFSET()) = value; \ + } while (0) + +#define PUSH_F32(value) \ + do { \ + *(float32 *)(frame_lp + GET_OFFSET()) = value; \ + } while (0) + +#define PUSH_I64(value) \ + do { \ + uint32 *addr_tmp = frame_lp + GET_OFFSET(); \ + PUT_I64_TO_ADDR(addr_tmp, value); \ + } while (0) + +#define PUSH_F64(value) \ + do { \ + uint32 *addr_tmp = frame_lp + GET_OFFSET(); \ + PUT_F64_TO_ADDR(addr_tmp, value); \ + } while (0) + +#define PUSH_REF(value) \ + do { \ + uint32 *addr_tmp; \ + opnd_off = GET_OFFSET(); \ + addr_tmp = frame_lp + opnd_off; \ + PUT_REF_TO_ADDR(addr_tmp, value); \ + SET_FRAME_REF(opnd_off); \ + } while (0) + +#define PUSH_I31REF(value) \ + do { \ + uint32 *addr_tmp; \ + opnd_off = GET_OFFSET(); \ + addr_tmp = frame_lp + opnd_off; \ + PUT_REF_TO_ADDR(addr_tmp, value); \ + } while (0) + +#define POP_I32() (*(int32 *)(frame_lp + GET_OFFSET())) + +#define POP_F32() (*(float32 *)(frame_lp + GET_OFFSET())) + +#define POP_I64() (GET_I64_FROM_ADDR(frame_lp + GET_OFFSET())) + +#define POP_V128() (GET_V128_FROM_ADDR(frame_lp + GET_OFFSET())) + +#define POP_F64() (GET_F64_FROM_ADDR(frame_lp + GET_OFFSET())) + +#define POP_REF() \ + (opnd_off = GET_OFFSET(), CLEAR_FRAME_REF((unsigned)(opnd_off)), \ + GET_REF_FROM_ADDR(frame_lp + opnd_off)) + +#if WASM_ENABLE_GC != 0 +#define SYNC_FRAME_REF() frame->frame_ref = frame_ref +#define UPDATE_FRAME_REF() frame_ref = frame->frame_ref +#else +#define SYNC_FRAME_REF() (void)0 +#define UPDATE_FRAME_REF() (void)0 +#endif + +#define SYNC_ALL_TO_FRAME() \ + do { \ + frame->ip = frame_ip; \ + SYNC_FRAME_REF(); \ + } while (0) + +#define UPDATE_ALL_FROM_FRAME() \ + do { \ + frame_ip = frame->ip; \ + UPDATE_FRAME_REF(); \ + } while (0) + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#define UPDATE_FRAME_IP_END() (void)0 +#else +#define UPDATE_FRAME_IP_END() frame_ip_end = wasm_get_func_code_end(cur_func) +#endif + +#if WASM_ENABLE_GC != 0 +#define RECOVER_FRAME_REF() frame_ref = frame->frame_ref +#else +#define RECOVER_FRAME_REF() (void)0 +#endif + +#define RECOVER_CONTEXT(new_frame) \ + do { \ + frame = (new_frame); \ + cur_func = frame->function; \ + prev_frame = frame->prev_frame; \ + frame_ip = frame->ip; \ + UPDATE_FRAME_IP_END(); \ + frame_lp = frame->lp; \ + RECOVER_FRAME_REF(); \ + } while (0) + +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 +#define GET_OPCODE() opcode = *frame_ip++; +#else +#define GET_OPCODE() \ + opcode = *frame_ip; \ + frame_ip += 2; +#endif + +#define DEF_OP_EQZ(ctype, src_op_type) \ + do { \ + SET_OPERAND(I32, 2, (GET_OPERAND(ctype, src_op_type, 0) == 0)); \ + frame_ip += 4; \ + } while (0) + +#define DEF_OP_CMP(src_type, src_op_type, cond) \ + do { \ + SET_OPERAND(I32, 4, \ + GET_OPERAND(src_type, src_op_type, 2) \ + cond GET_OPERAND(src_type, src_op_type, 0)); \ + frame_ip += 6; \ + } while (0) + +#define DEF_OP_BIT_COUNT(src_type, src_op_type, operation) \ + do { \ + SET_OPERAND( \ + src_op_type, 2, \ + (src_type)operation(GET_OPERAND(src_type, src_op_type, 0))); \ + frame_ip += 4; \ + } while (0) + +#define DEF_OP_NUMERIC(src_type1, src_type2, src_op_type, operation) \ + do { \ + SET_OPERAND(src_op_type, 4, \ + GET_OPERAND(src_type1, src_op_type, 2) \ + operation GET_OPERAND(src_type2, src_op_type, 0)); \ + frame_ip += 6; \ + } while (0) + +#define DEF_OP_REINTERPRET(src_type, src_op_type) \ + do { \ + SET_OPERAND(src_op_type, 2, GET_OPERAND(src_type, src_op_type, 0)); \ + frame_ip += 4; \ + } while (0) + +#define DEF_OP_NUMERIC_64 DEF_OP_NUMERIC + +#define DEF_OP_NUMERIC2(src_type1, src_type2, src_op_type, operation) \ + do { \ + SET_OPERAND(src_op_type, 4, \ + GET_OPERAND(src_type1, src_op_type, 2) operation( \ + GET_OPERAND(src_type2, src_op_type, 0) % 32)); \ + frame_ip += 6; \ + } while (0) + +#define DEF_OP_NUMERIC2_64(src_type1, src_type2, src_op_type, operation) \ + do { \ + SET_OPERAND(src_op_type, 4, \ + GET_OPERAND(src_type1, src_op_type, 2) operation( \ + GET_OPERAND(src_type2, src_op_type, 0) % 64)); \ + frame_ip += 6; \ + } while (0) + +#define DEF_ATOMIC_RMW_OPCODE(OP_NAME, op) \ + case WASM_OP_ATOMIC_RMW_I32_##OP_NAME: \ + case WASM_OP_ATOMIC_RMW_I32_##OP_NAME##8_U: \ + case WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U: \ + { \ + uint32 readv, sval; \ + \ + sval = POP_I32(); \ + addr = POP_I32(); \ + \ + if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##8_U) { \ + CHECK_MEMORY_OVERFLOW(1); \ + CHECK_ATOMIC_MEMORY_ACCESS(1); \ + \ + shared_memory_lock(memory); \ + readv = (uint32)(*(uint8 *)maddr); \ + *(uint8 *)maddr = (uint8)(readv op sval); \ + shared_memory_unlock(memory); \ + } \ + else if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U) { \ + CHECK_MEMORY_OVERFLOW(2); \ + CHECK_ATOMIC_MEMORY_ACCESS(2); \ + \ + shared_memory_lock(memory); \ + readv = (uint32)LOAD_U16(maddr); \ + STORE_U16(maddr, (uint16)(readv op sval)); \ + shared_memory_unlock(memory); \ + } \ + else { \ + CHECK_MEMORY_OVERFLOW(4); \ + CHECK_ATOMIC_MEMORY_ACCESS(4); \ + \ + shared_memory_lock(memory); \ + readv = LOAD_I32(maddr); \ + STORE_U32(maddr, readv op sval); \ + shared_memory_unlock(memory); \ + } \ + PUSH_I32(readv); \ + break; \ + } \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME: \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##8_U: \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U: \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U: \ + { \ + uint64 readv, sval; \ + \ + sval = (uint64)POP_I64(); \ + addr = POP_I32(); \ + \ + if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##8_U) { \ + CHECK_MEMORY_OVERFLOW(1); \ + CHECK_ATOMIC_MEMORY_ACCESS(1); \ + \ + shared_memory_lock(memory); \ + readv = (uint64)(*(uint8 *)maddr); \ + *(uint8 *)maddr = (uint8)(readv op sval); \ + shared_memory_unlock(memory); \ + } \ + else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U) { \ + CHECK_MEMORY_OVERFLOW(2); \ + CHECK_ATOMIC_MEMORY_ACCESS(2); \ + \ + shared_memory_lock(memory); \ + readv = (uint64)LOAD_U16(maddr); \ + STORE_U16(maddr, (uint16)(readv op sval)); \ + shared_memory_unlock(memory); \ + } \ + else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U) { \ + CHECK_MEMORY_OVERFLOW(4); \ + CHECK_ATOMIC_MEMORY_ACCESS(4); \ + \ + shared_memory_lock(memory); \ + readv = (uint64)LOAD_U32(maddr); \ + STORE_U32(maddr, (uint32)(readv op sval)); \ + shared_memory_unlock(memory); \ + } \ + else { \ + uint64 op_result; \ + CHECK_MEMORY_OVERFLOW(8); \ + CHECK_ATOMIC_MEMORY_ACCESS(8); \ + \ + shared_memory_lock(memory); \ + readv = (uint64)LOAD_I64(maddr); \ + op_result = readv op sval; \ + STORE_I64(maddr, op_result); \ + shared_memory_unlock(memory); \ + } \ + PUSH_I64(readv); \ + break; \ + } + +#define DEF_OP_MATH(src_type, src_op_type, method) \ + do { \ + SET_OPERAND(src_op_type, 2, \ + (src_type)method(GET_OPERAND(src_type, src_op_type, 0))); \ + frame_ip += 4; \ + } while (0) + +#define TRUNC_FUNCTION(func_name, src_type, dst_type, signed_type) \ + static dst_type func_name(src_type src_value, src_type src_min, \ + src_type src_max, dst_type dst_min, \ + dst_type dst_max, bool is_sign) \ + { \ + dst_type dst_value = 0; \ + if (!isnan(src_value)) { \ + if (src_value <= src_min) \ + dst_value = dst_min; \ + else if (src_value >= src_max) \ + dst_value = dst_max; \ + else { \ + if (is_sign) \ + dst_value = (dst_type)(signed_type)src_value; \ + else \ + dst_value = (dst_type)src_value; \ + } \ + } \ + return dst_value; \ + } + +TRUNC_FUNCTION(trunc_f32_to_i32, float32, uint32, int32) +TRUNC_FUNCTION(trunc_f32_to_i64, float32, uint64, int64) +TRUNC_FUNCTION(trunc_f64_to_i32, float64, uint32, int32) +TRUNC_FUNCTION(trunc_f64_to_i64, float64, uint64, int64) + +static bool +trunc_f32_to_int(WASMModuleInstance *module, uint8 *frame_ip, uint32 *frame_lp, + float32 src_min, float32 src_max, bool saturating, bool is_i32, + bool is_sign) +{ + float32 src_value = GET_OPERAND(float32, F32, 0); + uint64 dst_value_i64; + uint32 dst_value_i32; + + if (!saturating) { + if (isnan(src_value)) { + wasm_set_exception(module, "invalid conversion to integer"); + return false; + } + else if (src_value <= src_min || src_value >= src_max) { + wasm_set_exception(module, "integer overflow"); + return false; + } + } + + if (is_i32) { + uint32 dst_min = is_sign ? INT32_MIN : 0; + uint32 dst_max = is_sign ? INT32_MAX : UINT32_MAX; + dst_value_i32 = trunc_f32_to_i32(src_value, src_min, src_max, dst_min, + dst_max, is_sign); + SET_OPERAND(I32, 2, dst_value_i32); + } + else { + uint64 dst_min = is_sign ? INT64_MIN : 0; + uint64 dst_max = is_sign ? INT64_MAX : UINT64_MAX; + dst_value_i64 = trunc_f32_to_i64(src_value, src_min, src_max, dst_min, + dst_max, is_sign); + SET_OPERAND(I64, 2, dst_value_i64); + } + return true; +} + +static bool +trunc_f64_to_int(WASMModuleInstance *module, uint8 *frame_ip, uint32 *frame_lp, + float64 src_min, float64 src_max, bool saturating, bool is_i32, + bool is_sign) +{ + float64 src_value = GET_OPERAND(float64, F64, 0); + uint64 dst_value_i64; + uint32 dst_value_i32; + + if (!saturating) { + if (isnan(src_value)) { + wasm_set_exception(module, "invalid conversion to integer"); + return false; + } + else if (src_value <= src_min || src_value >= src_max) { + wasm_set_exception(module, "integer overflow"); + return false; + } + } + + if (is_i32) { + uint32 dst_min = is_sign ? INT32_MIN : 0; + uint32 dst_max = is_sign ? INT32_MAX : UINT32_MAX; + dst_value_i32 = trunc_f64_to_i32(src_value, src_min, src_max, dst_min, + dst_max, is_sign); + SET_OPERAND(I32, 2, dst_value_i32); + } + else { + uint64 dst_min = is_sign ? INT64_MIN : 0; + uint64 dst_max = is_sign ? INT64_MAX : UINT64_MAX; + dst_value_i64 = trunc_f64_to_i64(src_value, src_min, src_max, dst_min, + dst_max, is_sign); + SET_OPERAND(I64, 2, dst_value_i64); + } + return true; +} + +#define DEF_OP_TRUNC_F32(min, max, is_i32, is_sign) \ + do { \ + if (!trunc_f32_to_int(module, frame_ip, frame_lp, min, max, false, \ + is_i32, is_sign)) \ + goto got_exception; \ + frame_ip += 4; \ + } while (0) + +#define DEF_OP_TRUNC_F64(min, max, is_i32, is_sign) \ + do { \ + if (!trunc_f64_to_int(module, frame_ip, frame_lp, min, max, false, \ + is_i32, is_sign)) \ + goto got_exception; \ + frame_ip += 4; \ + } while (0) + +#define DEF_OP_TRUNC_SAT_F32(min, max, is_i32, is_sign) \ + do { \ + (void)trunc_f32_to_int(module, frame_ip, frame_lp, min, max, true, \ + is_i32, is_sign); \ + frame_ip += 4; \ + } while (0) + +#define DEF_OP_TRUNC_SAT_F64(min, max, is_i32, is_sign) \ + do { \ + (void)trunc_f64_to_int(module, frame_ip, frame_lp, min, max, true, \ + is_i32, is_sign); \ + frame_ip += 4; \ + } while (0) + +#define DEF_OP_CONVERT(dst_type, dst_op_type, src_type, src_op_type) \ + do { \ + dst_type value = (dst_type)(src_type)POP_##src_op_type(); \ + PUSH_##dst_op_type(value); \ + } while (0) + +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 +#define CELL_SIZE sizeof(uint8) +#else +#define CELL_SIZE (sizeof(uint8) * 2) +#endif + +static bool +copy_stack_values(WASMModuleInstance *module, uint32 *frame_lp, uint32 arity, +#if WASM_ENABLE_GC != 0 + uint8 *frame_ref, +#endif + uint32 total_cell_num, const uint8 *cells, + const int16 *src_offsets, const uint16 *dst_offsets) +{ + /* To avoid the overlap issue between src offsets and dst offset, + * we use 2 steps to do the copy. First step, copy the src values + * to a tmp buf. Second step, copy the values from tmp buf to dst. + */ + bool ret = false; + uint32 buf[16] = { 0 }, i; + uint32 *tmp_buf = buf; + uint8 cell; + int16 src, buf_index = 0; + uint16 dst; +#if WASM_ENABLE_GC != 0 + uint8 ref_buf[4]; + uint8 *tmp_ref_buf = ref_buf; +#endif + + /* Allocate memory if the buf is not large enough */ + if (total_cell_num > sizeof(buf) / sizeof(uint32)) { + uint64 total_size = sizeof(uint32) * (uint64)total_cell_num; + if (total_size >= UINT32_MAX + || !(tmp_buf = wasm_runtime_malloc((uint32)total_size))) { + wasm_set_exception(module, "allocate memory failed"); + goto fail; + } + } + +#if WASM_ENABLE_GC != 0 + if (total_cell_num > sizeof(ref_buf) / sizeof(uint8)) { + uint64 total_size = sizeof(uint8) * (uint64)total_cell_num; + if (total_size >= UINT32_MAX + || !(tmp_ref_buf = wasm_runtime_malloc((uint32)total_size))) { + wasm_set_exception(module, "allocate memory failed"); + goto fail; + } + } +#endif + + /* 1) Copy values from src to tmp buf */ + for (i = 0; i < arity; i++) { + cell = cells[i * CELL_SIZE]; + src = src_offsets[i]; + if (cell == 1) { + tmp_buf[buf_index] = frame_lp[src]; +#if WASM_ENABLE_GC != 0 + tmp_ref_buf[buf_index] = frame_ref[src]; + frame_ref[src] = 0; +#endif + } + else { + tmp_buf[buf_index] = frame_lp[src]; + tmp_buf[buf_index + 1] = frame_lp[src + 1]; +#if WASM_ENABLE_GC != 0 + tmp_ref_buf[buf_index] = frame_ref[src]; + tmp_ref_buf[buf_index + 1] = frame_ref[src + 1]; + frame_ref[src] = 0; + frame_ref[src + 1] = 0; +#endif + } + buf_index += cell; + } + + /* 2) Copy values from tmp buf to dest */ + buf_index = 0; + for (i = 0; i < arity; i++) { + cell = cells[i * CELL_SIZE]; + dst = dst_offsets[i]; + if (cell == 1) { + frame_lp[dst] = tmp_buf[buf_index]; +#if WASM_ENABLE_GC != 0 + frame_ref[dst] = tmp_ref_buf[buf_index]; +#endif + } + else { + frame_lp[dst] = tmp_buf[buf_index]; + frame_lp[dst + 1] = tmp_buf[buf_index + 1]; +#if WASM_ENABLE_GC != 0 + frame_ref[dst] = tmp_ref_buf[buf_index]; + frame_ref[dst + 1] = tmp_ref_buf[buf_index + 1]; +#endif + } + buf_index += cell; + } + + ret = true; + +fail: + if (tmp_buf != buf) { + wasm_runtime_free(tmp_buf); + } + +#if WASM_ENABLE_GC != 0 + if (tmp_ref_buf != ref_buf) { + wasm_runtime_free(tmp_ref_buf); + } +#endif + + return ret; +} + +#if WASM_ENABLE_GC != 0 +#define RECOVER_BR_INFO() \ + do { \ + uint32 arity; \ + /* read arity */ \ + arity = read_uint32(frame_ip); \ + if (arity) { \ + uint32 total_cell; \ + uint16 *dst_offsets = NULL; \ + uint8 *cells; \ + int16 *src_offsets = NULL; \ + /* read total cell num */ \ + total_cell = read_uint32(frame_ip); \ + /* cells */ \ + cells = (uint8 *)frame_ip; \ + frame_ip += arity * CELL_SIZE; \ + /* src offsets */ \ + src_offsets = (int16 *)frame_ip; \ + frame_ip += arity * sizeof(int16); \ + /* dst offsets */ \ + dst_offsets = (uint16 *)frame_ip; \ + frame_ip += arity * sizeof(uint16); \ + if (arity == 1) { \ + if (cells[0] == 1) { \ + frame_lp[dst_offsets[0]] = frame_lp[src_offsets[0]]; \ + /* Ignore constants because they are not reference */ \ + if (src_offsets[0] >= 0) { \ + CLEAR_FRAME_REF((unsigned)(src_offsets[0])); \ + SET_FRAME_REF(dst_offsets[0]); \ + } \ + } \ + else if (cells[0] == 2) { \ + PUT_I64_TO_ADDR( \ + frame_lp + dst_offsets[0], \ + GET_I64_FROM_ADDR(frame_lp + src_offsets[0])); \ + /* Ignore constants because they are not reference */ \ + if (src_offsets[0] >= 0) { \ + CLEAR_FRAME_REF((unsigned)src_offsets[0]); \ + CLEAR_FRAME_REF((unsigned)(src_offsets[0] + 1)); \ + SET_FRAME_REF((unsigned)dst_offsets[0]); \ + SET_FRAME_REF((unsigned)(dst_offsets[0] + 1)); \ + } \ + } \ + else if (cells[0] == 4) { \ + PUT_V128_TO_ADDR( \ + frame_lp + dst_offsets[0], \ + GET_V128_FROM_ADDR(frame_lp + src_offsets[0])); \ + /* Ignore constants because they are not reference */ \ + if (src_offsets[0] >= 0) { \ + CLEAR_FRAME_REF((unsigned)src_offsets[0]); \ + CLEAR_FRAME_REF((unsigned)(src_offsets[0] + 1)); \ + CLEAR_FRAME_REF((unsigned)(src_offsets[0] + 2)); \ + CLEAR_FRAME_REF((unsigned)(src_offsets[0] + 3)); \ + SET_FRAME_REF((unsigned)dst_offsets[0]); \ + SET_FRAME_REF((unsigned)(dst_offsets[0] + 1)); \ + SET_FRAME_REF((unsigned)(dst_offsets[0] + 2)); \ + SET_FRAME_REF((unsigned)(dst_offsets[0] + 3)); \ + } \ + } \ + } \ + else { \ + if (!copy_stack_values(module, frame_lp, arity, frame_ref, \ + total_cell, cells, src_offsets, \ + dst_offsets)) \ + goto got_exception; \ + } \ + } \ + frame_ip = (uint8 *)LOAD_PTR(frame_ip); \ + } while (0) +#else +#define RECOVER_BR_INFO() \ + do { \ + uint32 arity; \ + /* read arity */ \ + arity = read_uint32(frame_ip); \ + if (arity) { \ + uint32 total_cell; \ + uint16 *dst_offsets = NULL; \ + uint8 *cells; \ + int16 *src_offsets = NULL; \ + /* read total cell num */ \ + total_cell = read_uint32(frame_ip); \ + /* cells */ \ + cells = (uint8 *)frame_ip; \ + frame_ip += arity * CELL_SIZE; \ + /* src offsets */ \ + src_offsets = (int16 *)frame_ip; \ + frame_ip += arity * sizeof(int16); \ + /* dst offsets */ \ + dst_offsets = (uint16 *)frame_ip; \ + frame_ip += arity * sizeof(uint16); \ + if (arity == 1) { \ + if (cells[0] == 1) \ + frame_lp[dst_offsets[0]] = frame_lp[src_offsets[0]]; \ + else if (cells[0] == 2) { \ + PUT_I64_TO_ADDR( \ + frame_lp + dst_offsets[0], \ + GET_I64_FROM_ADDR(frame_lp + src_offsets[0])); \ + } \ + else if (cells[0] == 4) { \ + PUT_V128_TO_ADDR( \ + frame_lp + dst_offsets[0], \ + GET_V128_FROM_ADDR(frame_lp + src_offsets[0])); \ + } \ + } \ + else { \ + if (!copy_stack_values(module, frame_lp, arity, total_cell, \ + cells, src_offsets, dst_offsets)) \ + goto got_exception; \ + } \ + } \ + frame_ip = (uint8 *)LOAD_PTR(frame_ip); \ + } while (0) +#endif + +#define SKIP_BR_INFO() \ + do { \ + uint32 arity; \ + /* read and skip arity */ \ + arity = read_uint32(frame_ip); \ + if (arity) { \ + /* skip total cell num */ \ + frame_ip += sizeof(uint32); \ + /* skip cells, src offsets and dst offsets */ \ + frame_ip += (CELL_SIZE + sizeof(int16) + sizeof(uint16)) * arity; \ + } \ + /* skip target address */ \ + frame_ip += sizeof(uint8 *); \ + } while (0) + +static inline int32 +sign_ext_8_32(int8 val) +{ + if (val & 0x80) + return (int32)val | (int32)0xffffff00; + return val; +} + +static inline int32 +sign_ext_16_32(int16 val) +{ + if (val & 0x8000) + return (int32)val | (int32)0xffff0000; + return val; +} + +static inline int64 +sign_ext_8_64(int8 val) +{ + if (val & 0x80) + return (int64)val | (int64)0xffffffffffffff00LL; + return val; +} + +static inline int64 +sign_ext_16_64(int16 val) +{ + if (val & 0x8000) + return (int64)val | (int64)0xffffffffffff0000LL; + return val; +} + +static inline int64 +sign_ext_32_64(int32 val) +{ + if (val & (int32)0x80000000) + return (int64)val | (int64)0xffffffff00000000LL; + return val; +} + +static inline void +word_copy(uint32 *dest, uint32 *src, unsigned num) +{ + bh_assert(dest != NULL); + bh_assert(src != NULL); + bh_assert(num > 0); + if (dest != src) { + /* No overlap buffer */ + bh_assert(!((src < dest) && (dest < src + num))); + for (; num > 0; num--) + *dest++ = *src++; + } +} + +static inline WASMInterpFrame * +ALLOC_FRAME(WASMExecEnv *exec_env, uint32 size, WASMInterpFrame *prev_frame) +{ + WASMInterpFrame *frame = wasm_exec_env_alloc_wasm_frame(exec_env, size); + + if (frame) { + frame->prev_frame = prev_frame; +#if WASM_ENABLE_PERF_PROFILING != 0 + frame->time_started = os_time_thread_cputime_us(); +#endif + } + else { + wasm_set_exception((WASMModuleInstance *)exec_env->module_inst, + "wasm operand stack overflow"); + } + + return frame; +} + +static inline void +FREE_FRAME(WASMExecEnv *exec_env, WASMInterpFrame *frame) +{ +#if WASM_ENABLE_PERF_PROFILING != 0 + if (frame->function) { + WASMInterpFrame *prev_frame = frame->prev_frame; + uint64 time_elapsed = os_time_thread_cputime_us() - frame->time_started; + + frame->function->total_exec_time += time_elapsed; + frame->function->total_exec_cnt++; + + /* parent function */ + if (prev_frame && prev_frame->function) + prev_frame->function->children_exec_time += time_elapsed; + } +#endif + wasm_exec_env_free_wasm_frame(exec_env, frame); +} + +static void +wasm_interp_call_func_native(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, + WASMFunctionInstance *cur_func, + WASMInterpFrame *prev_frame) +{ + WASMFunctionImport *func_import = cur_func->u.func_import; + CApiFuncImport *c_api_func_import = NULL; + unsigned local_cell_num = + cur_func->param_cell_num > 2 ? cur_func->param_cell_num : 2; + unsigned all_cell_num; + WASMInterpFrame *frame; + uint32 argv_ret[2], cur_func_index; + void *native_func_pointer = NULL; + bool ret = false; +#if WASM_ENABLE_GC != 0 + WASMFuncType *func_type; + uint8 *frame_ref; +#endif + + all_cell_num = local_cell_num; +#if WASM_ENABLE_GC != 0 + all_cell_num += (local_cell_num + 3) / 4; +#endif + + if (!wasm_runtime_detect_native_stack_overflow(exec_env)) { + return; + } + + if (!(frame = + ALLOC_FRAME(exec_env, wasm_interp_interp_frame_size(all_cell_num), + prev_frame))) + return; + + frame->function = cur_func; + frame->ip = NULL; + frame->lp = frame->operand; +#if WASM_ENABLE_GC != 0 + frame->frame_ref = (uint8 *)(frame->lp + local_cell_num); + init_frame_refs(frame->frame_ref, local_cell_num, cur_func); +#endif + + wasm_exec_env_set_cur_frame(exec_env, frame); + + cur_func_index = (uint32)(cur_func - module_inst->e->functions); + bh_assert(cur_func_index < module_inst->module->import_function_count); + if (!func_import->call_conv_wasm_c_api) { + native_func_pointer = module_inst->import_func_ptrs[cur_func_index]; + } + else if (module_inst->c_api_func_imports) { + c_api_func_import = module_inst->c_api_func_imports + cur_func_index; + native_func_pointer = c_api_func_import->func_ptr_linked; + } + + if (!native_func_pointer) { + char buf[128]; + snprintf(buf, sizeof(buf), + "failed to call unlinked import function (%s, %s)", + func_import->module_name, func_import->field_name); + wasm_set_exception((WASMModuleInstance *)module_inst, buf); + return; + } + + if (func_import->call_conv_wasm_c_api) { + ret = wasm_runtime_invoke_c_api_native( + (WASMModuleInstanceCommon *)module_inst, native_func_pointer, + func_import->func_type, cur_func->param_cell_num, frame->lp, + c_api_func_import->with_env_arg, c_api_func_import->env_arg); + if (ret) { + argv_ret[0] = frame->lp[0]; + argv_ret[1] = frame->lp[1]; + } + } +#if WASM_ENABLE_INVOKE_NATIVE != 0 + else if (!func_import->call_conv_raw) { + ret = wasm_runtime_invoke_native( + exec_env, native_func_pointer, func_import->func_type, + func_import->signature, func_import->attachment, frame->lp, + cur_func->param_cell_num, argv_ret); + } + else { + ret = wasm_runtime_invoke_native_raw( + exec_env, native_func_pointer, func_import->func_type, + func_import->signature, func_import->attachment, frame->lp, + cur_func->param_cell_num, argv_ret); + } +#endif + + if (!ret) + return; + +#if WASM_ENABLE_GC != 0 + func_type = cur_func->u.func_import->func_type; + if (func_type->result_count + && wasm_is_type_reftype(func_type->types[cur_func->param_count]) + && !wasm_is_reftype_i31ref(func_type->types[cur_func->param_count])) { + frame_ref = prev_frame->frame_ref + prev_frame->ret_offset; +#if UINTPTR_MAX == UINT64_MAX + *frame_ref = *(frame_ref + 1) = 1; +#else + *frame_ref = 1; +#endif + } +#endif + + if (cur_func->ret_cell_num == 1) { + prev_frame->lp[prev_frame->ret_offset] = argv_ret[0]; + } + else if (cur_func->ret_cell_num == 2) { + prev_frame->lp[prev_frame->ret_offset] = argv_ret[0]; + prev_frame->lp[prev_frame->ret_offset + 1] = argv_ret[1]; + } + + FREE_FRAME(exec_env, frame); + wasm_exec_env_set_cur_frame(exec_env, prev_frame); +} + +#if WASM_ENABLE_MULTI_MODULE != 0 +static void +wasm_interp_call_func_bytecode(WASMModuleInstance *module, + WASMExecEnv *exec_env, + WASMFunctionInstance *cur_func, + WASMInterpFrame *prev_frame); + +static void +wasm_interp_call_func_import(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, + WASMFunctionInstance *cur_func, + WASMInterpFrame *prev_frame) +{ + WASMModuleInstance *sub_module_inst = cur_func->import_module_inst; + WASMFunctionInstance *sub_func_inst = cur_func->import_func_inst; + WASMFunctionImport *func_import = cur_func->u.func_import; + uint8 *ip = prev_frame->ip; + char buf[128]; + WASMExecEnv *sub_module_exec_env = NULL; + uintptr_t aux_stack_origin_boundary = 0; + uintptr_t aux_stack_origin_bottom = 0; + + /* + * perform stack overflow check before calling + * wasm_interp_call_func_bytecode recursively. + */ + if (!wasm_runtime_detect_native_stack_overflow(exec_env)) { + return; + } + + if (!sub_func_inst) { + snprintf(buf, sizeof(buf), + "failed to call unlinked import function (%s, %s)", + func_import->module_name, func_import->field_name); + wasm_set_exception(module_inst, buf); + return; + } + + /* Switch exec_env but keep using the same one by replacing necessary + * variables */ + sub_module_exec_env = wasm_runtime_get_exec_env_singleton( + (WASMModuleInstanceCommon *)sub_module_inst); + if (!sub_module_exec_env) { + wasm_set_exception(module_inst, "create singleton exec_env failed"); + return; + } + + /* - module_inst */ + wasm_exec_env_set_module_inst(exec_env, + (WASMModuleInstanceCommon *)sub_module_inst); + /* - aux_stack_boundary */ + aux_stack_origin_boundary = exec_env->aux_stack_boundary; + exec_env->aux_stack_boundary = sub_module_exec_env->aux_stack_boundary; + /* - aux_stack_bottom */ + aux_stack_origin_bottom = exec_env->aux_stack_bottom; + exec_env->aux_stack_bottom = sub_module_exec_env->aux_stack_bottom; + + /* set ip NULL to make call_func_bytecode return after executing + this function */ + prev_frame->ip = NULL; + + /* call function of sub-module*/ + wasm_interp_call_func_bytecode(sub_module_inst, exec_env, sub_func_inst, + prev_frame); + + /* restore ip and other replaced */ + prev_frame->ip = ip; + exec_env->aux_stack_boundary = aux_stack_origin_boundary; + exec_env->aux_stack_bottom = aux_stack_origin_bottom; + wasm_exec_env_restore_module_inst(exec_env, + (WASMModuleInstanceCommon *)module_inst); +} +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 +#define CHECK_SUSPEND_FLAGS() \ + do { \ + WASM_SUSPEND_FLAGS_LOCK(exec_env->wait_lock); \ + if (WASM_SUSPEND_FLAGS_GET(exec_env->suspend_flags) \ + & WASM_SUSPEND_FLAG_TERMINATE) { \ + /* terminate current thread */ \ + WASM_SUSPEND_FLAGS_UNLOCK(exec_env->wait_lock); \ + return; \ + } \ + /* TODO: support suspend and breakpoint */ \ + WASM_SUSPEND_FLAGS_UNLOCK(exec_env->wait_lock); \ + } while (0) +#endif + +#if WASM_ENABLE_OPCODE_COUNTER != 0 +typedef struct OpcodeInfo { + char *name; + uint64 count; +} OpcodeInfo; + +/* clang-format off */ +#define HANDLE_OPCODE(op) \ + { \ + #op, 0 \ + } +DEFINE_GOTO_TABLE(OpcodeInfo, opcode_table); +#undef HANDLE_OPCODE +/* clang-format on */ + +static void +wasm_interp_dump_op_count() +{ + uint32 i; + uint64 total_count = 0; + for (i = 0; i < WASM_OP_IMPDEP; i++) + total_count += opcode_table[i].count; + + os_printf("total opcode count: %ld\n", total_count); + for (i = 0; i < WASM_OP_IMPDEP; i++) + if (opcode_table[i].count > 0) + os_printf("\t\t%s count:\t\t%ld,\t\t%.2f%%\n", opcode_table[i].name, + opcode_table[i].count, + opcode_table[i].count * 100.0f / total_count); +} +#endif + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 + +/* #define HANDLE_OP(opcode) HANDLE_##opcode:printf(#opcode"\n"); */ +#if WASM_ENABLE_OPCODE_COUNTER != 0 +#define HANDLE_OP(opcode) HANDLE_##opcode : opcode_table[opcode].count++; +#else +#define HANDLE_OP(opcode) HANDLE_##opcode: +#endif +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 +#define FETCH_OPCODE_AND_DISPATCH() \ + do { \ + const void *p_label_addr = *(void **)frame_ip; \ + frame_ip += sizeof(void *); \ + CHECK_INSTRUCTION_LIMIT(); \ + goto *p_label_addr; \ + } while (0) +#else +#if UINTPTR_MAX == UINT64_MAX +#define FETCH_OPCODE_AND_DISPATCH() \ + do { \ + const void *p_label_addr; \ + bh_assert(((uintptr_t)frame_ip & 1) == 0); \ + /* int32 relative offset was emitted in 64-bit target */ \ + p_label_addr = label_base + (int32)LOAD_U32_WITH_2U16S(frame_ip); \ + frame_ip += sizeof(int32); \ + CHECK_INSTRUCTION_LIMIT(); \ + goto *p_label_addr; \ + } while (0) +#else +#define FETCH_OPCODE_AND_DISPATCH() \ + do { \ + const void *p_label_addr; \ + bh_assert(((uintptr_t)frame_ip & 1) == 0); \ + /* uint32 label address was emitted in 32-bit target */ \ + p_label_addr = (void *)(uintptr_t)LOAD_U32_WITH_2U16S(frame_ip); \ + frame_ip += sizeof(int32); \ + CHECK_INSTRUCTION_LIMIT(); \ + goto *p_label_addr; \ + } while (0) +#endif +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#define HANDLE_OP_END() FETCH_OPCODE_AND_DISPATCH() + +#else /* else of WASM_ENABLE_LABELS_AS_VALUES */ + +#define HANDLE_OP(opcode) case opcode: +#define HANDLE_OP_END() continue + +#endif /* end of WASM_ENABLE_LABELS_AS_VALUES */ + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +static void **global_handle_table; +#endif + +static inline uint8 * +get_global_addr(uint8 *global_data, WASMGlobalInstance *global) +{ +#if WASM_ENABLE_MULTI_MODULE == 0 + return global_data + global->data_offset; +#else + return global->import_global_inst + ? global->import_module_inst->global_data + + global->import_global_inst->data_offset + : global_data + global->data_offset; +#endif +} + +static void +wasm_interp_call_func_bytecode(WASMModuleInstance *module, + WASMExecEnv *exec_env, + WASMFunctionInstance *cur_func, + WASMInterpFrame *prev_frame) +{ + WASMMemoryInstance *memory = wasm_get_default_memory(module); +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY_OPT != 0 + uint64 linear_mem_size = 0; + if (memory) +#if WASM_ENABLE_THREAD_MGR == 0 + linear_mem_size = memory->memory_data_size; +#else + linear_mem_size = GET_LINEAR_MEMORY_SIZE(memory); +#endif +#endif + WASMGlobalInstance *globals = module->e ? module->e->globals : NULL; + WASMGlobalInstance *global; + uint8 *global_data = module->global_data; + uint8 opcode_IMPDEP = WASM_OP_IMPDEP; + WASMInterpFrame *frame = NULL; + /* Points to this special opcode so as to jump to the + * call_method_from_entry. */ + register uint8 *frame_ip = &opcode_IMPDEP; /* cache of frame->ip */ + register uint32 *frame_lp = NULL; /* cache of frame->lp */ +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 && UINTPTR_MAX == UINT64_MAX + /* cache of label base addr */ + register uint8 *label_base = &&HANDLE_WASM_OP_UNREACHABLE; +#endif +#endif +#if WASM_ENABLE_GC != 0 + register uint8 *frame_ref = NULL; /* cache of frame->ref */ + uint32 local_cell_num = 0; + int16 opnd_off; +#endif + uint8 *frame_ip_end = frame_ip + 1; + uint32 cond, count, fidx, tidx, frame_size = 0; + uint32 all_cell_num = 0; + int16 addr1, addr2, addr_ret = 0; + int32 didx, val; + uint8 *maddr = NULL; + uint32 local_idx, local_offset, global_idx; + uint8 opcode = 0, local_type, *global_addr; + +#if WASM_ENABLE_INSTRUCTION_METERING != 0 + int instructions_left = -1; + if (exec_env) { + instructions_left = exec_env->instructions_to_execute; + } +#endif +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 + bool disable_bounds_checks = !wasm_runtime_is_bounds_checks_enabled( + (WASMModuleInstanceCommon *)module); +#else + bool disable_bounds_checks = false; +#endif +#endif +#if WASM_ENABLE_GC != 0 + WASMObjectRef gc_obj; + WASMStructObjectRef struct_obj; + WASMArrayObjectRef array_obj; + WASMFuncObjectRef func_obj; + WASMI31ObjectRef i31_obj; + WASMExternrefObjectRef externref_obj; + uint32 type_idx; +#if WASM_ENABLE_STRINGREF != 0 + WASMString str_obj; + WASMStringrefObjectRef stringref_obj; + WASMStringviewWTF8ObjectRef stringview_wtf8_obj; + WASMStringviewWTF16ObjectRef stringview_wtf16_obj; + WASMStringviewIterObjectRef stringview_iter_obj; +#endif +#endif +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + bool is_return_call = false; +#endif +#if WASM_ENABLE_SHARED_HEAP != 0 + /* TODO: currently flowing two variables are only dummy for shared heap + * boundary check, need to be updated when multi-memory or memory64 + * proposals are to be implemented */ + bool is_memory64 = false; + uint32 memidx = 0; + (void)is_memory64; + (void)memidx; +/* #endif */ +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#define HANDLE_OPCODE(op) &&HANDLE_##op + DEFINE_GOTO_TABLE(const void *, handle_table); +#undef HANDLE_OPCODE + if (exec_env == NULL) { + global_handle_table = (void **)handle_table; + return; + } +#endif + +#if WASM_ENABLE_LABELS_AS_VALUES == 0 + while (frame_ip < frame_ip_end) { + opcode = *frame_ip++; +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + frame_ip++; +#endif + switch (opcode) { +#else + goto *handle_table[WASM_OP_IMPDEP]; +#endif + /* control instructions */ + HANDLE_OP(WASM_OP_UNREACHABLE) + { + wasm_set_exception(module, "unreachable"); + goto got_exception; + } + + HANDLE_OP(WASM_OP_IF) + { + cond = (uint32)POP_I32(); + + if (cond == 0) { + uint8 *else_addr = (uint8 *)LOAD_PTR(frame_ip); + if (else_addr == NULL) { + frame_ip = + (uint8 *)LOAD_PTR(frame_ip + sizeof(uint8 *)); + } + else { + frame_ip = else_addr; + } + } + else { + frame_ip += sizeof(uint8 *) * 2; + } + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_ELSE) + { + frame_ip = (uint8 *)LOAD_PTR(frame_ip); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_BR) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + recover_br_info: + RECOVER_BR_INFO(); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_BR_IF) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + cond = frame_lp[GET_OFFSET()]; + + if (cond) + goto recover_br_info; + else + SKIP_BR_INFO(); + + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_BR_TABLE) + { + uint32 arity, br_item_size; + +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + count = read_uint32(frame_ip); + didx = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + + if (!(didx >= 0 && (uint32)didx < count)) + didx = count; + + /* all br items must have the same arity and item size, + so we only calculate the first item size */ + arity = LOAD_U32_WITH_2U16S(frame_ip); + br_item_size = sizeof(uint32); /* arity */ + if (arity) { + /* total cell num */ + br_item_size += sizeof(uint32); + /* cells, src offsets and dst offsets */ + br_item_size += + (CELL_SIZE + sizeof(int16) + sizeof(uint16)) * arity; + } + /* target address */ + br_item_size += sizeof(uint8 *); + + frame_ip += br_item_size * didx; + goto recover_br_info; + } + + HANDLE_OP(WASM_OP_RETURN) + { + uint32 ret_idx; + WASMFuncType *func_type; + int32 off; + uint32 ret_offset; + uint8 *ret_types; + if (cur_func->is_import_func) + func_type = cur_func->u.func_import->func_type; + else + func_type = cur_func->u.func->func_type; + + /* types of each return value */ + ret_types = func_type->types + func_type->param_count; + ret_offset = prev_frame->ret_offset; + + for (ret_idx = 0, + off = (int32)sizeof(int16) * (func_type->result_count - 1); + ret_idx < func_type->result_count; + ret_idx++, off -= (int32)sizeof(int16)) { + if (ret_types[ret_idx] == VALUE_TYPE_I64 + || ret_types[ret_idx] == VALUE_TYPE_F64) { + PUT_I64_TO_ADDR(prev_frame->lp + ret_offset, + GET_OPERAND(uint64, I64, off)); + ret_offset += 2; + } + else if (ret_types[ret_idx] == VALUE_TYPE_V128) { + PUT_V128_TO_ADDR(prev_frame->lp + ret_offset, + GET_OPERAND_V128(off)); + ret_offset += 4; + } +#if WASM_ENABLE_GC != 0 + else if (wasm_is_type_reftype(ret_types[ret_idx])) { + PUT_REF_TO_ADDR(prev_frame->lp + ret_offset, + GET_OPERAND(void *, REF, off)); + if (!wasm_is_reftype_i31ref(ret_types[ret_idx])) { + *(prev_frame->frame_ref + ret_offset) = 1; +#if UINTPTR_MAX == UINT64_MAX + *(prev_frame->frame_ref + ret_offset + 1) = 1; +#endif + } + ret_offset += REF_CELL_NUM; + } +#endif + else { + prev_frame->lp[ret_offset] = + GET_OPERAND(uint32, I32, off); + ret_offset++; + } + } + goto return_func; + } + + HANDLE_OP(WASM_OP_CALL_INDIRECT) +#if WASM_ENABLE_TAIL_CALL != 0 + HANDLE_OP(WASM_OP_RETURN_CALL_INDIRECT) +#endif + { + WASMFuncType *cur_type, *cur_func_type; + WASMTableInstance *tbl_inst; + uint32 tbl_idx; + +#if WASM_ENABLE_TAIL_CALL != 0 + GET_OPCODE(); +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + + tidx = read_uint32(frame_ip); + cur_type = (WASMFuncType *)module->module->types[tidx]; + + tbl_idx = read_uint32(frame_ip); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + val = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + + if ((uint32)val >= tbl_inst->cur_size) { + wasm_set_exception(module, "undefined element"); + goto got_exception; + } + + /* clang-format off */ +#if WASM_ENABLE_GC == 0 + fidx = (uint32)tbl_inst->elems[val]; + if (fidx == (uint32)-1) { + wasm_set_exception(module, "uninitialized element"); + goto got_exception; + } +#else + func_obj = (WASMFuncObjectRef)tbl_inst->elems[val]; + if (!func_obj) { + wasm_set_exception(module, "uninitialized element"); + goto got_exception; + } + fidx = wasm_func_obj_get_func_idx_bound(func_obj); +#endif + /* clang-format on */ + + /* + * we might be using a table injected by host or + * another module. in that case, we don't validate + * the elem value while loading + */ + if (fidx >= module->e->function_count) { + wasm_set_exception(module, "unknown function"); + goto got_exception; + } + + /* always call module own functions */ + cur_func = module->e->functions + fidx; + + if (cur_func->is_import_func) + cur_func_type = cur_func->u.func_import->func_type; + else + cur_func_type = cur_func->u.func->func_type; + + /* clang-format off */ +#if WASM_ENABLE_GC == 0 + if (cur_type != cur_func_type) { + wasm_set_exception(module, "indirect call type mismatch"); + goto got_exception; + } +#else + if (!wasm_func_type_is_super_of(cur_type, cur_func_type)) { + wasm_set_exception(module, "indirect call type mismatch"); + goto got_exception; + } +#endif + /* clang-format on */ + +#if WASM_ENABLE_TAIL_CALL != 0 + if (opcode == WASM_OP_RETURN_CALL_INDIRECT) + goto call_func_from_return_call; +#endif + goto call_func_from_interp; + } + +#if WASM_ENABLE_EXCE_HANDLING != 0 + HANDLE_OP(WASM_OP_TRY) + HANDLE_OP(WASM_OP_CATCH) + HANDLE_OP(WASM_OP_THROW) + HANDLE_OP(WASM_OP_RETHROW) + HANDLE_OP(WASM_OP_DELEGATE) + HANDLE_OP(WASM_OP_CATCH_ALL) + HANDLE_OP(EXT_OP_TRY) + { + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } +#endif + + /* parametric instructions */ + HANDLE_OP(WASM_OP_SELECT) + { + cond = frame_lp[GET_OFFSET()]; + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + addr_ret = GET_OFFSET(); + + if (!cond) { + if (addr_ret != addr1) + frame_lp[addr_ret] = frame_lp[addr1]; + } + else { + if (addr_ret != addr2) + frame_lp[addr_ret] = frame_lp[addr2]; + } + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_SELECT_64) + { + cond = frame_lp[GET_OFFSET()]; + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + addr_ret = GET_OFFSET(); + + if (!cond) { + if (addr_ret != addr1) + PUT_I64_TO_ADDR(frame_lp + addr_ret, + GET_I64_FROM_ADDR(frame_lp + addr1)); + } + else { + if (addr_ret != addr2) + PUT_I64_TO_ADDR(frame_lp + addr_ret, + GET_I64_FROM_ADDR(frame_lp + addr2)); + } + HANDLE_OP_END(); + } +#if WASM_ENABLE_SIMDE != 0 + HANDLE_OP(WASM_OP_SELECT_128) + { + cond = frame_lp[GET_OFFSET()]; + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + addr_ret = GET_OFFSET(); + + if (!cond) { + if (addr_ret != addr1) + PUT_V128_TO_ADDR(frame_lp + addr_ret, + GET_V128_FROM_ADDR(frame_lp + addr1)); + } + else { + if (addr_ret != addr2) + PUT_V128_TO_ADDR(frame_lp + addr_ret, + GET_V128_FROM_ADDR(frame_lp + addr2)); + } + HANDLE_OP_END(); + } +#endif + +#if WASM_ENABLE_GC != 0 + HANDLE_OP(WASM_OP_SELECT_T) + { + cond = frame_lp[GET_OFFSET()]; + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + addr_ret = GET_OFFSET(); + + if (!cond) { + if (addr_ret != addr1) + PUT_REF_TO_ADDR(frame_lp + addr_ret, + GET_REF_FROM_ADDR(frame_lp + addr1)); + } + else { + if (addr_ret != addr2) + PUT_REF_TO_ADDR(frame_lp + addr_ret, + GET_REF_FROM_ADDR(frame_lp + addr2)); + } + { + uint8 orig_ref = 0; + /* Ignore constants because they are not reference */ + if (addr1 >= 0) { + orig_ref = *FRAME_REF(addr1); + CLEAR_FRAME_REF(addr1); + } + if (addr2 >= 0) { + CLEAR_FRAME_REF(addr2); + } + if (orig_ref) { + SET_FRAME_REF(addr_ret); + } + } + + HANDLE_OP_END(); + } +#endif + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + HANDLE_OP(WASM_OP_TABLE_GET) + { + uint32 tbl_idx, elem_idx; + WASMTableInstance *tbl_inst; + + tbl_idx = read_uint32(frame_ip); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + elem_idx = POP_I32(); + if (elem_idx >= tbl_inst->cur_size) { + wasm_set_exception(module, "out of bounds table access"); + goto got_exception; + } + +#if WASM_ENABLE_GC == 0 + PUSH_I32(tbl_inst->elems[elem_idx]); +#else + PUSH_REF(tbl_inst->elems[elem_idx]); +#endif + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_TABLE_SET) + { + uint32 tbl_idx, elem_idx; + WASMTableInstance *tbl_inst; + table_elem_type_t elem_val; + + tbl_idx = read_uint32(frame_ip); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + +#if WASM_ENABLE_GC == 0 + elem_val = POP_I32(); +#else + elem_val = POP_REF(); +#endif + elem_idx = POP_I32(); + if (elem_idx >= tbl_inst->cur_size) { + wasm_set_exception(module, "out of bounds table access"); + goto got_exception; + } + + tbl_inst->elems[elem_idx] = elem_val; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_REF_NULL) + { +#if WASM_ENABLE_GC == 0 + PUSH_I32(NULL_REF); +#else + PUSH_REF(NULL_REF); +#endif + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_REF_IS_NULL) + { +#if WASM_ENABLE_GC == 0 + uint32 ref_val; + ref_val = POP_I32(); +#else + void *ref_val; + ref_val = POP_REF(); +#endif + PUSH_I32(ref_val == NULL_REF ? 1 : 0); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_REF_FUNC) + { + uint32 func_idx = read_uint32(frame_ip); + +#if WASM_ENABLE_GC == 0 + PUSH_I32(func_idx); +#else + SYNC_ALL_TO_FRAME(); + if (!(gc_obj = wasm_create_func_obj(module, func_idx, true, + NULL, 0))) { + goto got_exception; + } + PUSH_REF(gc_obj); +#endif + HANDLE_OP_END(); + } +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_GC != 0 + HANDLE_OP(WASM_OP_CALL_REF) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + func_obj = POP_REF(); + if (!func_obj) { + wasm_set_exception(module, "null function reference"); + goto got_exception; + } + + fidx = wasm_func_obj_get_func_idx_bound(func_obj); + cur_func = module->e->functions + fidx; + goto call_func_from_interp; + } + HANDLE_OP(WASM_OP_RETURN_CALL_REF) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + func_obj = POP_REF(); + if (!func_obj) { + wasm_set_exception(module, "null function reference"); + goto got_exception; + } + + fidx = wasm_func_obj_get_func_idx_bound(func_obj); + cur_func = module->e->functions + fidx; + goto call_func_from_return_call; + } + HANDLE_OP(WASM_OP_REF_AS_NON_NULL) + { + gc_obj = POP_REF(); + if (gc_obj == NULL_REF) { + wasm_set_exception(module, "null reference"); + goto got_exception; + } + PUSH_REF(gc_obj); + HANDLE_OP_END(); + } + HANDLE_OP(WASM_OP_REF_EQ) + { + WASMObjectRef gc_obj1, gc_obj2; + gc_obj2 = POP_REF(); + gc_obj1 = POP_REF(); + val = wasm_obj_equal(gc_obj1, gc_obj2); + PUSH_I32(val); + HANDLE_OP_END(); + } + HANDLE_OP(WASM_OP_BR_ON_NULL) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + opnd_off = GET_OFFSET(); + gc_obj = GET_REF_FROM_ADDR(frame_lp + opnd_off); + if (gc_obj == NULL_REF) { + CLEAR_FRAME_REF(opnd_off); + goto recover_br_info; + } + else { + SKIP_BR_INFO(); + } + HANDLE_OP_END(); + } + HANDLE_OP(WASM_OP_BR_ON_NON_NULL) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + opnd_off = GET_OFFSET(); + gc_obj = GET_REF_FROM_ADDR(frame_lp + opnd_off); + if (gc_obj != NULL_REF) { + goto recover_br_info; + } + else { + CLEAR_FRAME_REF(opnd_off); + SKIP_BR_INFO(); + } + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_GC_PREFIX) + { + GET_OPCODE(); + + switch (opcode) { + case WASM_OP_STRUCT_NEW: + case WASM_OP_STRUCT_NEW_DEFAULT: + { + WASMModule *wasm_module = module->module; + WASMStructType *struct_type; + WASMRttType *rtt_type; + WASMValue field_value = { 0 }; + + type_idx = read_uint32(frame_ip); + struct_type = + (WASMStructType *)module->module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)struct_type, type_idx, + wasm_module->rtt_types, + wasm_module->type_count, + &wasm_module->rtt_type_lock))) { + wasm_set_exception(module, + "create rtt type failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + struct_obj = wasm_struct_obj_new(exec_env, rtt_type); + if (!struct_obj) { + wasm_set_exception(module, + "create struct object failed"); + goto got_exception; + } + + if (opcode == WASM_OP_STRUCT_NEW) { + WASMStructFieldType *fields = struct_type->fields; + int32 field_count = (int32)struct_type->field_count; + int32 field_idx; + uint8 field_type; + + for (field_idx = field_count - 1; field_idx >= 0; + field_idx--) { + field_type = fields[field_idx].field_type; + if (wasm_is_type_reftype(field_type)) { + field_value.gc_obj = POP_REF(); + } + else if (field_type == VALUE_TYPE_I32 + || field_type == VALUE_TYPE_F32 + || field_type == PACKED_TYPE_I8 + || field_type == PACKED_TYPE_I16) { + field_value.i32 = POP_I32(); + } + else { + field_value.i64 = POP_I64(); + } + wasm_struct_obj_set_field(struct_obj, field_idx, + &field_value); + } + } + PUSH_REF(struct_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRUCT_GET: + case WASM_OP_STRUCT_GET_S: + case WASM_OP_STRUCT_GET_U: + { + WASMStructType *struct_type; + WASMValue field_value = { 0 }; + uint32 field_idx; + uint8 field_type; + + type_idx = read_uint32(frame_ip); + field_idx = read_uint32(frame_ip); + + struct_type = + (WASMStructType *)module->module->types[type_idx]; + + struct_obj = POP_REF(); + + if (!struct_obj) { + wasm_set_exception(module, + "null structure reference"); + goto got_exception; + } + + wasm_struct_obj_get_field( + struct_obj, field_idx, + opcode == WASM_OP_STRUCT_GET_S ? true : false, + &field_value); + + field_type = struct_type->fields[field_idx].field_type; + if (wasm_is_reftype_i31ref(field_type)) { + PUSH_I31REF(field_value.gc_obj); + } + else if (wasm_is_type_reftype(field_type)) { + PUSH_REF(field_value.gc_obj); + } + else if (field_type == VALUE_TYPE_I32 + || field_type == VALUE_TYPE_F32 + || field_type == PACKED_TYPE_I8 + || field_type == PACKED_TYPE_I16) { + PUSH_I32(field_value.i32); + } + else { + PUSH_I64(field_value.i64); + } + HANDLE_OP_END(); + } + case WASM_OP_STRUCT_SET: + { + WASMStructType *struct_type; + WASMValue field_value = { 0 }; + uint32 field_idx; + uint8 field_type; + + type_idx = read_uint32(frame_ip); + field_idx = read_uint32(frame_ip); + + struct_type = + (WASMStructType *)module->module->types[type_idx]; + field_type = struct_type->fields[field_idx].field_type; + + if (wasm_is_type_reftype(field_type)) { + field_value.gc_obj = POP_REF(); + } + else if (field_type == VALUE_TYPE_I32 + || field_type == VALUE_TYPE_F32 + || field_type == PACKED_TYPE_I8 + || field_type == PACKED_TYPE_I16) { + field_value.i32 = POP_I32(); + } + else { + field_value.i64 = POP_I64(); + } + + struct_obj = POP_REF(); + if (!struct_obj) { + wasm_set_exception(module, + "null structure reference"); + goto got_exception; + } + + wasm_struct_obj_set_field(struct_obj, field_idx, + &field_value); + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_NEW: + case WASM_OP_ARRAY_NEW_DEFAULT: + case WASM_OP_ARRAY_NEW_FIXED: + { + WASMModule *wasm_module = module->module; + WASMArrayType *array_type; + WASMRttType *rtt_type; + WASMValue array_elem = { 0 }; + uint32 array_len, i; + + type_idx = read_uint32(frame_ip); + array_type = + (WASMArrayType *)wasm_module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)array_type, type_idx, + wasm_module->rtt_types, + wasm_module->type_count, + &wasm_module->rtt_type_lock))) { + wasm_set_exception(module, + "create rtt type failed"); + goto got_exception; + } + + if (opcode != WASM_OP_ARRAY_NEW_FIXED) + array_len = POP_I32(); + else + array_len = read_uint32(frame_ip); + + if (opcode == WASM_OP_ARRAY_NEW) { + if (wasm_is_type_reftype(array_type->elem_type)) { + array_elem.gc_obj = POP_REF(); + } + else if (array_type->elem_type == VALUE_TYPE_I32 + || array_type->elem_type == VALUE_TYPE_F32 + || array_type->elem_type == PACKED_TYPE_I8 + || array_type->elem_type + == PACKED_TYPE_I16) { + array_elem.i32 = POP_I32(); + } + else { + array_elem.i64 = POP_I64(); + } + } + + SYNC_ALL_TO_FRAME(); + array_obj = wasm_array_obj_new(exec_env, rtt_type, + array_len, &array_elem); + if (!array_obj) { + wasm_set_exception(module, + "create array object failed"); + goto got_exception; + } + + if (opcode == WASM_OP_ARRAY_NEW_FIXED) { + for (i = 0; i < array_len; i++) { + if (wasm_is_type_reftype( + array_type->elem_type)) { + array_elem.gc_obj = POP_REF(); + } + else if (array_type->elem_type == VALUE_TYPE_I32 + || array_type->elem_type + == VALUE_TYPE_F32 + || array_type->elem_type + == PACKED_TYPE_I8 + || array_type->elem_type + == PACKED_TYPE_I16) { + array_elem.i32 = POP_I32(); + } + else { + array_elem.i64 = POP_I64(); + } + wasm_array_obj_set_elem( + array_obj, array_len - 1 - i, &array_elem); + } + } + + PUSH_REF(array_obj); + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_NEW_DATA: + { + WASMModule *wasm_module = module->module; + WASMArrayType *array_type; + WASMRttType *rtt_type; + WASMValue array_elem = { 0 }; + WASMDataSeg *data_seg; + uint8 *array_elem_base; + uint32 array_len, data_seg_idx, data_seg_offset; + uint32 elem_size = 0; + uint64 total_size; + + type_idx = read_uint32(frame_ip); + data_seg_idx = read_uint32(frame_ip); + data_seg = wasm_module->data_segments[data_seg_idx]; + + array_type = + (WASMArrayType *)wasm_module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)array_type, type_idx, + wasm_module->rtt_types, + wasm_module->type_count, + &wasm_module->rtt_type_lock))) { + wasm_set_exception(module, + "create rtt type failed"); + goto got_exception; + } + + array_len = POP_I32(); + data_seg_offset = POP_I32(); + + switch (array_type->elem_type) { + case PACKED_TYPE_I8: + elem_size = 1; + break; + case PACKED_TYPE_I16: + elem_size = 2; + break; + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: + elem_size = 4; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + elem_size = 8; + break; + default: + bh_assert(0); + } + + total_size = (uint64)elem_size * array_len; + if (data_seg_offset >= data_seg->data_length + || total_size + > data_seg->data_length - data_seg_offset) { + wasm_set_exception(module, + "data segment out of bounds"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + array_obj = wasm_array_obj_new(exec_env, rtt_type, + array_len, &array_elem); + if (!array_obj) { + wasm_set_exception(module, + "create array object failed"); + goto got_exception; + } + + array_elem_base = + (uint8 *)wasm_array_obj_first_elem_addr(array_obj); + bh_memcpy_s(array_elem_base, (uint32)total_size, + data_seg->data + data_seg_offset, + (uint32)total_size); + + PUSH_REF(array_obj); + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_NEW_ELEM: + { + /* TODO */ + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } + case WASM_OP_ARRAY_GET: + case WASM_OP_ARRAY_GET_S: + case WASM_OP_ARRAY_GET_U: + { + WASMArrayType *array_type; + WASMValue array_elem = { 0 }; + uint32 elem_idx, elem_size_log; + + type_idx = read_uint32(frame_ip); + array_type = + (WASMArrayType *)module->module->types[type_idx]; + + elem_idx = POP_I32(); + array_obj = POP_REF(); + + if (!array_obj) { + wasm_set_exception(module, "null array reference"); + goto got_exception; + } + if (elem_idx >= wasm_array_obj_length(array_obj)) { + wasm_set_exception(module, + "out of bounds array access"); + goto got_exception; + } + + wasm_array_obj_get_elem( + array_obj, elem_idx, + opcode == WASM_OP_ARRAY_GET_S ? true : false, + &array_elem); + elem_size_log = wasm_array_obj_elem_size_log(array_obj); + + if (wasm_is_reftype_i31ref(array_type->elem_type)) { + PUSH_I31REF(array_elem.gc_obj); + } + else if (wasm_is_type_reftype(array_type->elem_type)) { + PUSH_REF(array_elem.gc_obj); + } + else if (elem_size_log < 3) { + PUSH_I32(array_elem.i32); + } + else { + PUSH_I64(array_elem.i64); + } + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_SET: + { + WASMArrayType *array_type; + WASMValue array_elem = { 0 }; + uint32 elem_idx; + + type_idx = read_uint32(frame_ip); + array_type = + (WASMArrayType *)module->module->types[type_idx]; + if (wasm_is_type_reftype(array_type->elem_type)) { + array_elem.gc_obj = POP_REF(); + } + else if (array_type->elem_type == VALUE_TYPE_I32 + || array_type->elem_type == VALUE_TYPE_F32 + || array_type->elem_type == PACKED_TYPE_I8 + || array_type->elem_type == PACKED_TYPE_I16) { + array_elem.i32 = POP_I32(); + } + else { + array_elem.i64 = POP_I64(); + } + + elem_idx = POP_I32(); + array_obj = POP_REF(); + + if (!array_obj) { + wasm_set_exception(module, "null array reference"); + goto got_exception; + } + if (elem_idx >= wasm_array_obj_length(array_obj)) { + wasm_set_exception(module, + "out of bounds array access"); + goto got_exception; + } + + wasm_array_obj_set_elem(array_obj, elem_idx, + &array_elem); + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_LEN: + { + uint32 array_len; + array_obj = POP_REF(); + if (!array_obj) { + wasm_set_exception(module, "null array reference"); + goto got_exception; + } + array_len = wasm_array_obj_length(array_obj); + PUSH_I32(array_len); + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_FILL: + { + WASMArrayType *array_type; + WASMValue fill_value = { 0 }; + uint32 start_offset, len; + + type_idx = read_uint32(frame_ip); + + array_type = + (WASMArrayType *)module->module->types[type_idx]; + + len = POP_I32(); + if (wasm_is_type_reftype(array_type->elem_type)) { + fill_value.gc_obj = POP_REF(); + } + else if (array_type->elem_type == VALUE_TYPE_I32 + || array_type->elem_type == VALUE_TYPE_F32 + || array_type->elem_type == PACKED_TYPE_I8 + || array_type->elem_type == PACKED_TYPE_I16) { + fill_value.i32 = POP_I32(); + } + else { + fill_value.i64 = POP_I64(); + } + start_offset = POP_I32(); + array_obj = POP_REF(); + + if (!array_obj) { + wasm_set_exception(module, "null array reference"); + goto got_exception; + } + + if (len > 0) { + if ((uint64)start_offset + len + >= wasm_array_obj_length(array_obj)) { + wasm_set_exception( + module, "out of bounds array access"); + goto got_exception; + } + + wasm_array_obj_fill(array_obj, start_offset, len, + &fill_value); + } + + HANDLE_OP_END(); + } + case WASM_OP_ARRAY_COPY: + { + uint32 dst_offset, src_offset, len, src_type_index; + WASMArrayObjectRef src_obj, dst_obj; + + type_idx = read_uint32(frame_ip); + src_type_index = read_uint32(frame_ip); + + len = POP_I32(); + src_offset = POP_I32(); + src_obj = POP_REF(); + dst_offset = POP_I32(); + dst_obj = POP_REF(); + + if (!src_obj || !dst_obj) { + wasm_set_exception(module, "null array reference"); + goto got_exception; + } + + if (len > 0) { + if ((dst_offset > UINT32_MAX - len) + || (dst_offset + len + > wasm_array_obj_length(dst_obj)) + || (src_offset > UINT32_MAX - len) + || (src_offset + len + > wasm_array_obj_length(src_obj))) { + wasm_set_exception( + module, "out of bounds array access"); + goto got_exception; + } + + wasm_array_obj_copy(dst_obj, dst_offset, src_obj, + src_offset, len); + } + + (void)src_type_index; + HANDLE_OP_END(); + } + + case WASM_OP_REF_I31: + { + uint32 i31_val; + + i31_val = POP_I32(); + i31_obj = wasm_i31_obj_new(i31_val); + PUSH_I31REF(i31_obj); + HANDLE_OP_END(); + } + case WASM_OP_I31_GET_S: + case WASM_OP_I31_GET_U: + { + uint32 i31_val; + + i31_obj = (WASMI31ObjectRef)POP_REF(); + if (!i31_obj) { + wasm_set_exception(module, "null i31 reference"); + goto got_exception; + } + i31_val = (uint32)(((uintptr_t)i31_obj) >> 1); + if (opcode == WASM_OP_I31_GET_S + && (i31_val & 0x40000000) /* bit 30 is 1 */) + /* set bit 31 to 1 */ + i31_val |= 0x80000000; + PUSH_I32(i31_val); + HANDLE_OP_END(); + } + + case WASM_OP_REF_TEST: + case WASM_OP_REF_CAST: + case WASM_OP_REF_TEST_NULLABLE: + case WASM_OP_REF_CAST_NULLABLE: + { + int32 heap_type; + + heap_type = (int32)read_uint32(frame_ip); + + gc_obj = POP_REF(); + if (!gc_obj) { + if (opcode == WASM_OP_REF_TEST + || opcode == WASM_OP_REF_TEST_NULLABLE) { + if (opcode == WASM_OP_REF_TEST) + PUSH_I32(0); + else + PUSH_I32(1); + } + else if (opcode == WASM_OP_REF_CAST) { + wasm_set_exception(module, "cast failure"); + goto got_exception; + } + else { + PUSH_REF(gc_obj); + } + } + else { + bool castable = false; + + if (heap_type >= 0) { + WASMModule *wasm_module = module->module; + castable = wasm_obj_is_instance_of( + gc_obj, (uint32)heap_type, + wasm_module->types, + wasm_module->type_count); + } + else { + castable = + wasm_obj_is_type_of(gc_obj, heap_type); + } + + if (opcode == WASM_OP_REF_TEST + || opcode == WASM_OP_REF_TEST_NULLABLE) { + if (castable) + PUSH_I32(1); + else + PUSH_I32(0); + } + else if (!castable) { + wasm_set_exception(module, "cast failure"); + goto got_exception; + } + else { + PUSH_REF(gc_obj); + } + } + HANDLE_OP_END(); + } + + case WASM_OP_BR_ON_CAST: + case WASM_OP_BR_ON_CAST_FAIL: + { + int32 heap_type, heap_type_dst; + uint8 castflags; + uint16 opnd_off_br; + +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + castflags = *frame_ip++; + heap_type = (int32)read_uint32(frame_ip); + heap_type_dst = (int32)read_uint32(frame_ip); + + opnd_off = GET_OFFSET(); + opnd_off_br = GET_OFFSET(); + gc_obj = GET_REF_FROM_ADDR(frame_lp + opnd_off); + PUT_REF_TO_ADDR(frame_lp + opnd_off_br, gc_obj); + + if (!gc_obj) { + /* + * castflags should be 0~3: + * 0: (non-null, non-null) + * 1: (null, non-null) + * 2: (non-null, null) + * 3: (null, null) + */ + if ( + /* op is BR_ON_CAST and dst reftype is nullable + */ + ((opcode == WASM_OP_BR_ON_CAST) + && ((castflags == 2) || (castflags == 3))) + /* op is BR_ON_CAST_FAIL and dst reftype is + non-nullable */ + || ((opcode == WASM_OP_BR_ON_CAST_FAIL) + && ((castflags == 0) + || (castflags == 1)))) { + CLEAR_FRAME_REF(opnd_off); + if (!wasm_is_reftype_i31ref(heap_type)) { + SET_FRAME_REF(opnd_off_br); + } + goto recover_br_info; + } + } + else { + bool castable = false; + + if (heap_type_dst >= 0) { + WASMModule *wasm_module = module->module; + castable = wasm_obj_is_instance_of( + gc_obj, (uint32)heap_type_dst, + wasm_module->types, + wasm_module->type_count); + } + else { + castable = + wasm_obj_is_type_of(gc_obj, heap_type_dst); + } + + if ((castable && (opcode == WASM_OP_BR_ON_CAST)) + || (!castable + && (opcode == WASM_OP_BR_ON_CAST_FAIL))) { + CLEAR_FRAME_REF(opnd_off); + if (!wasm_is_reftype_i31ref(heap_type)) { + SET_FRAME_REF(opnd_off_br); + } + goto recover_br_info; + } + } + SKIP_BR_INFO(); + + (void)heap_type_dst; + HANDLE_OP_END(); + } + + case WASM_OP_ANY_CONVERT_EXTERN: + { + externref_obj = POP_REF(); + if (externref_obj == NULL_REF) + PUSH_REF(NULL_REF); + else { + gc_obj = wasm_externref_obj_to_internal_obj( + externref_obj); + PUSH_REF(gc_obj); + } + HANDLE_OP_END(); + } + case WASM_OP_EXTERN_CONVERT_ANY: + { + gc_obj = POP_REF(); + if (gc_obj == NULL_REF) + PUSH_REF(NULL_REF); + else { + if (!(externref_obj = + wasm_internal_obj_to_externref_obj( + exec_env, gc_obj))) { + wasm_set_exception( + module, "create externref object failed"); + goto got_exception; + } + PUSH_REF(externref_obj); + } + HANDLE_OP_END(); + } + +#if WASM_ENABLE_STRINGREF != 0 + case WASM_OP_STRING_NEW_UTF8: + case WASM_OP_STRING_NEW_WTF16: + case WASM_OP_STRING_NEW_LOSSY_UTF8: + case WASM_OP_STRING_NEW_WTF8: + { + uint32 mem_idx, addr, bytes_length, offset = 0; + EncodingFlag flag = WTF8; + + mem_idx = (uint32)read_uint32(frame_ip); + bytes_length = POP_I32(); + addr = POP_I32(); + + CHECK_MEMORY_OVERFLOW(bytes_length); + + if (opcode == WASM_OP_STRING_NEW_WTF16) { + flag = WTF16; + } + else if (opcode == WASM_OP_STRING_NEW_UTF8) { + flag = UTF8; + } + else if (opcode == WASM_OP_STRING_NEW_LOSSY_UTF8) { + flag = LOSSY_UTF8; + } + else if (opcode == WASM_OP_STRING_NEW_WTF8) { + flag = WTF8; + } + + str_obj = wasm_string_new_with_encoding( + maddr, bytes_length, flag); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + + (void)mem_idx; + HANDLE_OP_END(); + } + case WASM_OP_STRING_CONST: + { + WASMModule *wasm_module = module->module; + uint32 contents; + + contents = (uint32)read_uint32(frame_ip); + + str_obj = wasm_string_new_const( + (const char *) + wasm_module->string_literal_ptrs[contents], + wasm_module->string_literal_lengths[contents]); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!str_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_MEASURE_UTF8: + case WASM_OP_STRING_MEASURE_WTF8: + case WASM_OP_STRING_MEASURE_WTF16: + { + int32 target_bytes_length; + EncodingFlag flag = WTF8; + + stringref_obj = POP_REF(); + + if (opcode == WASM_OP_STRING_MEASURE_WTF16) { + flag = WTF16; + } + else if (opcode == WASM_OP_STRING_MEASURE_UTF8) { + flag = UTF8; + } + else if (opcode == WASM_OP_STRING_MEASURE_WTF8) { + flag = LOSSY_UTF8; + } + target_bytes_length = wasm_string_measure( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj), + flag); + + PUSH_I32(target_bytes_length); + HANDLE_OP_END(); + } + case WASM_OP_STRING_ENCODE_UTF8: + case WASM_OP_STRING_ENCODE_WTF16: + case WASM_OP_STRING_ENCODE_LOSSY_UTF8: + case WASM_OP_STRING_ENCODE_WTF8: + { + uint32 mem_idx, addr; + int32 target_bytes_length; + WASMMemoryInstance *memory_inst; + EncodingFlag flag = WTF8; + + mem_idx = (uint32)read_uint32(frame_ip); + addr = POP_I32(); + stringref_obj = POP_REF(); + + str_obj = (WASMString)wasm_stringref_obj_get_value( + stringref_obj); + +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)addr, 1)) + shared_heap_addr_app_to_native((uint64)addr, maddr); + else +#endif + { + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + } + + if (opcode == WASM_OP_STRING_ENCODE_WTF16) { + flag = WTF16; + count = wasm_string_measure(str_obj, flag); + target_bytes_length = wasm_string_encode( + str_obj, 0, count, maddr, NULL, flag); + } + else { + if (opcode == WASM_OP_STRING_ENCODE_UTF8) { + flag = UTF8; + } + else if (opcode + == WASM_OP_STRING_ENCODE_LOSSY_UTF8) { + flag = LOSSY_UTF8; + } + else if (opcode == WASM_OP_STRING_ENCODE_WTF8) { + flag = WTF8; + } + count = wasm_string_measure(str_obj, flag); + target_bytes_length = wasm_string_encode( + str_obj, 0, count, maddr, NULL, flag); + + if (target_bytes_length == -1) { + wasm_set_exception( + module, "isolated surrogate is seen"); + goto got_exception; + } + } + if (target_bytes_length < 0) { + wasm_set_exception(module, + "stringref encode failed"); + goto got_exception; + } + + PUSH_I32(target_bytes_length); + HANDLE_OP_END(); + } + case WASM_OP_STRING_CONCAT: + { + WASMStringrefObjectRef stringref_obj1, stringref_obj2; + + stringref_obj2 = POP_REF(); + stringref_obj1 = POP_REF(); + + str_obj = wasm_string_concat( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj1), + (WASMString)wasm_stringref_obj_get_value( + stringref_obj2)); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_EQ: + { + WASMStringrefObjectRef stringref_obj1, stringref_obj2; + int32 is_eq; + + stringref_obj2 = POP_REF(); + stringref_obj1 = POP_REF(); + + is_eq = wasm_string_eq( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj1), + (WASMString)wasm_stringref_obj_get_value( + stringref_obj2)); + + PUSH_I32(is_eq); + HANDLE_OP_END(); + } + case WASM_OP_STRING_IS_USV_SEQUENCE: + { + int32 is_usv_sequence; + + stringref_obj = POP_REF(); + + is_usv_sequence = wasm_string_is_usv_sequence( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj)); + + PUSH_I32(is_usv_sequence); + HANDLE_OP_END(); + } + case WASM_OP_STRING_AS_WTF8: + { + stringref_obj = POP_REF(); + + str_obj = wasm_string_create_view( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj), + STRING_VIEW_WTF8); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringview_wtf8_obj = + wasm_stringview_wtf8_obj_new(exec_env, str_obj); + if (!stringview_wtf8_obj) { + wasm_set_exception(module, + "create stringview wtf8 failed"); + goto got_exception; + } + + PUSH_REF(stringview_wtf8_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF8_ADVANCE: + { + uint32 next_pos, bytes, pos; + + bytes = POP_I32(); + pos = POP_I32(); + stringview_wtf8_obj = POP_REF(); + + next_pos = wasm_string_advance( + (WASMString)wasm_stringview_wtf8_obj_get_value( + stringview_wtf8_obj), + pos, bytes, NULL); + + PUSH_I32(next_pos); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8: + case WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8: + case WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8: + { + uint32 mem_idx, addr, pos, bytes, next_pos; + int32 bytes_written; + WASMMemoryInstance *memory_inst; + EncodingFlag flag = WTF8; + + if (opcode == WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8) { + flag = UTF8; + } + else if (opcode + == WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8) { + flag = LOSSY_UTF8; + } + else if (opcode + == WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8) { + flag = WTF8; + } + + mem_idx = (uint32)read_uint32(frame_ip); + bytes = POP_I32(); + pos = POP_I32(); + addr = POP_I32(); + stringview_wtf8_obj = POP_REF(); + +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)addr, 1)) + shared_heap_addr_app_to_native((uint64)addr, maddr); + else +#endif + { + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + } + + bytes_written = wasm_string_encode( + (WASMString)wasm_stringview_wtf8_obj_get_value( + stringview_wtf8_obj), + pos, bytes, maddr, &next_pos, flag); + + if (bytes_written < 0) { + if (bytes_written == Isolated_Surrogate) { + wasm_set_exception( + module, "isolated surrogate is seen"); + } + else { + wasm_set_exception(module, "encode failed"); + } + + goto got_exception; + } + + PUSH_I32(next_pos); + PUSH_I32(bytes_written); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF8_SLICE: + { + uint32 start, end; + + end = POP_I32(); + start = POP_I32(); + stringview_wtf8_obj = POP_REF(); + + str_obj = wasm_string_slice( + (WASMString)wasm_stringview_wtf8_obj_get_value( + stringview_wtf8_obj), + start, end, STRING_VIEW_WTF8); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_AS_WTF16: + { + stringref_obj = POP_REF(); + + str_obj = wasm_string_create_view( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj), + STRING_VIEW_WTF16); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringview_wtf16_obj = + wasm_stringview_wtf16_obj_new(exec_env, str_obj); + if (!stringview_wtf16_obj) { + wasm_set_exception( + module, "create stringview wtf16 failed"); + goto got_exception; + } + + PUSH_REF(stringview_wtf16_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF16_LENGTH: + { + int32 code_units_length; + + stringview_wtf16_obj = POP_REF(); + + code_units_length = wasm_string_wtf16_get_length( + (WASMString)wasm_stringview_wtf16_obj_get_value( + stringview_wtf16_obj)); + + PUSH_I32(code_units_length); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF16_GET_CODEUNIT: + { + int32 pos; + uint32 code_unit; + + pos = POP_I32(); + stringview_wtf16_obj = POP_REF(); + + code_unit = (uint32)wasm_string_get_wtf16_codeunit( + (WASMString)wasm_stringview_wtf16_obj_get_value( + stringview_wtf16_obj), + pos); + + PUSH_I32(code_unit); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF16_ENCODE: + { + uint32 mem_idx, addr, pos, len, offset = 0; + int32 written_code_units = 0; + + mem_idx = (uint32)read_uint32(frame_ip); + len = POP_I32(); + pos = POP_I32(); + addr = POP_I32(); + stringview_wtf16_obj = POP_REF(); + + CHECK_MEMORY_OVERFLOW(len * sizeof(uint16)); + + /* check 2-byte alignment */ + if (((uintptr_t)maddr & (((uintptr_t)1 << 2) - 1)) + != 0) { + wasm_set_exception(module, + "unaligned memory access"); + goto got_exception; + } + + written_code_units = wasm_string_encode( + (WASMString)wasm_stringview_wtf16_obj_get_value( + stringview_wtf16_obj), + pos, len, maddr, NULL, WTF16); + + PUSH_I32(written_code_units); + (void)mem_idx; + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_WTF16_SLICE: + { + uint32 start, end; + + end = POP_I32(); + start = POP_I32(); + stringview_wtf16_obj = POP_REF(); + + str_obj = wasm_string_slice( + (WASMString)wasm_stringview_wtf16_obj_get_value( + stringview_wtf16_obj), + start, end, STRING_VIEW_WTF16); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_AS_ITER: + { + stringref_obj = POP_REF(); + + str_obj = wasm_string_create_view( + (WASMString)wasm_stringref_obj_get_value( + stringref_obj), + STRING_VIEW_ITER); + + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringview_iter_obj = + wasm_stringview_iter_obj_new(exec_env, str_obj, 0); + if (!stringview_iter_obj) { + wasm_set_exception(module, + "create stringview iter failed"); + goto got_exception; + } + + PUSH_REF(stringview_iter_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_ITER_NEXT: + { + uint32 code_point; + + stringview_iter_obj = POP_REF(); + + code_point = wasm_string_next_codepoint( + (WASMString)wasm_stringview_iter_obj_get_value( + stringview_iter_obj), + wasm_stringview_iter_obj_get_pos( + stringview_iter_obj)); + + PUSH_I32(code_point); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_ITER_ADVANCE: + case WASM_OP_STRINGVIEW_ITER_REWIND: + { + uint32 code_points_count, code_points_consumed = 0, + cur_pos, next_pos = 0; + + code_points_count = POP_I32(); + stringview_iter_obj = POP_REF(); + + str_obj = + (WASMString)wasm_stringview_iter_obj_get_value( + stringview_iter_obj); + cur_pos = wasm_stringview_iter_obj_get_pos( + stringview_iter_obj); + + if (opcode == WASM_OP_STRINGVIEW_ITER_ADVANCE) { + next_pos = wasm_string_advance( + str_obj, cur_pos, code_points_count, + &code_points_consumed); + } + else if (opcode == WASM_OP_STRINGVIEW_ITER_REWIND) { + next_pos = wasm_string_rewind( + str_obj, cur_pos, code_points_count, + &code_points_consumed); + } + + wasm_stringview_iter_obj_update_pos(stringview_iter_obj, + next_pos); + + PUSH_I32(code_points_consumed); + HANDLE_OP_END(); + } + case WASM_OP_STRINGVIEW_ITER_SLICE: + { + uint32 code_points_count, cur_pos; + + code_points_count = POP_I32(); + stringview_iter_obj = POP_REF(); + + cur_pos = wasm_stringview_iter_obj_get_pos( + stringview_iter_obj); + + str_obj = wasm_string_slice( + (WASMString)wasm_stringview_iter_obj_get_value( + stringview_iter_obj), + cur_pos, cur_pos + code_points_count, + STRING_VIEW_ITER); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_NEW_UTF8_ARRAY: + case WASM_OP_STRING_NEW_WTF16_ARRAY: + case WASM_OP_STRING_NEW_LOSSY_UTF8_ARRAY: + case WASM_OP_STRING_NEW_WTF8_ARRAY: + { + uint32 start, end, array_len; + EncodingFlag flag = WTF8; + WASMArrayType *array_type; + void *arr_start_addr; + + end = POP_I32(); + start = POP_I32(); + array_obj = POP_REF(); + + array_type = (WASMArrayType *)wasm_obj_get_defined_type( + (WASMObjectRef)array_obj); + arr_start_addr = + wasm_array_obj_elem_addr(array_obj, start); + array_len = wasm_array_obj_length(array_obj); + + if (start > end || end > array_len) { + wasm_set_exception(module, + "out of bounds array access"); + goto got_exception; + } + + if (opcode == WASM_OP_STRING_NEW_WTF16_ARRAY) { + if (array_type->elem_type != VALUE_TYPE_I16) { + wasm_set_exception(module, + "array type mismatch"); + goto got_exception; + } + flag = WTF16; + } + else { + if (array_type->elem_type != VALUE_TYPE_I8) { + wasm_set_exception(module, + "array type mismatch"); + goto got_exception; + } + if (opcode == WASM_OP_STRING_NEW_UTF8_ARRAY) { + flag = UTF8; + } + else if (opcode == WASM_OP_STRING_NEW_WTF8_ARRAY) { + flag = WTF8; + } + else if (opcode + == WASM_OP_STRING_NEW_LOSSY_UTF8_ARRAY) { + flag = LOSSY_UTF8; + } + } + + str_obj = wasm_string_new_with_encoding( + arr_start_addr, (end - start), flag); + if (!str_obj) { + wasm_set_exception(module, + "create string object failed"); + goto got_exception; + } + + SYNC_ALL_TO_FRAME(); + stringref_obj = + wasm_stringref_obj_new(exec_env, str_obj); + if (!stringref_obj) { + wasm_set_exception(module, + "create stringref failed"); + goto got_exception; + } + + PUSH_REF(stringref_obj); + HANDLE_OP_END(); + } + case WASM_OP_STRING_ENCODE_UTF8_ARRAY: + case WASM_OP_STRING_ENCODE_WTF16_ARRAY: + case WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY: + case WASM_OP_STRING_ENCODE_WTF8_ARRAY: + { + uint32 start, array_len, count; + int32 bytes_written; + EncodingFlag flag = WTF8; + WASMArrayType *array_type; + void *arr_start_addr; + + start = POP_I32(); + array_obj = POP_REF(); + stringref_obj = POP_REF(); + + str_obj = (WASMString)wasm_stringref_obj_get_value( + stringref_obj); + + array_type = (WASMArrayType *)wasm_obj_get_defined_type( + (WASMObjectRef)array_obj); + arr_start_addr = + wasm_array_obj_elem_addr(array_obj, start); + array_len = wasm_array_obj_length(array_obj); + + if (start > array_len) { + wasm_set_exception(module, + "out of bounds array access"); + goto got_exception; + } + + if (opcode == WASM_OP_STRING_ENCODE_WTF16_ARRAY) { + if (array_type->elem_type != VALUE_TYPE_I16) { + wasm_set_exception(module, + "array type mismatch"); + goto got_exception; + } + flag = WTF16; + } + else { + if (array_type->elem_type != VALUE_TYPE_I8) { + wasm_set_exception(module, + "array type mismatch"); + goto got_exception; + } + if (opcode == WASM_OP_STRING_ENCODE_UTF8_ARRAY) { + flag = UTF8; + } + else if (opcode + == WASM_OP_STRING_ENCODE_WTF8_ARRAY) { + flag = WTF8; + } + else if ( + opcode + == WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY) { + flag = LOSSY_UTF8; + } + } + + count = wasm_string_measure(str_obj, flag); + + bytes_written = wasm_string_encode( + str_obj, 0, count, arr_start_addr, NULL, flag); + + if (bytes_written < 0) { + if (bytes_written == Isolated_Surrogate) { + wasm_set_exception( + module, "isolated surrogate is seen"); + } + else if (bytes_written == Insufficient_Space) { + wasm_set_exception( + module, "array space is insufficient"); + } + else { + wasm_set_exception(module, "encode failed"); + } + + goto got_exception; + } + + PUSH_I32(bytes_written); + HANDLE_OP_END(); + } +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ + + default: + { + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } + } + } +#endif /* end of WASM_ENABLE_GC != 0 */ + + /* variable instructions */ + HANDLE_OP(EXT_OP_SET_LOCAL_FAST) + HANDLE_OP(EXT_OP_TEE_LOCAL_FAST) + { + /* clang-format off */ +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + local_offset = *frame_ip++; +#else + local_offset = *frame_ip; + frame_ip += 2; +#endif + /* clang-format on */ + *(uint32 *)(frame_lp + local_offset) = + GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + HANDLE_OP_END(); + } + + HANDLE_OP(EXT_OP_SET_LOCAL_FAST_I64) + HANDLE_OP(EXT_OP_TEE_LOCAL_FAST_I64) + { + /* clang-format off */ +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + local_offset = *frame_ip++; +#else + local_offset = *frame_ip; + frame_ip += 2; +#endif + /* clang-format on */ + PUT_I64_TO_ADDR((uint32 *)(frame_lp + local_offset), + GET_OPERAND(uint64, I64, 0)); + frame_ip += 2; + HANDLE_OP_END(); + } + +#if WASM_ENABLE_SIMDE != 0 + HANDLE_OP(EXT_OP_SET_LOCAL_FAST_V128) + HANDLE_OP(EXT_OP_TEE_LOCAL_FAST_V128) + { + /* clang-format off */ +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + local_offset = *frame_ip++; +#else + local_offset = *frame_ip; + frame_ip += 2; +#endif + /* clang-format on */ + PUT_V128_TO_ADDR((uint32 *)(frame_lp + local_offset), + GET_OPERAND_V128(0)); + frame_ip += 2; + HANDLE_OP_END(); + } +#endif + HANDLE_OP(WASM_OP_GET_GLOBAL) + { + global_idx = read_uint32(frame_ip); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); + addr_ret = GET_OFFSET(); + /* clang-format off */ +#if WASM_ENABLE_GC == 0 + frame_lp[addr_ret] = *(uint32 *)global_addr; +#else + if (!wasm_is_type_reftype(global->type)) + frame_lp[addr_ret] = *(uint32 *)global_addr; + else { + PUT_REF_TO_ADDR(frame_lp + addr_ret, + GET_REF_FROM_ADDR((uint32 *)global_addr)); + if (!wasm_is_reftype_i31ref(global->type)) { + SET_FRAME_REF(addr_ret); + } + } +#endif + /* clang-format on */ + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_GET_GLOBAL_64) + { + global_idx = read_uint32(frame_ip); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); + addr_ret = GET_OFFSET(); + PUT_I64_TO_ADDR(frame_lp + addr_ret, + GET_I64_FROM_ADDR((uint32 *)global_addr)); + HANDLE_OP_END(); + } +#if WASM_ENABLE_SIMDE != 0 + HANDLE_OP(WASM_OP_GET_GLOBAL_V128) + { + global_idx = read_uint32(frame_ip); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); + addr_ret = GET_OFFSET(); + PUT_V128_TO_ADDR(frame_lp + addr_ret, + GET_V128_FROM_ADDR((uint32 *)global_addr)); + HANDLE_OP_END(); + } +#endif + HANDLE_OP(WASM_OP_SET_GLOBAL) + { + global_idx = read_uint32(frame_ip); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); + addr1 = GET_OFFSET(); + /* clang-format off */ +#if WASM_ENABLE_GC == 0 + *(int32 *)global_addr = frame_lp[addr1]; +#else + if (!wasm_is_type_reftype(global->type)) + *(int32 *)global_addr = frame_lp[addr1]; + else { + PUT_REF_TO_ADDR((uint32 *)global_addr, + GET_REF_FROM_ADDR(frame_lp + addr1)); + CLEAR_FRAME_REF(addr1); + } +#endif + /* clang-format on */ + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_SET_GLOBAL_AUX_STACK) + { + uint64 aux_stack_top; + + global_idx = read_uint32(frame_ip); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); + /* TODO: Memory64 the data type depends on mem idx type */ + aux_stack_top = (uint64)frame_lp[GET_OFFSET()]; + if (aux_stack_top <= (uint64)exec_env->aux_stack_boundary) { + wasm_set_exception(module, "wasm auxiliary stack overflow"); + goto got_exception; + } + if (aux_stack_top > (uint64)exec_env->aux_stack_bottom) { + wasm_set_exception(module, + "wasm auxiliary stack underflow"); + goto got_exception; + } + *(int32 *)global_addr = (uint32)aux_stack_top; +#if WASM_ENABLE_MEMORY_PROFILING != 0 + if (module->module->aux_stack_top_global_index != (uint32)-1) { + uint32 aux_stack_used = + (uint32)(module->module->aux_stack_bottom + - *(uint32 *)global_addr); + if (aux_stack_used > module->e->max_aux_stack_used) + module->e->max_aux_stack_used = aux_stack_used; + } +#endif + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_SET_GLOBAL_64) + { + global_idx = read_uint32(frame_ip); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); + addr1 = GET_OFFSET(); + PUT_I64_TO_ADDR((uint32 *)global_addr, + GET_I64_FROM_ADDR(frame_lp + addr1)); + HANDLE_OP_END(); + } +#if WASM_ENABLE_SIMDE != 0 + HANDLE_OP(WASM_OP_SET_GLOBAL_V128) + { + global_idx = read_uint32(frame_ip); + bh_assert(global_idx < module->e->global_count); + global = globals + global_idx; + global_addr = get_global_addr(global_data, global); + addr1 = GET_OFFSET(); + PUT_V128_TO_ADDR((uint32 *)global_addr, + GET_V128_FROM_ADDR(frame_lp + addr1)); + HANDLE_OP_END(); + } +#endif + + /* memory load instructions */ + HANDLE_OP(WASM_OP_I32_LOAD) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(4); + frame_lp[addr_ret] = LOAD_I32(maddr); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(8); + PUT_I64_TO_ADDR(frame_lp + addr_ret, LOAD_I64(maddr)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LOAD8_S) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(1); + frame_lp[addr_ret] = sign_ext_8_32(*(int8 *)maddr); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LOAD8_U) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(1); + frame_lp[addr_ret] = (uint32)(*(uint8 *)(maddr)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LOAD16_S) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(2); + frame_lp[addr_ret] = sign_ext_16_32(LOAD_I16(maddr)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LOAD16_U) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(2); + frame_lp[addr_ret] = (uint32)(LOAD_U16(maddr)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD8_S) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(1); + PUT_I64_TO_ADDR(frame_lp + addr_ret, + sign_ext_8_64(*(int8 *)maddr)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD8_U) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(1); + PUT_I64_TO_ADDR(frame_lp + addr_ret, (uint64)(*(uint8 *)maddr)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD16_S) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(2); + PUT_I64_TO_ADDR(frame_lp + addr_ret, + sign_ext_16_64(LOAD_I16(maddr))); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD16_U) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(2); + PUT_I64_TO_ADDR(frame_lp + addr_ret, (uint64)(LOAD_U16(maddr))); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD32_S) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(4); + PUT_I64_TO_ADDR(frame_lp + addr_ret, + sign_ext_32_64(LOAD_I32(maddr))); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LOAD32_U) + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = GET_OPERAND(uint32, I32, 0); + frame_ip += 2; + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(4); + PUT_I64_TO_ADDR(frame_lp + addr_ret, (uint64)(LOAD_U32(maddr))); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_STORE) + { + uint32 offset, addr; + uint32 sval; + offset = read_uint32(frame_ip); + sval = GET_OPERAND(uint32, I32, 0); + addr = GET_OPERAND(uint32, I32, 2); + frame_ip += 4; + CHECK_MEMORY_OVERFLOW(4); + STORE_U32(maddr, sval); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_STORE8) + { + uint32 offset, addr; + uint32 sval; + offset = read_uint32(frame_ip); + sval = GET_OPERAND(uint32, I32, 0); + addr = GET_OPERAND(uint32, I32, 2); + frame_ip += 4; + CHECK_MEMORY_OVERFLOW(1); + STORE_U8(maddr, (uint8_t)sval); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_STORE16) + { + uint32 offset, addr; + uint32 sval; + offset = read_uint32(frame_ip); + sval = GET_OPERAND(uint32, I32, 0); + addr = GET_OPERAND(uint32, I32, 2); + frame_ip += 4; + CHECK_MEMORY_OVERFLOW(2); + STORE_U16(maddr, (uint16)sval); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_STORE) + { + uint32 offset, addr; + uint64 sval; + offset = read_uint32(frame_ip); + sval = GET_OPERAND(uint64, I64, 0); + addr = GET_OPERAND(uint32, I32, 2); + frame_ip += 4; + CHECK_MEMORY_OVERFLOW(8); + STORE_I64(maddr, sval); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_STORE8) + { + uint32 offset, addr; + uint64 sval; + offset = read_uint32(frame_ip); + sval = GET_OPERAND(uint64, I64, 0); + addr = GET_OPERAND(uint32, I32, 2); + frame_ip += 4; + CHECK_MEMORY_OVERFLOW(1); + *(uint8 *)maddr = (uint8)sval; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_STORE16) + { + uint32 offset, addr; + uint64 sval; + offset = read_uint32(frame_ip); + sval = GET_OPERAND(uint64, I64, 0); + addr = GET_OPERAND(uint32, I32, 2); + frame_ip += 4; + CHECK_MEMORY_OVERFLOW(2); + STORE_U16(maddr, (uint16)sval); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_STORE32) + { + uint32 offset, addr; + uint64 sval; + offset = read_uint32(frame_ip); + sval = GET_OPERAND(uint64, I64, 0); + addr = GET_OPERAND(uint32, I32, 2); + frame_ip += 4; + CHECK_MEMORY_OVERFLOW(4); + STORE_U32(maddr, (uint32)sval); + HANDLE_OP_END(); + } + + /* memory size and memory grow instructions */ + HANDLE_OP(WASM_OP_MEMORY_SIZE) + { + uint32 reserved; + addr_ret = GET_OFFSET(); + frame_lp[addr_ret] = memory->cur_page_count; + (void)reserved; + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_MEMORY_GROW) + { + uint32 reserved, delta, + prev_page_count = memory->cur_page_count; + + addr1 = GET_OFFSET(); + addr_ret = GET_OFFSET(); + delta = (uint32)frame_lp[addr1]; + + /* TODO: multi-memory wasm_enlarge_memory_with_idx() */ + if (!wasm_enlarge_memory(module, delta)) { + /* failed to memory.grow, return -1 */ + frame_lp[addr_ret] = -1; + } + else { + /* success, return previous page count */ + frame_lp[addr_ret] = prev_page_count; + /* update memory size, no need to update memory ptr as + it isn't changed in wasm_enlarge_memory */ +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY != 0 + linear_mem_size = GET_LINEAR_MEMORY_SIZE(memory); +#endif + } + + (void)reserved; + HANDLE_OP_END(); + } + + /* constant instructions */ + HANDLE_OP(WASM_OP_F64_CONST) + HANDLE_OP(WASM_OP_I64_CONST) + { + uint8 *orig_ip = frame_ip; + + frame_ip += sizeof(uint64); + addr_ret = GET_OFFSET(); + + bh_memcpy_s(frame_lp + addr_ret, sizeof(uint64), orig_ip, + sizeof(uint64)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_CONST) + HANDLE_OP(WASM_OP_I32_CONST) + { + uint8 *orig_ip = frame_ip; + + frame_ip += sizeof(uint32); + addr_ret = GET_OFFSET(); + + bh_memcpy_s(frame_lp + addr_ret, sizeof(uint32), orig_ip, + sizeof(uint32)); + HANDLE_OP_END(); + } + + /* comparison instructions of i32 */ + HANDLE_OP(WASM_OP_I32_EQZ) + { + DEF_OP_EQZ(int32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_EQ) + { + DEF_OP_CMP(uint32, I32, ==); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_NE) + { + DEF_OP_CMP(uint32, I32, !=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LT_S) + { + DEF_OP_CMP(int32, I32, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LT_U) + { + DEF_OP_CMP(uint32, I32, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_GT_S) + { + DEF_OP_CMP(int32, I32, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_GT_U) + { + DEF_OP_CMP(uint32, I32, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LE_S) + { + DEF_OP_CMP(int32, I32, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_LE_U) + { + DEF_OP_CMP(uint32, I32, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_GE_S) + { + DEF_OP_CMP(int32, I32, >=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_GE_U) + { + DEF_OP_CMP(uint32, I32, >=); + HANDLE_OP_END(); + } + + /* comparison instructions of i64 */ + HANDLE_OP(WASM_OP_I64_EQZ) + { + DEF_OP_EQZ(int64, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_EQ) + { + DEF_OP_CMP(uint64, I64, ==); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_NE) + { + DEF_OP_CMP(uint64, I64, !=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LT_S) + { + DEF_OP_CMP(int64, I64, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LT_U) + { + DEF_OP_CMP(uint64, I64, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_GT_S) + { + DEF_OP_CMP(int64, I64, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_GT_U) + { + DEF_OP_CMP(uint64, I64, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LE_S) + { + DEF_OP_CMP(int64, I64, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_LE_U) + { + DEF_OP_CMP(uint64, I64, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_GE_S) + { + DEF_OP_CMP(int64, I64, >=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_GE_U) + { + DEF_OP_CMP(uint64, I64, >=); + HANDLE_OP_END(); + } + + /* comparison instructions of f32 */ + HANDLE_OP(WASM_OP_F32_EQ) + { + DEF_OP_CMP(float32, F32, ==); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_NE) + { + DEF_OP_CMP(float32, F32, !=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_LT) + { + DEF_OP_CMP(float32, F32, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_GT) + { + DEF_OP_CMP(float32, F32, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_LE) + { + DEF_OP_CMP(float32, F32, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_GE) + { + DEF_OP_CMP(float32, F32, >=); + HANDLE_OP_END(); + } + + /* comparison instructions of f64 */ + HANDLE_OP(WASM_OP_F64_EQ) + { + DEF_OP_CMP(float64, F64, ==); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_NE) + { + DEF_OP_CMP(float64, F64, !=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_LT) + { + DEF_OP_CMP(float64, F64, <); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_GT) + { + DEF_OP_CMP(float64, F64, >); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_LE) + { + DEF_OP_CMP(float64, F64, <=); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_GE) + { + DEF_OP_CMP(float64, F64, >=); + HANDLE_OP_END(); + } + + /* numeric instructions of i32 */ + HANDLE_OP(WASM_OP_I32_CLZ) + { + DEF_OP_BIT_COUNT(uint32, I32, clz32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_CTZ) + { + DEF_OP_BIT_COUNT(uint32, I32, ctz32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_POPCNT) + { + DEF_OP_BIT_COUNT(uint32, I32, popcount32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_ADD) + { + DEF_OP_NUMERIC(uint32, uint32, I32, +); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_SUB) + { + DEF_OP_NUMERIC(uint32, uint32, I32, -); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_MUL) + { + DEF_OP_NUMERIC(uint32, uint32, I32, *); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_DIV_S) + { + int32 a, b; + + b = frame_lp[GET_OFFSET()]; + a = frame_lp[GET_OFFSET()]; + addr_ret = GET_OFFSET(); + if (a == (int32)0x80000000 && b == -1) { + wasm_set_exception(module, "integer overflow"); + goto got_exception; + } + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + frame_lp[addr_ret] = (a / b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_DIV_U) + { + uint32 a, b; + + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + addr_ret = GET_OFFSET(); + + b = (uint32)frame_lp[addr1]; + a = (uint32)frame_lp[addr2]; + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + frame_lp[addr_ret] = (a / b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_REM_S) + { + int32 a, b; + + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + addr_ret = GET_OFFSET(); + + b = frame_lp[addr1]; + a = frame_lp[addr2]; + if (a == (int32)0x80000000 && b == -1) { + frame_lp[addr_ret] = 0; + HANDLE_OP_END(); + } + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + frame_lp[addr_ret] = (a % b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_REM_U) + { + uint32 a, b; + + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + addr_ret = GET_OFFSET(); + + b = (uint32)frame_lp[addr1]; + a = (uint32)frame_lp[addr2]; + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + frame_lp[addr_ret] = (a % b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_AND) + { + DEF_OP_NUMERIC(uint32, uint32, I32, &); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_OR) + { + DEF_OP_NUMERIC(uint32, uint32, I32, |); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_XOR) + { + DEF_OP_NUMERIC(uint32, uint32, I32, ^); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_SHL) + { + DEF_OP_NUMERIC2(uint32, uint32, I32, <<); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_SHR_S) + { + DEF_OP_NUMERIC2(int32, uint32, I32, >>); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_SHR_U) + { + DEF_OP_NUMERIC2(uint32, uint32, I32, >>); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_ROTL) + { + uint32 a, b; + + b = (uint32)frame_lp[GET_OFFSET()]; + a = (uint32)frame_lp[GET_OFFSET()]; + frame_lp[GET_OFFSET()] = rotl32(a, b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_ROTR) + { + uint32 a, b; + + b = (uint32)frame_lp[GET_OFFSET()]; + a = (uint32)frame_lp[GET_OFFSET()]; + frame_lp[GET_OFFSET()] = rotr32(a, b); + HANDLE_OP_END(); + } + + /* numeric instructions of i64 */ + HANDLE_OP(WASM_OP_I64_CLZ) + { + DEF_OP_BIT_COUNT(uint64, I64, clz64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_CTZ) + { + DEF_OP_BIT_COUNT(uint64, I64, ctz64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_POPCNT) + { + DEF_OP_BIT_COUNT(uint64, I64, popcount64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_ADD) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, +); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_SUB) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, -); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_MUL) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, *); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_DIV_S) + { + int64 a, b; + + b = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + a = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + if (a == (int64)0x8000000000000000LL && b == -1) { + wasm_set_exception(module, "integer overflow"); + goto got_exception; + } + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + PUT_I64_TO_ADDR(frame_lp + GET_OFFSET(), a / b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_DIV_U) + { + uint64 a, b; + + b = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + a = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + PUT_I64_TO_ADDR(frame_lp + GET_OFFSET(), a / b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_REM_S) + { + int64 a, b; + + b = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + a = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + if (a == (int64)0x8000000000000000LL && b == -1) { + *(int64 *)(frame_lp + GET_OFFSET()) = 0; + HANDLE_OP_END(); + } + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + PUT_I64_TO_ADDR(frame_lp + GET_OFFSET(), a % b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_REM_U) + { + uint64 a, b; + + b = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + a = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + if (b == 0) { + wasm_set_exception(module, "integer divide by zero"); + goto got_exception; + } + PUT_I64_TO_ADDR(frame_lp + GET_OFFSET(), a % b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_AND) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, &); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_OR) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, |); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_XOR) + { + DEF_OP_NUMERIC_64(uint64, uint64, I64, ^); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_SHL) + { + DEF_OP_NUMERIC2_64(uint64, uint64, I64, <<); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_SHR_S) + { + DEF_OP_NUMERIC2_64(int64, uint64, I64, >>); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_SHR_U) + { + DEF_OP_NUMERIC2_64(uint64, uint64, I64, >>); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_ROTL) + { + uint64 a, b; + + b = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + a = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + PUT_I64_TO_ADDR(frame_lp + GET_OFFSET(), rotl64(a, b)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_ROTR) + { + uint64 a, b; + + b = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + a = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + PUT_I64_TO_ADDR(frame_lp + GET_OFFSET(), rotr64(a, b)); + HANDLE_OP_END(); + } + + /* numeric instructions of f32 */ + HANDLE_OP(WASM_OP_F32_ABS) + { + DEF_OP_MATH(float32, F32, fabsf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_NEG) + { + uint32 u32 = frame_lp[GET_OFFSET()]; + uint32 sign_bit = u32 & ((uint32)1 << 31); + addr_ret = GET_OFFSET(); + if (sign_bit) + frame_lp[addr_ret] = u32 & ~((uint32)1 << 31); + else + frame_lp[addr_ret] = u32 | ((uint32)1 << 31); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_CEIL) + { + DEF_OP_MATH(float32, F32, ceilf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_FLOOR) + { + DEF_OP_MATH(float32, F32, floorf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_TRUNC) + { + DEF_OP_MATH(float32, F32, truncf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_NEAREST) + { + DEF_OP_MATH(float32, F32, rintf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_SQRT) + { + DEF_OP_MATH(float32, F32, sqrtf); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_ADD) + { + DEF_OP_NUMERIC(float32, float32, F32, +); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_SUB) + { + DEF_OP_NUMERIC(float32, float32, F32, -); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_MUL) + { + DEF_OP_NUMERIC(float32, float32, F32, *); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_DIV) + { + DEF_OP_NUMERIC(float32, float32, F32, /); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_MIN) + { + float32 a, b; + + b = *(float32 *)(frame_lp + GET_OFFSET()); + a = *(float32 *)(frame_lp + GET_OFFSET()); + + *(float32 *)(frame_lp + GET_OFFSET()) = f32_min(a, b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_MAX) + { + float32 a, b; + + b = *(float32 *)(frame_lp + GET_OFFSET()); + a = *(float32 *)(frame_lp + GET_OFFSET()); + + *(float32 *)(frame_lp + GET_OFFSET()) = f32_max(a, b); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_COPYSIGN) + { + float32 a, b; + + b = *(float32 *)(frame_lp + GET_OFFSET()); + a = *(float32 *)(frame_lp + GET_OFFSET()); + *(float32 *)(frame_lp + GET_OFFSET()) = local_copysignf(a, b); + HANDLE_OP_END(); + } + + /* numeric instructions of f64 */ + HANDLE_OP(WASM_OP_F64_ABS) + { + DEF_OP_MATH(float64, F64, fabs); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_NEG) + { + uint64 u64 = GET_I64_FROM_ADDR(frame_lp + GET_OFFSET()); + uint64 sign_bit = u64 & (((uint64)1) << 63); + if (sign_bit) + PUT_I64_TO_ADDR(frame_lp + GET_OFFSET(), + (u64 & ~(((uint64)1) << 63))); + else + PUT_I64_TO_ADDR(frame_lp + GET_OFFSET(), + (u64 | (((uint64)1) << 63))); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_CEIL) + { + DEF_OP_MATH(float64, F64, ceil); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_FLOOR) + { + DEF_OP_MATH(float64, F64, floor); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_TRUNC) + { + DEF_OP_MATH(float64, F64, trunc); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_NEAREST) + { + DEF_OP_MATH(float64, F64, rint); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_SQRT) + { + DEF_OP_MATH(float64, F64, sqrt); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_ADD) + { + DEF_OP_NUMERIC_64(float64, float64, F64, +); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_SUB) + { + DEF_OP_NUMERIC_64(float64, float64, F64, -); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_MUL) + { + DEF_OP_NUMERIC_64(float64, float64, F64, *); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_DIV) + { + DEF_OP_NUMERIC_64(float64, float64, F64, /); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_MIN) + { + float64 a, b; + + b = POP_F64(); + a = POP_F64(); + + PUSH_F64(f64_min(a, b)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_MAX) + { + float64 a, b; + + b = POP_F64(); + a = POP_F64(); + + PUSH_F64(f64_max(a, b)); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_COPYSIGN) + { + float64 a, b; + + b = POP_F64(); + a = POP_F64(); + PUSH_F64(local_copysign(a, b)); + HANDLE_OP_END(); + } + + /* conversions of i32 */ + HANDLE_OP(WASM_OP_I32_WRAP_I64) + { + int32 value = (int32)(POP_I64() & 0xFFFFFFFFLL); + PUSH_I32(value); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_TRUNC_S_F32) + { + /* We don't use INT32_MIN/INT32_MAX/UINT32_MIN/UINT32_MAX, + since float/double values of ieee754 cannot precisely + represent all int32/uint32/int64/uint64 values, e.g.: + UINT32_MAX is 4294967295, but (float32)4294967295 is + 4294967296.0f, but not 4294967295.0f. */ + DEF_OP_TRUNC_F32(-2147483904.0f, 2147483648.0f, true, true); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_TRUNC_U_F32) + { + DEF_OP_TRUNC_F32(-1.0f, 4294967296.0f, true, false); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_TRUNC_S_F64) + { + DEF_OP_TRUNC_F64(-2147483649.0, 2147483648.0, true, true); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_TRUNC_U_F64) + { + DEF_OP_TRUNC_F64(-1.0, 4294967296.0, true, false); + HANDLE_OP_END(); + } + + /* conversions of i64 */ + HANDLE_OP(WASM_OP_I64_EXTEND_S_I32) + { + DEF_OP_CONVERT(int64, I64, int32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_EXTEND_U_I32) + { + DEF_OP_CONVERT(int64, I64, uint32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_TRUNC_S_F32) + { + DEF_OP_TRUNC_F32(-9223373136366403584.0f, + 9223372036854775808.0f, false, true); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_TRUNC_U_F32) + { + DEF_OP_TRUNC_F32(-1.0f, 18446744073709551616.0f, false, false); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_TRUNC_S_F64) + { + DEF_OP_TRUNC_F64(-9223372036854777856.0, 9223372036854775808.0, + false, true); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_TRUNC_U_F64) + { + DEF_OP_TRUNC_F64(-1.0, 18446744073709551616.0, false, false); + HANDLE_OP_END(); + } + + /* conversions of f32 */ + HANDLE_OP(WASM_OP_F32_CONVERT_S_I32) + { + DEF_OP_CONVERT(float32, F32, int32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_CONVERT_U_I32) + { + DEF_OP_CONVERT(float32, F32, uint32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_CONVERT_S_I64) + { + DEF_OP_CONVERT(float32, F32, int64, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_CONVERT_U_I64) + { + DEF_OP_CONVERT(float32, F32, uint64, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F32_DEMOTE_F64) + { + DEF_OP_CONVERT(float32, F32, float64, F64); + HANDLE_OP_END(); + } + + /* conversions of f64 */ + HANDLE_OP(WASM_OP_F64_CONVERT_S_I32) + { + DEF_OP_CONVERT(float64, F64, int32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_CONVERT_U_I32) + { + DEF_OP_CONVERT(float64, F64, uint32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_CONVERT_S_I64) + { + DEF_OP_CONVERT(float64, F64, int64, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_CONVERT_U_I64) + { + DEF_OP_CONVERT(float64, F64, uint64, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_F64_PROMOTE_F32) + { + DEF_OP_CONVERT(float64, F64, float32, F32); + HANDLE_OP_END(); + } + + /* reinterpretations */ + HANDLE_OP(WASM_OP_I32_REINTERPRET_F32) + HANDLE_OP(WASM_OP_F32_REINTERPRET_I32) + { + DEF_OP_REINTERPRET(uint32, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_REINTERPRET_F64) + HANDLE_OP(WASM_OP_F64_REINTERPRET_I64) + { + DEF_OP_REINTERPRET(int64, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(EXT_OP_COPY_STACK_TOP) + { + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + frame_lp[addr2] = frame_lp[addr1]; + +#if WASM_ENABLE_GC != 0 + /* Ignore constants because they are not reference */ + if (addr1 >= 0) { + if (*FRAME_REF(addr1)) { + CLEAR_FRAME_REF(addr1); + SET_FRAME_REF(addr2); + } + } +#endif + + HANDLE_OP_END(); + } + + HANDLE_OP(EXT_OP_COPY_STACK_TOP_I64) + { + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + + PUT_I64_TO_ADDR(frame_lp + addr2, + GET_I64_FROM_ADDR(frame_lp + addr1)); + +#if WASM_ENABLE_GC != 0 + /* Ignore constants because they are not reference */ + if (addr1 >= 0) { + if (*FRAME_REF(addr1)) { + CLEAR_FRAME_REF(addr1); + SET_FRAME_REF(addr2); + } + } +#endif + + HANDLE_OP_END(); + } +#if WASM_ENABLE_SIMDE != 0 + HANDLE_OP(EXT_OP_COPY_STACK_TOP_V128) + { + addr1 = GET_OFFSET(); + addr2 = GET_OFFSET(); + + PUT_V128_TO_ADDR(frame_lp + addr2, + GET_V128_FROM_ADDR(frame_lp + addr1)); + +#if WASM_ENABLE_GC != 0 + /* Ignore constants because they are not reference */ + if (addr1 >= 0) { + if (*FRAME_REF(addr1)) { + CLEAR_FRAME_REF(addr1); + SET_FRAME_REF(addr2); + } + } +#endif + + HANDLE_OP_END(); + } +#endif + + HANDLE_OP(EXT_OP_COPY_STACK_VALUES) + { + uint32 values_count, total_cell; + uint8 *cells; + int16 *src_offsets = NULL; + uint16 *dst_offsets = NULL; + + /* read values_count */ + values_count = read_uint32(frame_ip); + /* read total cell num */ + total_cell = read_uint32(frame_ip); + /* cells */ + cells = (uint8 *)frame_ip; + frame_ip += values_count * CELL_SIZE; + /* src offsets */ + src_offsets = (int16 *)frame_ip; + frame_ip += values_count * sizeof(int16); + /* dst offsets */ + dst_offsets = (uint16 *)frame_ip; + frame_ip += values_count * sizeof(uint16); + + if (!copy_stack_values(module, frame_lp, values_count, +#if WASM_ENABLE_GC != 0 + frame_ref, +#endif + total_cell, cells, src_offsets, + dst_offsets)) + goto got_exception; + + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_SET_LOCAL) + { + opcode = WASM_OP_SET_LOCAL; + goto handle_op_set_tee_local; + } + HANDLE_OP(WASM_OP_TEE_LOCAL) + { + opcode = WASM_OP_TEE_LOCAL; + handle_op_set_tee_local: + + GET_LOCAL_INDEX_TYPE_AND_OFFSET(); + addr1 = GET_OFFSET(); + + if (local_type == VALUE_TYPE_I32 || local_type == VALUE_TYPE_F32 +#if WASM_ENABLE_REF_TYPES != 0 && WASM_ENABLE_GC == 0 + || local_type == VALUE_TYPE_FUNCREF + || local_type == VALUE_TYPE_EXTERNREF +#endif + ) { + *(int32 *)(frame_lp + local_offset) = frame_lp[addr1]; + } + else if (local_type == VALUE_TYPE_I64 + || local_type == VALUE_TYPE_F64) { + PUT_I64_TO_ADDR((uint32 *)(frame_lp + local_offset), + GET_I64_FROM_ADDR(frame_lp + addr1)); + } +#if WASM_ENABLE_GC != 0 + else if (wasm_is_type_reftype(local_type)) { + PUT_REF_TO_ADDR((uint32 *)(frame_lp + local_offset), + GET_REF_FROM_ADDR(frame_lp + addr1)); + if (opcode == WASM_OP_SET_LOCAL) { + CLEAR_FRAME_REF(addr1); + } + } +#endif + else { + wasm_set_exception(module, "invalid local type"); + goto got_exception; + } + + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_EXTEND8_S) + { + DEF_OP_CONVERT(int32, I32, int8, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I32_EXTEND16_S) + { + DEF_OP_CONVERT(int32, I32, int16, I32); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_EXTEND8_S) + { + DEF_OP_CONVERT(int64, I64, int8, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_EXTEND16_S) + { + DEF_OP_CONVERT(int64, I64, int16, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_I64_EXTEND32_S) + { + DEF_OP_CONVERT(int64, I64, int32, I64); + HANDLE_OP_END(); + } + + HANDLE_OP(WASM_OP_MISC_PREFIX) + { + GET_OPCODE(); + switch (opcode) { + case WASM_OP_I32_TRUNC_SAT_S_F32: + DEF_OP_TRUNC_SAT_F32(-2147483904.0f, 2147483648.0f, + true, true); + break; + case WASM_OP_I32_TRUNC_SAT_U_F32: + DEF_OP_TRUNC_SAT_F32(-1.0f, 4294967296.0f, true, false); + break; + case WASM_OP_I32_TRUNC_SAT_S_F64: + DEF_OP_TRUNC_SAT_F64(-2147483649.0, 2147483648.0, true, + true); + break; + case WASM_OP_I32_TRUNC_SAT_U_F64: + DEF_OP_TRUNC_SAT_F64(-1.0, 4294967296.0, true, false); + break; + case WASM_OP_I64_TRUNC_SAT_S_F32: + DEF_OP_TRUNC_SAT_F32(-9223373136366403584.0f, + 9223372036854775808.0f, false, + true); + break; + case WASM_OP_I64_TRUNC_SAT_U_F32: + DEF_OP_TRUNC_SAT_F32(-1.0f, 18446744073709551616.0f, + false, false); + break; + case WASM_OP_I64_TRUNC_SAT_S_F64: + DEF_OP_TRUNC_SAT_F64(-9223372036854777856.0, + 9223372036854775808.0, false, + true); + break; + case WASM_OP_I64_TRUNC_SAT_U_F64: + DEF_OP_TRUNC_SAT_F64(-1.0, 18446744073709551616.0, + false, false); + break; +#if WASM_ENABLE_BULK_MEMORY != 0 + case WASM_OP_MEMORY_INIT: + { + uint32 addr, segment; + uint64 bytes, offset, seg_len; + uint8 *data; + + segment = read_uint32(frame_ip); + + bytes = (uint64)(uint32)POP_I32(); + offset = (uint64)(uint32)POP_I32(); + addr = POP_I32(); + +#if WASM_ENABLE_THREAD_MGR != 0 + linear_mem_size = get_linear_mem_size(); +#endif + +#ifndef OS_ENABLE_HW_BOUND_CHECK + CHECK_BULK_MEMORY_OVERFLOW(addr, bytes, maddr); +#else +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)(uint32)addr, + bytes)) + shared_heap_addr_app_to_native((uint64)(uint32)addr, + maddr); + else +#endif + { + if ((uint64)(uint32)addr + bytes > linear_mem_size) + goto out_of_bounds; + maddr = memory->memory_data + (uint32)addr; + } +#endif + if (bh_bitmap_get_bit(module->e->common.data_dropped, + segment)) { + seg_len = 0; + data = NULL; + } + else { + seg_len = + (uint64)module->module->data_segments[segment] + ->data_length; + data = module->module->data_segments[segment]->data; + } + if (offset + bytes > seg_len) + goto out_of_bounds; + + bh_memcpy_s(maddr, (uint32)(linear_mem_size - addr), + data + offset, (uint32)bytes); + break; + } + case WASM_OP_DATA_DROP: + { + uint32 segment; + + segment = read_uint32(frame_ip); + bh_bitmap_set_bit(module->e->common.data_dropped, + segment); + break; + } +#endif /* WASM_ENABLE_BULK_MEMORY */ +#if WASM_ENABLE_BULK_MEMORY_OPT != 0 + case WASM_OP_MEMORY_COPY: + { + uint32 dst, src, len; + uint8 *mdst, *msrc; + + len = POP_I32(); + src = POP_I32(); + dst = POP_I32(); + +#if WASM_ENABLE_THREAD_MGR != 0 + linear_mem_size = get_linear_mem_size(); +#endif + +#ifndef OS_ENABLE_HW_BOUND_CHECK + CHECK_BULK_MEMORY_OVERFLOW(src, len, msrc); + CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); +#else /* else of OS_ENABLE_HW_BOUND_CHECK */ +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)src, len)) + shared_heap_addr_app_to_native((uint64)src, msrc); + else +#endif + { + if ((uint64)(uint32)src + len > linear_mem_size) + goto out_of_bounds; + msrc = memory->memory_data + (uint32)src; + } + +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)dst, len)) { + shared_heap_addr_app_to_native((uint64)dst, mdst); + } + else +#endif + { + if ((uint64)(uint32)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + (uint32)dst; + } +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ + + /* + * avoid unnecessary operations + * + * since dst and src both are valid indexes in the + * linear memory, mdst and msrc can't be NULL + * + * The spec. converts memory.copy into i32.load8 and + * i32.store8; the following are runtime-specific + * optimizations. + * + */ + if (len && mdst != msrc) { + /* allowing the destination and source to overlap */ + memmove(mdst, msrc, len); + } + break; + } + case WASM_OP_MEMORY_FILL: + { + uint32 dst, len; + uint8 fill_val, *mdst; + + len = POP_I32(); + fill_val = POP_I32(); + dst = POP_I32(); + +#if WASM_ENABLE_THREAD_MGR != 0 + linear_mem_size = get_linear_mem_size(); +#endif + +#ifndef OS_ENABLE_HW_BOUND_CHECK + CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); +#else +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)(uint32)dst, len)) + shared_heap_addr_app_to_native((uint64)(uint32)dst, + mdst); + else +#endif + { + if ((uint64)(uint32)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + (uint32)dst; + } +#endif + + memset(mdst, fill_val, len); + break; + } +#endif /* WASM_ENABLE_BULK_MEMORY_OPT */ +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + case WASM_OP_TABLE_INIT: + { + uint32 tbl_idx, elem_idx; + uint32 n, s, d; + WASMTableInstance *tbl_inst; + table_elem_type_t *table_elems; + InitializerExpression *tbl_seg_init_values = NULL, + *init_values; + uint64 i; + uint32 tbl_seg_len = 0; + + elem_idx = read_uint32(frame_ip); + bh_assert(elem_idx < module->module->table_seg_count); + + tbl_idx = read_uint32(frame_ip); + bh_assert(tbl_idx < module->module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + n = (uint32)POP_I32(); + s = (uint32)POP_I32(); + d = (uint32)POP_I32(); + + if (!bh_bitmap_get_bit(module->e->common.elem_dropped, + elem_idx)) { + /* table segment isn't dropped */ + tbl_seg_init_values = + module->module->table_segments[elem_idx] + .init_values; + tbl_seg_len = + module->module->table_segments[elem_idx] + .value_count; + } + + if (offset_len_out_of_bounds(s, n, tbl_seg_len) + || offset_len_out_of_bounds(d, n, + tbl_inst->cur_size)) { + wasm_set_exception(module, + "out of bounds table access"); + goto got_exception; + } + + if (!n) { + break; + } + + table_elems = tbl_inst->elems + d; + init_values = tbl_seg_init_values + s; +#if WASM_ENABLE_GC != 0 + SYNC_ALL_TO_FRAME(); +#endif + for (i = 0; i < n; i++) { + /* UINT32_MAX indicates that it is a null ref */ + bh_assert(init_values[i].init_expr_type + == INIT_EXPR_TYPE_REFNULL_CONST + || init_values[i].init_expr_type + == INIT_EXPR_TYPE_FUNCREF_CONST); +#if WASM_ENABLE_GC == 0 + table_elems[i] = (table_elem_type_t)init_values[i] + .u.unary.v.ref_index; +#else + if (init_values[i].u.unary.v.ref_index + != UINT32_MAX) { + if (!(func_obj = wasm_create_func_obj( + module, + init_values[i].u.unary.v.ref_index, + true, NULL, 0))) { + goto got_exception; + } + table_elems[i] = func_obj; + } + else { + table_elems[i] = NULL_REF; + } +#endif + } + + break; + } + case WASM_OP_ELEM_DROP: + { + uint32 elem_idx = read_uint32(frame_ip); + bh_assert(elem_idx < module->module->table_seg_count); + bh_bitmap_set_bit(module->e->common.elem_dropped, + elem_idx); + break; + } + case WASM_OP_TABLE_COPY: + { + uint32 src_tbl_idx, dst_tbl_idx; + uint32 n, s, d; + WASMTableInstance *src_tbl_inst, *dst_tbl_inst; + + dst_tbl_idx = read_uint32(frame_ip); + bh_assert(dst_tbl_idx < module->table_count); + + dst_tbl_inst = wasm_get_table_inst(module, dst_tbl_idx); + + src_tbl_idx = read_uint32(frame_ip); + bh_assert(src_tbl_idx < module->table_count); + + src_tbl_inst = wasm_get_table_inst(module, src_tbl_idx); + + n = (uint32)POP_I32(); + s = (uint32)POP_I32(); + d = (uint32)POP_I32(); + + if (offset_len_out_of_bounds(d, n, + dst_tbl_inst->cur_size) + || offset_len_out_of_bounds( + s, n, src_tbl_inst->cur_size)) { + wasm_set_exception(module, + "out of bounds table access"); + goto got_exception; + } + + /* if s >= d, copy from front to back */ + /* if s < d, copy from back to front */ + /* merge all together */ + bh_memmove_s((uint8 *)dst_tbl_inst + + offsetof(WASMTableInstance, elems) + + d * sizeof(table_elem_type_t), + (uint32)((dst_tbl_inst->cur_size - d) + * sizeof(table_elem_type_t)), + (uint8 *)src_tbl_inst + + offsetof(WASMTableInstance, elems) + + s * sizeof(table_elem_type_t), + (uint32)(n * sizeof(table_elem_type_t))); + break; + } + case WASM_OP_TABLE_GROW: + { + uint32 tbl_idx, n, orig_tbl_sz; + WASMTableInstance *tbl_inst; + table_elem_type_t init_val; + + tbl_idx = read_uint32(frame_ip); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + orig_tbl_sz = tbl_inst->cur_size; + + n = POP_I32(); +#if WASM_ENABLE_GC == 0 + init_val = POP_I32(); +#else + init_val = POP_REF(); +#endif + + if (!wasm_enlarge_table(module, tbl_idx, n, init_val)) { + PUSH_I32(-1); + } + else { + PUSH_I32(orig_tbl_sz); + } + + break; + } + case WASM_OP_TABLE_SIZE: + { + uint32 tbl_idx; + WASMTableInstance *tbl_inst; + + tbl_idx = read_uint32(frame_ip); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + PUSH_I32(tbl_inst->cur_size); + break; + } + case WASM_OP_TABLE_FILL: + { + uint32 tbl_idx, n, i; + WASMTableInstance *tbl_inst; + table_elem_type_t fill_val; + + tbl_idx = read_uint32(frame_ip); + bh_assert(tbl_idx < module->table_count); + + tbl_inst = wasm_get_table_inst(module, tbl_idx); + + n = POP_I32(); +#if WASM_ENABLE_GC == 0 + fill_val = POP_I32(); +#else + fill_val = POP_REF(); +#endif + i = POP_I32(); + + if (offset_len_out_of_bounds(i, n, + tbl_inst->cur_size)) { + wasm_set_exception(module, + "out of bounds table access"); + goto got_exception; + } + + for (; n != 0; i++, n--) { + tbl_inst->elems[i] = fill_val; + } + + break; + } +#endif /* WASM_ENABLE_REF_TYPES */ + default: + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } + HANDLE_OP_END(); + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + HANDLE_OP(WASM_OP_ATOMIC_PREFIX) + { + uint32 offset = 0, addr; + + GET_OPCODE(); + + if (opcode != WASM_OP_ATOMIC_FENCE) { + offset = read_uint32(frame_ip); + } + + switch (opcode) { + case WASM_OP_ATOMIC_NOTIFY: + { + uint32 notify_count, ret; + + notify_count = POP_I32(); + addr = POP_I32(); + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(4); + + ret = wasm_runtime_atomic_notify( + (WASMModuleInstanceCommon *)module, maddr, + notify_count); + if (ret == (uint32)-1) + goto got_exception; + + PUSH_I32(ret); + break; + } + case WASM_OP_ATOMIC_WAIT32: + { + uint64 timeout; + uint32 expect, ret; + + timeout = POP_I64(); + expect = POP_I32(); + addr = POP_I32(); + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(4); + + ret = wasm_runtime_atomic_wait( + (WASMModuleInstanceCommon *)module, maddr, + (uint64)expect, timeout, false); + if (ret == (uint32)-1) + goto got_exception; + +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + + PUSH_I32(ret); + break; + } + case WASM_OP_ATOMIC_WAIT64: + { + uint64 timeout, expect; + uint32 ret; + + timeout = POP_I64(); + expect = POP_I64(); + addr = POP_I32(); + CHECK_MEMORY_OVERFLOW(8); + CHECK_ATOMIC_MEMORY_ACCESS(8); + + ret = wasm_runtime_atomic_wait( + (WASMModuleInstanceCommon *)module, maddr, expect, + timeout, true); + if (ret == (uint32)-1) + goto got_exception; + +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + + PUSH_I32(ret); + break; + } + case WASM_OP_ATOMIC_FENCE: + { + os_atomic_thread_fence(os_memory_order_seq_cst); + break; + } + + case WASM_OP_ATOMIC_I32_LOAD: + case WASM_OP_ATOMIC_I32_LOAD8_U: + case WASM_OP_ATOMIC_I32_LOAD16_U: + { + uint32 readv; + + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(1); + shared_memory_lock(memory); + readv = (uint32)(*(uint8 *)maddr); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I32_LOAD16_U) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(2); + shared_memory_lock(memory); + readv = (uint32)LOAD_U16(maddr); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(4); + shared_memory_lock(memory); + readv = LOAD_I32(maddr); + shared_memory_unlock(memory); + } + + PUSH_I32(readv); + break; + } + + case WASM_OP_ATOMIC_I64_LOAD: + case WASM_OP_ATOMIC_I64_LOAD8_U: + case WASM_OP_ATOMIC_I64_LOAD16_U: + case WASM_OP_ATOMIC_I64_LOAD32_U: + { + uint64 readv; + + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(1); + shared_memory_lock(memory); + readv = (uint64)(*(uint8 *)maddr); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I64_LOAD16_U) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(2); + shared_memory_lock(memory); + readv = (uint64)LOAD_U16(maddr); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I64_LOAD32_U) { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(4); + shared_memory_lock(memory); + readv = (uint64)LOAD_U32(maddr); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(8); + CHECK_ATOMIC_MEMORY_ACCESS(8); + shared_memory_lock(memory); + readv = LOAD_I64(maddr); + shared_memory_unlock(memory); + } + + PUSH_I64(readv); + break; + } + case WASM_OP_ATOMIC_I32_STORE: + case WASM_OP_ATOMIC_I32_STORE8: + case WASM_OP_ATOMIC_I32_STORE16: + { + uint32 sval; + + sval = (uint32)POP_I32(); + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_I32_STORE8) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(1); + shared_memory_lock(memory); + *(uint8 *)maddr = (uint8)sval; + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I32_STORE16) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(2); + shared_memory_lock(memory); + STORE_U16(maddr, (uint16)sval); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(4); + shared_memory_lock(memory); + STORE_U32(maddr, sval); + shared_memory_unlock(memory); + } + break; + } + + case WASM_OP_ATOMIC_I64_STORE: + case WASM_OP_ATOMIC_I64_STORE8: + case WASM_OP_ATOMIC_I64_STORE16: + case WASM_OP_ATOMIC_I64_STORE32: + { + uint64 sval; + + sval = (uint64)POP_I64(); + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_I64_STORE8) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(1); + shared_memory_lock(memory); + *(uint8 *)maddr = (uint8)sval; + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I64_STORE16) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(2); + shared_memory_lock(memory); + STORE_U16(maddr, (uint16)sval); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_I64_STORE32) { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(4); + shared_memory_lock(memory); + STORE_U32(maddr, (uint32)sval); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(8); + CHECK_ATOMIC_MEMORY_ACCESS(8); + shared_memory_lock(memory); + STORE_I64(maddr, sval); + shared_memory_unlock(memory); + } + break; + } + + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG: + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U: + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U: + { + uint32 readv, sval, expect; + + sval = POP_I32(); + expect = POP_I32(); + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(1); + + expect = (uint8)expect; + shared_memory_lock(memory); + readv = (uint32)(*(uint8 *)maddr); + if (readv == expect) + *(uint8 *)maddr = (uint8)(sval); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(2); + + expect = (uint16)expect; + shared_memory_lock(memory); + readv = (uint32)LOAD_U16(maddr); + if (readv == expect) + STORE_U16(maddr, (uint16)(sval)); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(4); + + shared_memory_lock(memory); + readv = LOAD_I32(maddr); + if (readv == expect) + STORE_U32(maddr, sval); + shared_memory_unlock(memory); + } + PUSH_I32(readv); + break; + } + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U: + { + uint64 readv, sval, expect; + + sval = (uint64)POP_I64(); + expect = (uint64)POP_I64(); + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U) { + CHECK_MEMORY_OVERFLOW(1); + CHECK_ATOMIC_MEMORY_ACCESS(1); + + expect = (uint8)expect; + shared_memory_lock(memory); + readv = (uint64)(*(uint8 *)maddr); + if (readv == expect) + *(uint8 *)maddr = (uint8)(sval); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U) { + CHECK_MEMORY_OVERFLOW(2); + CHECK_ATOMIC_MEMORY_ACCESS(2); + + expect = (uint16)expect; + shared_memory_lock(memory); + readv = (uint64)LOAD_U16(maddr); + if (readv == expect) + STORE_U16(maddr, (uint16)(sval)); + shared_memory_unlock(memory); + } + else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U) { + CHECK_MEMORY_OVERFLOW(4); + CHECK_ATOMIC_MEMORY_ACCESS(4); + + expect = (uint32)expect; + shared_memory_lock(memory); + readv = (uint64)LOAD_U32(maddr); + if (readv == expect) + STORE_U32(maddr, (uint32)(sval)); + shared_memory_unlock(memory); + } + else { + CHECK_MEMORY_OVERFLOW(8); + CHECK_ATOMIC_MEMORY_ACCESS(8); + + shared_memory_lock(memory); + readv = (uint64)LOAD_I64(maddr); + if (readv == expect) + STORE_I64(maddr, sval); + shared_memory_unlock(memory); + } + PUSH_I64(readv); + break; + } + + DEF_ATOMIC_RMW_OPCODE(ADD, +); + DEF_ATOMIC_RMW_OPCODE(SUB, -); + DEF_ATOMIC_RMW_OPCODE(AND, &); + DEF_ATOMIC_RMW_OPCODE(OR, |); + DEF_ATOMIC_RMW_OPCODE(XOR, ^); + /* xchg, ignore the read value, and store the given + value: readv * 0 + sval */ + DEF_ATOMIC_RMW_OPCODE(XCHG, *0 +); + } + + HANDLE_OP_END(); + } +#endif + + HANDLE_OP(WASM_OP_IMPDEP) + { + frame = prev_frame; + frame_ip = frame->ip; +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + is_return_call = false; +#endif + goto call_func_from_entry; + } +#if WASM_ENABLE_SIMDE != 0 +#define SIMD_V128_TO_SIMDE_V128(s_v) \ + ({ \ + bh_assert(sizeof(V128) == sizeof(simde_v128_t)); \ + simde_v128_t se_v; \ + bh_memcpy_s(&se_v, sizeof(simde_v128_t), &(s_v), sizeof(V128)); \ + se_v; \ + }) + +#define SIMDE_V128_TO_SIMD_V128(sv, v) \ + do { \ + bh_assert(sizeof(V128) == sizeof(simde_v128_t)); \ + bh_memcpy_s(&(v), sizeof(V128), &(sv), sizeof(simde_v128_t)); \ + } while (0) + + HANDLE_OP(WASM_OP_SIMD_PREFIX) + { + GET_OPCODE(); + + switch (opcode) { + /* Memory */ + case SIMD_v128_load: + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + addr = POP_I32(); + addr_ret = GET_OFFSET(); + CHECK_MEMORY_OVERFLOW(16); + PUT_V128_TO_ADDR(frame_lp + addr_ret, LOAD_V128(maddr)); + break; + } +#define SIMD_LOAD_OP(simde_func) \ + do { \ + uint32 offset, addr; \ + offset = read_uint32(frame_ip); \ + addr = POP_I32(); \ + addr_ret = GET_OFFSET(); \ + CHECK_MEMORY_OVERFLOW(8); \ + \ + simde_v128_t simde_result = simde_func(maddr); \ + \ + V128 result; \ + SIMDE_V128_TO_SIMD_V128(simde_result, result); \ + PUT_V128_TO_ADDR(frame_lp + addr_ret, result); \ + \ + } while (0) + case SIMD_v128_load8x8_s: + { + SIMD_LOAD_OP(simde_wasm_i16x8_load8x8); + break; + } + case SIMD_v128_load8x8_u: + { + SIMD_LOAD_OP(simde_wasm_u16x8_load8x8); + break; + } + case SIMD_v128_load16x4_s: + { + SIMD_LOAD_OP(simde_wasm_i32x4_load16x4); + break; + } + case SIMD_v128_load16x4_u: + { + SIMD_LOAD_OP(simde_wasm_u32x4_load16x4); + break; + } + case SIMD_v128_load32x2_s: + { + SIMD_LOAD_OP(simde_wasm_i64x2_load32x2); + break; + } + case SIMD_v128_load32x2_u: + { + SIMD_LOAD_OP(simde_wasm_u64x2_load32x2); + break; + } +#define SIMD_LOAD_SPLAT_OP(simde_func, width) \ + do { \ + uint32 offset, addr; \ + offset = read_uint32(frame_ip); \ + addr = POP_I32(); \ + addr_ret = GET_OFFSET(); \ + CHECK_MEMORY_OVERFLOW(width / 8); \ + \ + simde_v128_t simde_result = simde_func(maddr); \ + \ + V128 result; \ + SIMDE_V128_TO_SIMD_V128(simde_result, result); \ + \ + PUT_V128_TO_ADDR(frame_lp + addr_ret, result); \ + } while (0) + + case SIMD_v128_load8_splat: + { + SIMD_LOAD_SPLAT_OP(simde_wasm_v128_load8_splat, 8); + break; + } + case SIMD_v128_load16_splat: + { + SIMD_LOAD_SPLAT_OP(simde_wasm_v128_load16_splat, 16); + break; + } + case SIMD_v128_load32_splat: + { + SIMD_LOAD_SPLAT_OP(simde_wasm_v128_load32_splat, 32); + break; + } + case SIMD_v128_load64_splat: + { + SIMD_LOAD_SPLAT_OP(simde_wasm_v128_load64_splat, 64); + break; + } + case SIMD_v128_store: + { + uint32 offset, addr; + offset = read_uint32(frame_ip); + V128 data = POP_V128(); + addr = POP_I32(); + + CHECK_MEMORY_OVERFLOW(16); + STORE_V128(maddr, data); + break; + } + + /* Basic */ + case SIMD_v128_const: + { + uint8 *orig_ip = frame_ip; + + frame_ip += sizeof(V128); + addr_ret = GET_OFFSET(); + + PUT_V128_TO_ADDR(frame_lp + addr_ret, *(V128 *)orig_ip); + break; + } + /* TODO: Add a faster SIMD implementation */ + case SIMD_v8x16_shuffle: + { + V128 indices; + bh_memcpy_s(&indices, sizeof(V128), frame_ip, + sizeof(V128)); + frame_ip += sizeof(V128); + + V128 v2 = POP_V128(); + V128 v1 = POP_V128(); + addr_ret = GET_OFFSET(); + + V128 result; + for (int i = 0; i < 16; i++) { + uint8_t index = indices.i8x16[i]; + if (index < 16) { + result.i8x16[i] = v1.i8x16[index]; + } + else { + result.i8x16[i] = v2.i8x16[index - 16]; + } + } + + PUT_V128_TO_ADDR(frame_lp + addr_ret, result); + break; + } + case SIMD_v8x16_swizzle: + { + V128 v2 = POP_V128(); + V128 v1 = POP_V128(); + addr_ret = GET_OFFSET(); + simde_v128_t simde_result = simde_wasm_i8x16_swizzle( + SIMD_V128_TO_SIMDE_V128(v1), + SIMD_V128_TO_SIMDE_V128(v2)); + + V128 result; + SIMDE_V128_TO_SIMD_V128(simde_result, result); + + PUT_V128_TO_ADDR(frame_lp + addr_ret, result); + break; + } + + /* Splat */ +#define SIMD_SPLAT_OP(simde_func, pop_func, val_type) \ + do { \ + val_type v = pop_func(); \ + addr_ret = GET_OFFSET(); \ + \ + simde_v128_t simde_result = simde_func(v); \ + \ + V128 result; \ + SIMDE_V128_TO_SIMD_V128(simde_result, result); \ + \ + PUT_V128_TO_ADDR(frame_lp + addr_ret, result); \ + } while (0) + +#define SIMD_SPLAT_OP_I32(simde_func) SIMD_SPLAT_OP(simde_func, POP_I32, uint32) +#define SIMD_SPLAT_OP_I64(simde_func) SIMD_SPLAT_OP(simde_func, POP_I64, uint64) +#define SIMD_SPLAT_OP_F32(simde_func) \ + SIMD_SPLAT_OP(simde_func, POP_F32, float32) +#define SIMD_SPLAT_OP_F64(simde_func) \ + SIMD_SPLAT_OP(simde_func, POP_F64, float64) + + case SIMD_i8x16_splat: + { + val = POP_I32(); + addr_ret = GET_OFFSET(); + + simde_v128_t simde_result = simde_wasm_i8x16_splat(val); + + V128 result; + SIMDE_V128_TO_SIMD_V128(simde_result, result); + + PUT_V128_TO_ADDR(frame_lp + addr_ret, result); + break; + } + case SIMD_i16x8_splat: + { + SIMD_SPLAT_OP_I32(simde_wasm_i16x8_splat); + break; + } + case SIMD_i32x4_splat: + { + SIMD_SPLAT_OP_I32(simde_wasm_i32x4_splat); + break; + } + case SIMD_i64x2_splat: + { + SIMD_SPLAT_OP_I64(simde_wasm_i64x2_splat); + break; + } + case SIMD_f32x4_splat: + { + SIMD_SPLAT_OP_F32(simde_wasm_f32x4_splat); + break; + } + case SIMD_f64x2_splat: + { + SIMD_SPLAT_OP_F64(simde_wasm_f64x2_splat); + break; + } +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 +#define SIMD_LANE_HANDLE_UNALIGNED_ACCESS() +#else +#define SIMD_LANE_HANDLE_UNALIGNED_ACCESS() (void)*frame_ip++ +#endif /* WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 */ + +#define SIMD_EXTRACT_LANE_OP(register, return_type, push_elem) \ + do { \ + uint8 lane = *frame_ip++; \ + SIMD_LANE_HANDLE_UNALIGNED_ACCESS(); \ + V128 v = POP_V128(); \ + push_elem((return_type)(v.register[lane])); \ + } while (0) +#define SIMD_REPLACE_LANE_OP(register, return_type, pop_elem) \ + do { \ + uint8 lane = *frame_ip++; \ + SIMD_LANE_HANDLE_UNALIGNED_ACCESS(); \ + return_type replacement = pop_elem(); \ + V128 v = POP_V128(); \ + v.register[lane] = replacement; \ + addr_ret = GET_OFFSET(); \ + PUT_V128_TO_ADDR(frame_lp + addr_ret, v); \ + } while (0) + case SIMD_i8x16_extract_lane_s: + { + SIMD_EXTRACT_LANE_OP(i8x16, int8, PUSH_I32); + break; + } + case SIMD_i8x16_extract_lane_u: + { + SIMD_EXTRACT_LANE_OP(i8x16, uint8, PUSH_I32); + break; + } + case SIMD_i8x16_replace_lane: + { + SIMD_REPLACE_LANE_OP(i8x16, int8, POP_I32); + break; + } + case SIMD_i16x8_extract_lane_s: + { + SIMD_EXTRACT_LANE_OP(i16x8, int16, PUSH_I32); + break; + } + case SIMD_i16x8_extract_lane_u: + { + SIMD_EXTRACT_LANE_OP(i16x8, uint16, PUSH_I32); + break; + } + case SIMD_i16x8_replace_lane: + { + SIMD_REPLACE_LANE_OP(i16x8, int16, POP_I32); + break; + } + case SIMD_i32x4_extract_lane: + { + SIMD_EXTRACT_LANE_OP(i32x4, int32, PUSH_I32); + break; + } + case SIMD_i32x4_replace_lane: + { + SIMD_REPLACE_LANE_OP(i32x4, int32, POP_I32); + break; + } + case SIMD_i64x2_extract_lane: + { + SIMD_EXTRACT_LANE_OP(i64x2, int64, PUSH_I64); + break; + } + case SIMD_i64x2_replace_lane: + { + SIMD_REPLACE_LANE_OP(i64x2, int64, POP_I64); + break; + } + case SIMD_f32x4_extract_lane: + { + SIMD_EXTRACT_LANE_OP(f32x4, float32, PUSH_F32); + break; + } + case SIMD_f32x4_replace_lane: + { + SIMD_REPLACE_LANE_OP(f32x4, float32, POP_F32); + break; + } + case SIMD_f64x2_extract_lane: + { + SIMD_EXTRACT_LANE_OP(f64x2, float64, PUSH_F64); + break; + } + case SIMD_f64x2_replace_lane: + { + SIMD_REPLACE_LANE_OP(f64x2, float64, POP_F64); + break; + } + +#define SIMD_DOUBLE_OP(simde_func) \ + do { \ + V128 v2 = POP_V128(); \ + V128 v1 = POP_V128(); \ + addr_ret = GET_OFFSET(); \ + \ + simde_v128_t simde_result = simde_func(SIMD_V128_TO_SIMDE_V128(v1), \ + SIMD_V128_TO_SIMDE_V128(v2)); \ + \ + V128 result; \ + SIMDE_V128_TO_SIMD_V128(simde_result, result); \ + \ + PUT_V128_TO_ADDR(frame_lp + addr_ret, result); \ + } while (0) + + /* i8x16 comparison operations */ + case SIMD_i8x16_eq: + { + V128 v2 = POP_V128(); + V128 v1 = POP_V128(); + addr_ret = GET_OFFSET(); + + simde_v128_t simde_result = + simde_wasm_i8x16_eq(SIMD_V128_TO_SIMDE_V128(v1), + SIMD_V128_TO_SIMDE_V128(v2)); + + V128 result; + SIMDE_V128_TO_SIMD_V128(simde_result, result); + + PUT_V128_TO_ADDR(frame_lp + addr_ret, result); + break; + } + case SIMD_i8x16_ne: + { + SIMD_DOUBLE_OP(simde_wasm_i8x16_ne); + break; + } + case SIMD_i8x16_lt_s: + { + SIMD_DOUBLE_OP(simde_wasm_i8x16_lt); + break; + } + case SIMD_i8x16_lt_u: + { + SIMD_DOUBLE_OP(simde_wasm_u8x16_lt); + break; + } + case SIMD_i8x16_gt_s: + { + SIMD_DOUBLE_OP(simde_wasm_i8x16_gt); + break; + } + case SIMD_i8x16_gt_u: + { + SIMD_DOUBLE_OP(simde_wasm_u8x16_gt); + break; + } + case SIMD_i8x16_le_s: + { + SIMD_DOUBLE_OP(simde_wasm_i8x16_le); + break; + } + case SIMD_i8x16_le_u: + { + SIMD_DOUBLE_OP(simde_wasm_u8x16_le); + break; + } + case SIMD_i8x16_ge_s: + { + SIMD_DOUBLE_OP(simde_wasm_i8x16_ge); + break; + } + case SIMD_i8x16_ge_u: + { + SIMD_DOUBLE_OP(simde_wasm_u8x16_ge); + break; + } + + /* i16x8 comparison operations */ + case SIMD_i16x8_eq: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_eq); + break; + } + case SIMD_i16x8_ne: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_ne); + break; + } + case SIMD_i16x8_lt_s: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_lt); + break; + } + case SIMD_i16x8_lt_u: + { + SIMD_DOUBLE_OP(simde_wasm_u16x8_lt); + break; + } + case SIMD_i16x8_gt_s: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_gt); + break; + } + case SIMD_i16x8_gt_u: + { + SIMD_DOUBLE_OP(simde_wasm_u16x8_gt); + break; + } + case SIMD_i16x8_le_s: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_le); + break; + } + case SIMD_i16x8_le_u: + { + SIMD_DOUBLE_OP(simde_wasm_u16x8_le); + break; + } + case SIMD_i16x8_ge_s: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_ge); + break; + } + case SIMD_i16x8_ge_u: + { + SIMD_DOUBLE_OP(simde_wasm_u16x8_ge); + break; + } + + /* i32x4 comparison operations */ + case SIMD_i32x4_eq: + { + SIMD_DOUBLE_OP(simde_wasm_i32x4_eq); + break; + } + case SIMD_i32x4_ne: + { + SIMD_DOUBLE_OP(simde_wasm_i32x4_ne); + break; + } + case SIMD_i32x4_lt_s: + { + SIMD_DOUBLE_OP(simde_wasm_i32x4_lt); + break; + } + case SIMD_i32x4_lt_u: + { + SIMD_DOUBLE_OP(simde_wasm_u32x4_lt); + break; + } + case SIMD_i32x4_gt_s: + { + SIMD_DOUBLE_OP(simde_wasm_i32x4_gt); + break; + } + case SIMD_i32x4_gt_u: + { + SIMD_DOUBLE_OP(simde_wasm_u32x4_gt); + break; + } + case SIMD_i32x4_le_s: + { + SIMD_DOUBLE_OP(simde_wasm_i32x4_le); + break; + } + case SIMD_i32x4_le_u: + { + SIMD_DOUBLE_OP(simde_wasm_u32x4_le); + break; + } + case SIMD_i32x4_ge_s: + { + SIMD_DOUBLE_OP(simde_wasm_i32x4_ge); + break; + } + case SIMD_i32x4_ge_u: + { + SIMD_DOUBLE_OP(simde_wasm_u32x4_ge); + break; + } + + /* f32x4 comparison operations */ + case SIMD_f32x4_eq: + { + SIMD_DOUBLE_OP(simde_wasm_f32x4_eq); + break; + } + case SIMD_f32x4_ne: + { + SIMD_DOUBLE_OP(simde_wasm_f32x4_ne); + break; + } + case SIMD_f32x4_lt: + { + SIMD_DOUBLE_OP(simde_wasm_f32x4_lt); + break; + } + case SIMD_f32x4_gt: + { + SIMD_DOUBLE_OP(simde_wasm_f32x4_gt); + break; + } + case SIMD_f32x4_le: + { + SIMD_DOUBLE_OP(simde_wasm_f32x4_le); + break; + } + case SIMD_f32x4_ge: + { + SIMD_DOUBLE_OP(simde_wasm_f32x4_ge); + break; + } + + /* f64x2 comparison operations */ + case SIMD_f64x2_eq: + { + SIMD_DOUBLE_OP(simde_wasm_f64x2_eq); + break; + } + case SIMD_f64x2_ne: + { + SIMD_DOUBLE_OP(simde_wasm_f64x2_ne); + break; + } + case SIMD_f64x2_lt: + { + SIMD_DOUBLE_OP(simde_wasm_f64x2_lt); + break; + } + case SIMD_f64x2_gt: + { + SIMD_DOUBLE_OP(simde_wasm_f64x2_gt); + break; + } + case SIMD_f64x2_le: + { + SIMD_DOUBLE_OP(simde_wasm_f64x2_le); + break; + } + case SIMD_f64x2_ge: + { + SIMD_DOUBLE_OP(simde_wasm_f64x2_ge); + break; + } + + /* v128 bitwise operations */ +#define SIMD_V128_BITWISE_OP_COMMON(result_expr_0, result_expr_1) \ + do { \ + V128 result; \ + result.i64x2[0] = (result_expr_0); \ + result.i64x2[1] = (result_expr_1); \ + addr_ret = GET_OFFSET(); \ + PUT_V128_TO_ADDR(frame_lp + addr_ret, result); \ + } while (0) + + case SIMD_v128_not: + { + V128 value = POP_V128(); + SIMD_V128_BITWISE_OP_COMMON(~value.i64x2[0], + ~value.i64x2[1]); + break; + } + case SIMD_v128_and: + { + V128 v2 = POP_V128(); + V128 v1 = POP_V128(); + SIMD_V128_BITWISE_OP_COMMON(v1.i64x2[0] & v2.i64x2[0], + v1.i64x2[1] & v2.i64x2[1]); + break; + } + case SIMD_v128_andnot: + { + V128 v2 = POP_V128(); + V128 v1 = POP_V128(); + SIMD_V128_BITWISE_OP_COMMON( + v1.i64x2[0] & (~v2.i64x2[0]), + v1.i64x2[1] & (~v2.i64x2[1])); + break; + } + case SIMD_v128_or: + { + V128 v2 = POP_V128(); + V128 v1 = POP_V128(); + SIMD_V128_BITWISE_OP_COMMON(v1.i64x2[0] | v2.i64x2[0], + v1.i64x2[1] | v2.i64x2[1]); + break; + } + case SIMD_v128_xor: + { + V128 v2 = POP_V128(); + V128 v1 = POP_V128(); + SIMD_V128_BITWISE_OP_COMMON(v1.i64x2[0] ^ v2.i64x2[0], + v1.i64x2[1] ^ v2.i64x2[1]); + break; + } + case SIMD_v128_bitselect: + { + V128 v1 = POP_V128(); + V128 v2 = POP_V128(); + V128 v3 = POP_V128(); + addr_ret = GET_OFFSET(); + + simde_v128_t simde_result = simde_wasm_v128_bitselect( + SIMD_V128_TO_SIMDE_V128(v3), + SIMD_V128_TO_SIMDE_V128(v2), + SIMD_V128_TO_SIMDE_V128(v1)); + + V128 result; + SIMDE_V128_TO_SIMD_V128(simde_result, result); + + PUT_V128_TO_ADDR(frame_lp + addr_ret, result); + break; + } + case SIMD_v128_any_true: + { + V128 value = POP_V128(); + addr_ret = GET_OFFSET(); + frame_lp[addr_ret] = + value.i64x2[0] != 0 || value.i64x2[1] != 0; + break; + } + +#define SIMD_LOAD_LANE_COMMON(vec, register, lane, width) \ + do { \ + addr_ret = GET_OFFSET(); \ + CHECK_MEMORY_OVERFLOW(width / 8); \ + if (width == 64) { \ + vec.register[lane] = GET_I64_FROM_ADDR((uint32 *)maddr); \ + } \ + else { \ + vec.register[lane] = *(uint##width *)(maddr); \ + } \ + PUT_V128_TO_ADDR(frame_lp + addr_ret, vec); \ + } while (0) + +#define SIMD_LOAD_LANE_OP(register, width) \ + do { \ + uint32 offset, addr; \ + offset = read_uint32(frame_ip); \ + V128 vec = POP_V128(); \ + addr = POP_I32(); \ + int lane = *frame_ip++; \ + SIMD_LANE_HANDLE_UNALIGNED_ACCESS(); \ + SIMD_LOAD_LANE_COMMON(vec, register, lane, width); \ + } while (0) + + case SIMD_v128_load8_lane: + { + SIMD_LOAD_LANE_OP(i8x16, 8); + break; + } + case SIMD_v128_load16_lane: + { + SIMD_LOAD_LANE_OP(i16x8, 16); + break; + } + case SIMD_v128_load32_lane: + { + SIMD_LOAD_LANE_OP(i32x4, 32); + break; + } + case SIMD_v128_load64_lane: + { + SIMD_LOAD_LANE_OP(i64x2, 64); + break; + } +#define SIMD_STORE_LANE_OP(register, width) \ + do { \ + uint32 offset, addr; \ + offset = read_uint32(frame_ip); \ + V128 vec = POP_V128(); \ + addr = POP_I32(); \ + int lane = *frame_ip++; \ + SIMD_LANE_HANDLE_UNALIGNED_ACCESS(); \ + CHECK_MEMORY_OVERFLOW(width / 8); \ + if (width == 64) { \ + STORE_I64(maddr, vec.register[lane]); \ + } \ + else { \ + *(uint##width *)(maddr) = vec.register[lane]; \ + } \ + } while (0) + + case SIMD_v128_store8_lane: + { + SIMD_STORE_LANE_OP(i8x16, 8); + break; + } + + case SIMD_v128_store16_lane: + { + SIMD_STORE_LANE_OP(i16x8, 16); + break; + } + + case SIMD_v128_store32_lane: + { + SIMD_STORE_LANE_OP(i32x4, 32); + break; + } + + case SIMD_v128_store64_lane: + { + SIMD_STORE_LANE_OP(i64x2, 64); + break; + } +#define SIMD_LOAD_ZERO_OP(register, width) \ + do { \ + uint32 offset, addr; \ + offset = read_uint32(frame_ip); \ + addr = POP_I32(); \ + int32 lane = 0; \ + V128 vec = { 0 }; \ + SIMD_LOAD_LANE_COMMON(vec, register, lane, width); \ + } while (0) + + case SIMD_v128_load32_zero: + { + SIMD_LOAD_ZERO_OP(i32x4, 32); + break; + } + case SIMD_v128_load64_zero: + { + SIMD_LOAD_ZERO_OP(i64x2, 64); + break; + } + +#define SIMD_SINGLE_OP(simde_func) \ + do { \ + V128 v1 = POP_V128(); \ + addr_ret = GET_OFFSET(); \ + \ + simde_v128_t simde_result = simde_func(SIMD_V128_TO_SIMDE_V128(v1)); \ + \ + V128 result; \ + SIMDE_V128_TO_SIMD_V128(simde_result, result); \ + \ + PUT_V128_TO_ADDR(frame_lp + addr_ret, result); \ + } while (0) + + /* Float conversion */ + case SIMD_f32x4_demote_f64x2_zero: + { + SIMD_SINGLE_OP(simde_wasm_f32x4_demote_f64x2_zero); + break; + } + case SIMD_f64x2_promote_low_f32x4_zero: + { + SIMD_SINGLE_OP(simde_wasm_f64x2_promote_low_f32x4); + break; + } + + /* i8x16 operations */ + case SIMD_i8x16_abs: + { + SIMD_SINGLE_OP(simde_wasm_i8x16_abs); + break; + } + case SIMD_i8x16_neg: + { + SIMD_SINGLE_OP(simde_wasm_i8x16_neg); + break; + } + case SIMD_i8x16_popcnt: + { + SIMD_SINGLE_OP(simde_wasm_i8x16_popcnt); + break; + } + case SIMD_i8x16_all_true: + { + V128 v1 = POP_V128(); + + bool result = simde_wasm_i8x16_all_true( + SIMD_V128_TO_SIMDE_V128(v1)); + + addr_ret = GET_OFFSET(); + frame_lp[addr_ret] = result; + break; + } + + case SIMD_i8x16_bitmask: + { + V128 v1 = POP_V128(); + + uint32_t result = simde_wasm_i8x16_bitmask( + SIMD_V128_TO_SIMDE_V128(v1)); + + addr_ret = GET_OFFSET(); + frame_lp[addr_ret] = result; + break; + } + case SIMD_i8x16_narrow_i16x8_s: + { + SIMD_DOUBLE_OP(simde_wasm_i8x16_narrow_i16x8); + break; + } + case SIMD_i8x16_narrow_i16x8_u: + { + SIMD_DOUBLE_OP(simde_wasm_u8x16_narrow_i16x8); + break; + } + case SIMD_f32x4_ceil: + { + SIMD_SINGLE_OP(simde_wasm_f32x4_ceil); + break; + } + case SIMD_f32x4_floor: + { + SIMD_SINGLE_OP(simde_wasm_f32x4_floor); + break; + } + case SIMD_f32x4_trunc: + { + SIMD_SINGLE_OP(simde_wasm_f32x4_trunc); + break; + } + case SIMD_f32x4_nearest: + { + SIMD_SINGLE_OP(simde_wasm_f32x4_nearest); + break; + } +#define SIMD_LANE_SHIFT(simde_func) \ + do { \ + int32 c = POP_I32(); \ + V128 v1 = POP_V128(); \ + addr_ret = GET_OFFSET(); \ + \ + simde_v128_t simde_result = \ + simde_func(SIMD_V128_TO_SIMDE_V128(v1), c); \ + \ + V128 result; \ + SIMDE_V128_TO_SIMD_V128(simde_result, result); \ + \ + PUT_V128_TO_ADDR(frame_lp + addr_ret, result); \ + } while (0) + case SIMD_i8x16_shl: + { + SIMD_LANE_SHIFT(simde_wasm_i8x16_shl); + break; + } + case SIMD_i8x16_shr_s: + { + SIMD_LANE_SHIFT(simde_wasm_i8x16_shr); + break; + } + case SIMD_i8x16_shr_u: + { + SIMD_LANE_SHIFT(simde_wasm_u8x16_shr); + break; + } + case SIMD_i8x16_add: + { + SIMD_DOUBLE_OP(simde_wasm_i8x16_add); + break; + } + case SIMD_i8x16_add_sat_s: + { + SIMD_DOUBLE_OP(simde_wasm_i8x16_add_sat); + break; + } + case SIMD_i8x16_add_sat_u: + { + SIMD_DOUBLE_OP(simde_wasm_u8x16_add_sat); + break; + } + case SIMD_i8x16_sub: + { + SIMD_DOUBLE_OP(simde_wasm_i8x16_sub); + break; + } + case SIMD_i8x16_sub_sat_s: + { + SIMD_DOUBLE_OP(simde_wasm_i8x16_sub_sat); + break; + } + case SIMD_i8x16_sub_sat_u: + { + SIMD_DOUBLE_OP(simde_wasm_u8x16_sub_sat); + break; + } + case SIMD_f64x2_ceil: + { + SIMD_SINGLE_OP(simde_wasm_f64x2_ceil); + break; + } + case SIMD_f64x2_floor: + { + SIMD_SINGLE_OP(simde_wasm_f64x2_floor); + break; + } + case SIMD_i8x16_min_s: + { + SIMD_DOUBLE_OP(simde_wasm_i8x16_min); + break; + } + case SIMD_i8x16_min_u: + { + SIMD_DOUBLE_OP(simde_wasm_u8x16_min); + break; + } + case SIMD_i8x16_max_s: + { + SIMD_DOUBLE_OP(simde_wasm_i8x16_max); + break; + } + case SIMD_i8x16_max_u: + { + SIMD_DOUBLE_OP(simde_wasm_u8x16_max); + break; + } + case SIMD_f64x2_trunc: + { + SIMD_SINGLE_OP(simde_wasm_f64x2_trunc); + break; + } + case SIMD_i8x16_avgr_u: + { + SIMD_DOUBLE_OP(simde_wasm_u8x16_avgr); + break; + } + case SIMD_i16x8_extadd_pairwise_i8x16_s: + { + SIMD_SINGLE_OP(simde_wasm_i16x8_extadd_pairwise_i8x16); + break; + } + case SIMD_i16x8_extadd_pairwise_i8x16_u: + { + SIMD_SINGLE_OP(simde_wasm_u16x8_extadd_pairwise_u8x16); + break; + } + case SIMD_i32x4_extadd_pairwise_i16x8_s: + { + SIMD_SINGLE_OP(simde_wasm_i32x4_extadd_pairwise_i16x8); + break; + } + case SIMD_i32x4_extadd_pairwise_i16x8_u: + { + SIMD_SINGLE_OP(simde_wasm_u32x4_extadd_pairwise_u16x8); + break; + } + + /* i16x8 operations */ + case SIMD_i16x8_abs: + { + SIMD_SINGLE_OP(simde_wasm_i16x8_abs); + break; + } + case SIMD_i16x8_neg: + { + SIMD_SINGLE_OP(simde_wasm_i16x8_neg); + break; + } + case SIMD_i16x8_q15mulr_sat_s: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_q15mulr_sat); + break; + } + case SIMD_i16x8_all_true: + { + V128 v1 = POP_V128(); + + bool result = simde_wasm_i16x8_all_true( + SIMD_V128_TO_SIMDE_V128(v1)); + + addr_ret = GET_OFFSET(); + frame_lp[addr_ret] = result; + break; + } + case SIMD_i16x8_bitmask: + { + V128 v1 = POP_V128(); + + uint32_t result = simde_wasm_i16x8_bitmask( + SIMD_V128_TO_SIMDE_V128(v1)); + + addr_ret = GET_OFFSET(); + frame_lp[addr_ret] = result; + break; + } + case SIMD_i16x8_narrow_i32x4_s: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_narrow_i32x4); + break; + } + case SIMD_i16x8_narrow_i32x4_u: + { + SIMD_DOUBLE_OP(simde_wasm_u16x8_narrow_i32x4); + break; + } + case SIMD_i16x8_extend_low_i8x16_s: + { + SIMD_SINGLE_OP(simde_wasm_i16x8_extend_low_i8x16); + break; + } + case SIMD_i16x8_extend_high_i8x16_s: + { + SIMD_SINGLE_OP(simde_wasm_i16x8_extend_high_i8x16); + break; + } + case SIMD_i16x8_extend_low_i8x16_u: + { + SIMD_SINGLE_OP(simde_wasm_u16x8_extend_low_u8x16); + break; + } + case SIMD_i16x8_extend_high_i8x16_u: + { + SIMD_SINGLE_OP(simde_wasm_u16x8_extend_high_u8x16); + break; + } + case SIMD_i16x8_shl: + { + SIMD_LANE_SHIFT(simde_wasm_i16x8_shl); + break; + } + case SIMD_i16x8_shr_s: + { + SIMD_LANE_SHIFT(simde_wasm_i16x8_shr); + break; + } + case SIMD_i16x8_shr_u: + { + SIMD_LANE_SHIFT(simde_wasm_u16x8_shr); + break; + } + case SIMD_i16x8_add: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_add); + break; + } + case SIMD_i16x8_add_sat_s: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_add_sat); + break; + } + case SIMD_i16x8_add_sat_u: + { + SIMD_DOUBLE_OP(simde_wasm_u16x8_add_sat); + break; + } + case SIMD_i16x8_sub: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_sub); + break; + } + case SIMD_i16x8_sub_sat_s: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_sub_sat); + break; + } + case SIMD_i16x8_sub_sat_u: + { + SIMD_DOUBLE_OP(simde_wasm_u16x8_sub_sat); + break; + } + case SIMD_f64x2_nearest: + { + SIMD_SINGLE_OP(simde_wasm_f64x2_nearest); + break; + } + case SIMD_i16x8_mul: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_mul); + break; + } + case SIMD_i16x8_min_s: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_min); + break; + } + case SIMD_i16x8_min_u: + { + SIMD_DOUBLE_OP(simde_wasm_u16x8_min); + break; + } + case SIMD_i16x8_max_s: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_max); + break; + } + case SIMD_i16x8_max_u: + { + SIMD_DOUBLE_OP(simde_wasm_u16x8_max); + break; + } + case SIMD_i16x8_avgr_u: + { + SIMD_DOUBLE_OP(simde_wasm_u16x8_avgr); + break; + } + case SIMD_i16x8_extmul_low_i8x16_s: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_extmul_low_i8x16); + break; + } + case SIMD_i16x8_extmul_high_i8x16_s: + { + SIMD_DOUBLE_OP(simde_wasm_i16x8_extmul_high_i8x16); + break; + } + case SIMD_i16x8_extmul_low_i8x16_u: + { + SIMD_DOUBLE_OP(simde_wasm_u16x8_extmul_low_u8x16); + break; + } + case SIMD_i16x8_extmul_high_i8x16_u: + { + SIMD_DOUBLE_OP(simde_wasm_u16x8_extmul_high_u8x16); + break; + } + + /* i32x4 operations */ + case SIMD_i32x4_abs: + { + SIMD_SINGLE_OP(simde_wasm_i32x4_abs); + break; + } + case SIMD_i32x4_neg: + { + SIMD_SINGLE_OP(simde_wasm_i32x4_neg); + break; + } + case SIMD_i32x4_all_true: + { + V128 v1 = POP_V128(); + + bool result = simde_wasm_i32x4_all_true( + SIMD_V128_TO_SIMDE_V128(v1)); + + addr_ret = GET_OFFSET(); + frame_lp[addr_ret] = result; + break; + } + case SIMD_i32x4_bitmask: + { + V128 v1 = POP_V128(); + + uint32_t result = simde_wasm_i32x4_bitmask( + SIMD_V128_TO_SIMDE_V128(v1)); + + addr_ret = GET_OFFSET(); + frame_lp[addr_ret] = result; + break; + } + case SIMD_i32x4_extend_low_i16x8_s: + { + SIMD_SINGLE_OP(simde_wasm_i32x4_extend_low_i16x8); + break; + } + case SIMD_i32x4_extend_high_i16x8_s: + { + SIMD_SINGLE_OP(simde_wasm_i32x4_extend_high_i16x8); + break; + } + case SIMD_i32x4_extend_low_i16x8_u: + { + SIMD_SINGLE_OP(simde_wasm_u32x4_extend_low_u16x8); + break; + } + case SIMD_i32x4_extend_high_i16x8_u: + { + SIMD_SINGLE_OP(simde_wasm_u32x4_extend_high_u16x8); + break; + } + case SIMD_i32x4_shl: + { + SIMD_LANE_SHIFT(simde_wasm_i32x4_shl); + break; + } + case SIMD_i32x4_shr_s: + { + SIMD_LANE_SHIFT(simde_wasm_i32x4_shr); + break; + } + case SIMD_i32x4_shr_u: + { + SIMD_LANE_SHIFT(simde_wasm_u32x4_shr); + break; + } + case SIMD_i32x4_add: + { + SIMD_DOUBLE_OP(simde_wasm_i32x4_add); + break; + } + case SIMD_i32x4_sub: + { + SIMD_DOUBLE_OP(simde_wasm_i32x4_sub); + break; + } + case SIMD_i32x4_mul: + { + SIMD_DOUBLE_OP(simde_wasm_i32x4_mul); + break; + } + case SIMD_i32x4_min_s: + { + SIMD_DOUBLE_OP(simde_wasm_i32x4_min); + break; + } + case SIMD_i32x4_min_u: + { + SIMD_DOUBLE_OP(simde_wasm_u32x4_min); + break; + } + case SIMD_i32x4_max_s: + { + SIMD_DOUBLE_OP(simde_wasm_i32x4_max); + break; + } + case SIMD_i32x4_max_u: + { + SIMD_DOUBLE_OP(simde_wasm_u32x4_max); + break; + } + case SIMD_i32x4_dot_i16x8_s: + { + SIMD_DOUBLE_OP(simde_wasm_i32x4_dot_i16x8); + break; + } + case SIMD_i32x4_extmul_low_i16x8_s: + { + SIMD_DOUBLE_OP(simde_wasm_i32x4_extmul_low_i16x8); + break; + } + case SIMD_i32x4_extmul_high_i16x8_s: + { + SIMD_DOUBLE_OP(simde_wasm_i32x4_extmul_high_i16x8); + break; + } + case SIMD_i32x4_extmul_low_i16x8_u: + { + SIMD_DOUBLE_OP(simde_wasm_u32x4_extmul_low_u16x8); + break; + } + case SIMD_i32x4_extmul_high_i16x8_u: + { + SIMD_DOUBLE_OP(simde_wasm_u32x4_extmul_high_u16x8); + break; + } + + /* i64x2 operations */ + case SIMD_i64x2_abs: + { + SIMD_SINGLE_OP(simde_wasm_i64x2_abs); + break; + } + case SIMD_i64x2_neg: + { + SIMD_SINGLE_OP(simde_wasm_i64x2_neg); + break; + } + case SIMD_i64x2_all_true: + { + V128 v1 = POP_V128(); + + bool result = simde_wasm_i64x2_all_true( + SIMD_V128_TO_SIMDE_V128(v1)); + + addr_ret = GET_OFFSET(); + frame_lp[addr_ret] = result; + break; + } + case SIMD_i64x2_bitmask: + { + V128 v1 = POP_V128(); + + uint32_t result = simde_wasm_i64x2_bitmask( + SIMD_V128_TO_SIMDE_V128(v1)); + + addr_ret = GET_OFFSET(); + frame_lp[addr_ret] = result; + break; + } + case SIMD_i64x2_extend_low_i32x4_s: + { + SIMD_SINGLE_OP(simde_wasm_i64x2_extend_low_i32x4); + break; + } + case SIMD_i64x2_extend_high_i32x4_s: + { + SIMD_SINGLE_OP(simde_wasm_i64x2_extend_high_i32x4); + break; + } + case SIMD_i64x2_extend_low_i32x4_u: + { + SIMD_SINGLE_OP(simde_wasm_u64x2_extend_low_u32x4); + break; + } + case SIMD_i64x2_extend_high_i32x4_u: + { + SIMD_SINGLE_OP(simde_wasm_u64x2_extend_high_u32x4); + break; + } + case SIMD_i64x2_shl: + { + SIMD_LANE_SHIFT(simde_wasm_i64x2_shl); + break; + } + case SIMD_i64x2_shr_s: + { + SIMD_LANE_SHIFT(simde_wasm_i64x2_shr); + break; + } + case SIMD_i64x2_shr_u: + { + SIMD_LANE_SHIFT(simde_wasm_u64x2_shr); + break; + } + case SIMD_i64x2_add: + { + SIMD_DOUBLE_OP(simde_wasm_i64x2_add); + break; + } + case SIMD_i64x2_sub: + { + SIMD_DOUBLE_OP(simde_wasm_i64x2_sub); + break; + } + case SIMD_i64x2_mul: + { + SIMD_DOUBLE_OP(simde_wasm_i64x2_mul); + break; + } + case SIMD_i64x2_eq: + { + SIMD_DOUBLE_OP(simde_wasm_i64x2_eq); + break; + } + case SIMD_i64x2_ne: + { + SIMD_DOUBLE_OP(simde_wasm_i64x2_ne); + break; + } + case SIMD_i64x2_lt_s: + { + SIMD_DOUBLE_OP(simde_wasm_i64x2_lt); + break; + } + case SIMD_i64x2_gt_s: + { + SIMD_DOUBLE_OP(simde_wasm_i64x2_gt); + break; + } + case SIMD_i64x2_le_s: + { + SIMD_DOUBLE_OP(simde_wasm_i64x2_le); + break; + } + case SIMD_i64x2_ge_s: + { + SIMD_DOUBLE_OP(simde_wasm_i64x2_ge); + break; + } + case SIMD_i64x2_extmul_low_i32x4_s: + { + SIMD_DOUBLE_OP(simde_wasm_i64x2_extmul_low_i32x4); + break; + } + case SIMD_i64x2_extmul_high_i32x4_s: + { + SIMD_DOUBLE_OP(simde_wasm_i64x2_extmul_high_i32x4); + break; + } + case SIMD_i64x2_extmul_low_i32x4_u: + { + SIMD_DOUBLE_OP(simde_wasm_u64x2_extmul_low_u32x4); + break; + } + case SIMD_i64x2_extmul_high_i32x4_u: + { + SIMD_DOUBLE_OP(simde_wasm_u64x2_extmul_high_u32x4); + break; + } + + /* f32x4 opertions */ + case SIMD_f32x4_abs: + { + SIMD_SINGLE_OP(simde_wasm_f32x4_abs); + break; + } + case SIMD_f32x4_neg: + { + SIMD_SINGLE_OP(simde_wasm_f32x4_neg); + break; + } + case SIMD_f32x4_sqrt: + { + SIMD_SINGLE_OP(simde_wasm_f32x4_sqrt); + break; + } + case SIMD_f32x4_add: + { + SIMD_DOUBLE_OP(simde_wasm_f32x4_add); + break; + } + case SIMD_f32x4_sub: + { + SIMD_DOUBLE_OP(simde_wasm_f32x4_sub); + break; + } + case SIMD_f32x4_mul: + { + SIMD_DOUBLE_OP(simde_wasm_f32x4_mul); + break; + } + case SIMD_f32x4_div: + { + SIMD_DOUBLE_OP(simde_wasm_f32x4_div); + break; + } + case SIMD_f32x4_min: + { + SIMD_DOUBLE_OP(simde_wasm_f32x4_min); + break; + } + case SIMD_f32x4_max: + { + SIMD_DOUBLE_OP(simde_wasm_f32x4_max); + break; + } + case SIMD_f32x4_pmin: + { + SIMD_DOUBLE_OP(simde_wasm_f32x4_pmin); + break; + } + case SIMD_f32x4_pmax: + { + SIMD_DOUBLE_OP(simde_wasm_f32x4_pmax); + break; + } + + /* f64x2 operations */ + case SIMD_f64x2_abs: + { + SIMD_SINGLE_OP(simde_wasm_f64x2_abs); + break; + } + case SIMD_f64x2_neg: + { + SIMD_SINGLE_OP(simde_wasm_f64x2_neg); + break; + } + case SIMD_f64x2_sqrt: + { + SIMD_SINGLE_OP(simde_wasm_f64x2_sqrt); + break; + } + case SIMD_f64x2_add: + { + SIMD_DOUBLE_OP(simde_wasm_f64x2_add); + break; + } + case SIMD_f64x2_sub: + { + SIMD_DOUBLE_OP(simde_wasm_f64x2_sub); + break; + } + case SIMD_f64x2_mul: + { + SIMD_DOUBLE_OP(simde_wasm_f64x2_mul); + break; + } + case SIMD_f64x2_div: + { + SIMD_DOUBLE_OP(simde_wasm_f64x2_div); + break; + } + case SIMD_f64x2_min: + { + SIMD_DOUBLE_OP(simde_wasm_f64x2_min); + break; + } + case SIMD_f64x2_max: + { + SIMD_DOUBLE_OP(simde_wasm_f64x2_max); + break; + } + case SIMD_f64x2_pmin: + { + SIMD_DOUBLE_OP(simde_wasm_f64x2_pmin); + break; + } + case SIMD_f64x2_pmax: + { + SIMD_DOUBLE_OP(simde_wasm_f64x2_pmax); + break; + } + + /* Conversion operations */ + case SIMD_i32x4_trunc_sat_f32x4_s: + { + SIMD_SINGLE_OP(simde_wasm_i32x4_trunc_sat_f32x4); + break; + } + case SIMD_i32x4_trunc_sat_f32x4_u: + { + SIMD_SINGLE_OP(simde_wasm_u32x4_trunc_sat_f32x4); + break; + } + case SIMD_f32x4_convert_i32x4_s: + { + SIMD_SINGLE_OP(simde_wasm_f32x4_convert_i32x4); + break; + } + case SIMD_f32x4_convert_i32x4_u: + { + SIMD_SINGLE_OP(simde_wasm_f32x4_convert_u32x4); + break; + } + case SIMD_i32x4_trunc_sat_f64x2_s_zero: + { + SIMD_SINGLE_OP(simde_wasm_i32x4_trunc_sat_f64x2_zero); + break; + } + case SIMD_i32x4_trunc_sat_f64x2_u_zero: + { + SIMD_SINGLE_OP(simde_wasm_u32x4_trunc_sat_f64x2_zero); + break; + } + case SIMD_f64x2_convert_low_i32x4_s: + { + SIMD_SINGLE_OP(simde_wasm_f64x2_convert_low_i32x4); + break; + } + case SIMD_f64x2_convert_low_i32x4_u: + { + SIMD_SINGLE_OP(simde_wasm_f64x2_convert_low_u32x4); + break; + } + + default: + wasm_set_exception(module, "unsupported SIMD opcode"); + } + HANDLE_OP_END(); + } +#endif + + HANDLE_OP(WASM_OP_CALL) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + fidx = read_uint32(frame_ip); +#if WASM_ENABLE_MULTI_MODULE != 0 + if (fidx >= module->e->function_count) { + wasm_set_exception(module, "unknown function"); + goto got_exception; + } +#endif + cur_func = module->e->functions + fidx; + goto call_func_from_interp; + } + +#if WASM_ENABLE_TAIL_CALL != 0 + HANDLE_OP(WASM_OP_RETURN_CALL) + { +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + fidx = read_uint32(frame_ip); +#if WASM_ENABLE_MULTI_MODULE != 0 + if (fidx >= module->e->function_count) { + wasm_set_exception(module, "unknown function"); + goto got_exception; + } +#endif + cur_func = module->e->functions + fidx; + goto call_func_from_return_call; + } +#endif /* WASM_ENABLE_TAIL_CALL */ + +#if WASM_ENABLE_LABELS_AS_VALUES == 0 + default: + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } +#endif + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 + HANDLE_OP(WASM_OP_UNUSED_0x0a) +#if WASM_ENABLE_TAIL_CALL == 0 + HANDLE_OP(WASM_OP_RETURN_CALL) + HANDLE_OP(WASM_OP_RETURN_CALL_INDIRECT) +#endif +#if WASM_ENABLE_SHARED_MEMORY == 0 + HANDLE_OP(WASM_OP_ATOMIC_PREFIX) +#endif +#if WASM_ENABLE_REF_TYPES == 0 && WASM_ENABLE_GC == 0 + HANDLE_OP(WASM_OP_TABLE_GET) + HANDLE_OP(WASM_OP_TABLE_SET) + HANDLE_OP(WASM_OP_REF_NULL) + HANDLE_OP(WASM_OP_REF_IS_NULL) + HANDLE_OP(WASM_OP_REF_FUNC) +#endif +#if WASM_ENABLE_GC == 0 + /* SELECT_T is converted to SELECT or SELECT_64 */ + HANDLE_OP(WASM_OP_SELECT_T) + HANDLE_OP(WASM_OP_CALL_REF) + HANDLE_OP(WASM_OP_RETURN_CALL_REF) + HANDLE_OP(WASM_OP_REF_EQ) + HANDLE_OP(WASM_OP_REF_AS_NON_NULL) + HANDLE_OP(WASM_OP_BR_ON_NULL) + HANDLE_OP(WASM_OP_BR_ON_NON_NULL) + HANDLE_OP(WASM_OP_GC_PREFIX) +#endif +#if WASM_ENABLE_EXCE_HANDLING == 0 + /* if exception handling is disabled, these opcodes issue a trap */ + HANDLE_OP(WASM_OP_TRY) + HANDLE_OP(WASM_OP_CATCH) + HANDLE_OP(WASM_OP_THROW) + HANDLE_OP(WASM_OP_RETHROW) + HANDLE_OP(WASM_OP_DELEGATE) + HANDLE_OP(WASM_OP_CATCH_ALL) + HANDLE_OP(EXT_OP_TRY) +#endif + HANDLE_OP(WASM_OP_UNUSED_0x16) + HANDLE_OP(WASM_OP_UNUSED_0x17) + HANDLE_OP(WASM_OP_UNUSED_0x27) + /* optimized op code */ + HANDLE_OP(WASM_OP_F32_STORE) + HANDLE_OP(WASM_OP_F64_STORE) + HANDLE_OP(WASM_OP_F32_LOAD) + HANDLE_OP(WASM_OP_F64_LOAD) + HANDLE_OP(EXT_OP_GET_LOCAL_FAST) + HANDLE_OP(WASM_OP_GET_LOCAL) + HANDLE_OP(WASM_OP_DROP) + HANDLE_OP(WASM_OP_DROP_64) + HANDLE_OP(WASM_OP_BLOCK) + HANDLE_OP(WASM_OP_LOOP) + HANDLE_OP(WASM_OP_END) + HANDLE_OP(WASM_OP_NOP) + HANDLE_OP(EXT_OP_BLOCK) + HANDLE_OP(EXT_OP_LOOP) + HANDLE_OP(EXT_OP_IF) + HANDLE_OP(EXT_OP_BR_TABLE_CACHE) + { + wasm_set_exception(module, "unsupported opcode"); + goto got_exception; + } +#endif + +#if WASM_ENABLE_LABELS_AS_VALUES == 0 + continue; +#else + FETCH_OPCODE_AND_DISPATCH(); +#endif + +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + call_func_from_return_call: + { + uint32 *lp_base = NULL, *lp = NULL; + int i; + + if (cur_func->param_cell_num > 0 + && !(lp_base = lp = wasm_runtime_malloc(cur_func->param_cell_num + * sizeof(uint32)))) { + wasm_set_exception(module, "allocate memory failed"); + goto got_exception; + } + for (i = 0; i < cur_func->param_count; i++) { + if (cur_func->param_types[i] == VALUE_TYPE_I64 + || cur_func->param_types[i] == VALUE_TYPE_F64) { + PUT_I64_TO_ADDR( + lp, GET_OPERAND(uint64, I64, + 2 * (cur_func->param_count - i - 1))); + lp += 2; + } + else { + *lp = GET_OPERAND(uint32, I32, + (2 * (cur_func->param_count - i - 1))); + lp++; + } + } + frame->lp = frame->operand + cur_func->const_cell_num; + if (lp - lp_base > 0) { + word_copy(frame->lp, lp_base, lp - lp_base); + } + if (lp_base) + wasm_runtime_free(lp_base); + FREE_FRAME(exec_env, frame); + frame_ip += cur_func->param_count * sizeof(int16); + wasm_exec_env_set_cur_frame(exec_env, (WASMRuntimeFrame *)prev_frame); + is_return_call = true; + goto call_func_from_entry; + } +#endif /* WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 */ + + call_func_from_interp: + { + /* Only do the copy when it's called from interpreter. */ + WASMInterpFrame *outs_area = wasm_exec_env_wasm_stack_top(exec_env); + int i; + +#if WASM_ENABLE_MULTI_MODULE != 0 + if (cur_func->is_import_func) { + outs_area->lp = outs_area->operand + + (cur_func->import_func_inst + ? cur_func->import_func_inst->const_cell_num + : 0); + } + else +#endif + { + outs_area->lp = outs_area->operand + cur_func->const_cell_num; + } + + if ((uint8 *)(outs_area->lp + cur_func->param_cell_num) + > exec_env->wasm_stack.top_boundary) { + wasm_set_exception(module, "wasm operand stack overflow"); + goto got_exception; + } + + for (i = 0; i < cur_func->param_count; i++) { + if (cur_func->param_types[i] == VALUE_TYPE_V128) { + PUT_V128_TO_ADDR( + outs_area->lp, + GET_OPERAND_V128(2 * (cur_func->param_count - i - 1))); + outs_area->lp += 4; + } + else if (cur_func->param_types[i] == VALUE_TYPE_I64 + || cur_func->param_types[i] == VALUE_TYPE_F64) { + PUT_I64_TO_ADDR( + outs_area->lp, + GET_OPERAND(uint64, I64, + 2 * (cur_func->param_count - i - 1))); + outs_area->lp += 2; + } +#if WASM_ENABLE_GC != 0 + else if (wasm_is_type_reftype(cur_func->param_types[i])) { + PUT_REF_TO_ADDR( + outs_area->lp, + GET_OPERAND(void *, REF, + 2 * (cur_func->param_count - i - 1))); + CLEAR_FRAME_REF( + *(uint16 *)(frame_ip + + (2 * (cur_func->param_count - i - 1)))); + outs_area->lp += REF_CELL_NUM; + } +#endif + else { + *outs_area->lp = GET_OPERAND( + uint32, I32, (2 * (cur_func->param_count - i - 1))); + outs_area->lp++; + } + } + frame_ip += cur_func->param_count * sizeof(int16); + if (cur_func->ret_cell_num != 0) { + /* Get the first return value's offset. Since loader emit + * all return values' offset so we must skip remain return + * values' offsets. + */ + WASMFuncType *func_type; + if (cur_func->is_import_func) + func_type = cur_func->u.func_import->func_type; + else + func_type = cur_func->u.func->func_type; + frame->ret_offset = GET_OFFSET(); + frame_ip += 2 * (func_type->result_count - 1); + } + SYNC_ALL_TO_FRAME(); + prev_frame = frame; +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + is_return_call = false; +#endif + } + + call_func_from_entry: + { + if (cur_func->is_import_func) { +#if WASM_ENABLE_MULTI_MODULE != 0 + if (cur_func->import_func_inst) { + wasm_interp_call_func_import(module, exec_env, cur_func, + prev_frame); + } + else +#endif + { + wasm_interp_call_func_native(module, exec_env, cur_func, + prev_frame); + } + +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + if (is_return_call) { + /* the frame was freed before tail calling and + the prev_frame was set as exec_env's cur_frame, + so here we recover context from prev_frame */ + RECOVER_CONTEXT(prev_frame); + } + else +#endif + { + prev_frame = frame->prev_frame; + cur_func = frame->function; + UPDATE_ALL_FROM_FRAME(); + } + + /* update memory size, no need to update memory ptr as + it isn't changed in wasm_enlarge_memory */ +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY != 0 + if (memory) + linear_mem_size = GET_LINEAR_MEMORY_SIZE(memory); +#endif + if (wasm_copy_exception(module, NULL)) + goto got_exception; + } + else { + WASMFunction *cur_wasm_func = cur_func->u.func; + uint32 cell_num_of_local_stack; +#if WASM_ENABLE_REF_TYPES != 0 && WASM_ENABLE_GC == 0 + uint32 i, local_cell_idx; +#endif + + cell_num_of_local_stack = cur_func->param_cell_num + + cur_func->local_cell_num + + cur_wasm_func->max_stack_cell_num; + all_cell_num = cur_func->const_cell_num + cell_num_of_local_stack; +#if WASM_ENABLE_GC != 0 + /* area of frame_ref */ + all_cell_num += (cell_num_of_local_stack + 3) / 4; + /* cells occupied by locals, POP_REF should not clear frame_ref for + * these cells */ + local_cell_num = + cur_func->param_cell_num + cur_func->local_cell_num; +#endif + /* param_cell_num, local_cell_num, const_cell_num and + max_stack_cell_num are all no larger than UINT16_MAX (checked + in loader), all_cell_num must be smaller than 1MB */ + bh_assert(all_cell_num < 1 * BH_MB); + + frame_size = wasm_interp_interp_frame_size(all_cell_num); + if (!(frame = ALLOC_FRAME(exec_env, frame_size, prev_frame))) { + frame = prev_frame; + goto got_exception; + } + + /* Initialize the interpreter context. */ + frame->function = cur_func; + frame_ip = wasm_get_func_code(cur_func); + frame_ip_end = wasm_get_func_code_end(cur_func); + + frame_lp = frame->lp = + frame->operand + cur_wasm_func->const_cell_num; + + /* Initialize the consts */ + if (cur_wasm_func->const_cell_num > 0) { + word_copy(frame->operand, (uint32 *)cur_wasm_func->consts, + cur_wasm_func->const_cell_num); + } + + /* Initialize the local variables */ + memset(frame_lp + cur_func->param_cell_num, 0, + (uint32)(cur_func->local_cell_num * 4)); + +#if WASM_ENABLE_REF_TYPES != 0 && WASM_ENABLE_GC == 0 + /* externref/funcref should be NULL_REF rather than 0 */ + local_cell_idx = cur_func->param_cell_num; + for (i = 0; i < cur_wasm_func->local_count; i++) { + if (cur_wasm_func->local_types[i] == VALUE_TYPE_EXTERNREF + || cur_wasm_func->local_types[i] == VALUE_TYPE_FUNCREF) { + *(frame_lp + local_cell_idx) = NULL_REF; + } + local_cell_idx += + wasm_value_type_cell_num(cur_wasm_func->local_types[i]); + } +#endif + +#if WASM_ENABLE_GC != 0 + /* frame->ip is used during GC root set enumeration, so we must + * initialized this field here */ + frame->ip = frame_ip; + frame_ref = frame->frame_ref = + (uint8 *)(frame->lp + (uint32)cell_num_of_local_stack); + init_frame_refs(frame_ref, (uint32)cell_num_of_local_stack, + cur_func); +#endif + + wasm_exec_env_set_cur_frame(exec_env, (WASMRuntimeFrame *)frame); + } +#if WASM_ENABLE_THREAD_MGR != 0 + CHECK_SUSPEND_FLAGS(); +#endif + HANDLE_OP_END(); + } + + return_func: + { + FREE_FRAME(exec_env, frame); + wasm_exec_env_set_cur_frame(exec_env, (WASMRuntimeFrame *)prev_frame); + + if (!prev_frame->ip) + /* Called from native. */ + return; + + RECOVER_CONTEXT(prev_frame); +#if WASM_ENABLE_GC != 0 + local_cell_num = cur_func->param_cell_num + cur_func->local_cell_num; +#endif + HANDLE_OP_END(); + } + + (void)frame_ip_end; + +#if WASM_ENABLE_SHARED_MEMORY != 0 + unaligned_atomic: + wasm_set_exception(module, "unaligned atomic"); + goto got_exception; +#endif + +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY_OPT != 0 + out_of_bounds: + wasm_set_exception(module, "out of bounds memory access"); +#endif + + got_exception: + SYNC_ALL_TO_FRAME(); + return; + +#if WASM_ENABLE_LABELS_AS_VALUES == 0 + } +#else + FETCH_OPCODE_AND_DISPATCH(); +#endif +} + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +void ** +wasm_interp_get_handle_table(void) +{ + WASMModuleInstance module; + memset(&module, 0, sizeof(WASMModuleInstance)); + wasm_interp_call_func_bytecode(&module, NULL, NULL, NULL); + return global_handle_table; +} +#endif + +#if WASM_ENABLE_GC != 0 +bool +wasm_interp_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap) +{ + WASMInterpFrame *frame; + WASMObjectRef gc_obj; + WASMFunctionInstance *cur_func; + uint8 *frame_ref; + uint32 local_cell_num, i; + + frame = wasm_exec_env_get_cur_frame(exec_env); + for (; frame; frame = frame->prev_frame) { + frame_ref = frame->frame_ref; + cur_func = frame->function; + + if (!cur_func) + continue; + + local_cell_num = cur_func->param_cell_num; + if (frame->ip) + local_cell_num += + cur_func->local_cell_num + cur_func->u.func->max_stack_cell_num; + + for (i = 0; i < local_cell_num; i++) { + if (frame_ref[i]) { + gc_obj = GET_REF_FROM_ADDR(frame->lp + i); + if (wasm_obj_is_created_from_heap(gc_obj)) { + if (mem_allocator_add_root((mem_allocator_t)heap, gc_obj)) { + return false; + } + } +#if UINTPTR_MAX == UINT64_MAX + bh_assert(frame_ref[i + 1]); + i++; +#endif + } + } + } + return true; +} +#endif + +void +wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, + WASMFunctionInstance *function, uint32 argc, + uint32 argv[]) +{ + WASMRuntimeFrame *prev_frame = wasm_exec_env_get_cur_frame(exec_env); + WASMInterpFrame *frame, *outs_area; + + /* Allocate sufficient cells for all kinds of return values. */ + unsigned all_cell_num = + function->ret_cell_num > 2 ? function->ret_cell_num : 2, + i; + /* This frame won't be used by JITed code, so only allocate interp + frame here. */ + unsigned frame_size; + +#if WASM_ENABLE_GC != 0 + all_cell_num += (all_cell_num + 3) / 4; +#endif + + frame_size = wasm_interp_interp_frame_size(all_cell_num); + + if (argc < function->param_cell_num) { + char buf[128]; + snprintf(buf, sizeof(buf), + "invalid argument count %" PRIu32 + ", must be no smaller than %" PRIu32, + argc, (uint32)function->param_cell_num); + wasm_set_exception(module_inst, buf); + return; + } + argc = function->param_cell_num; + +#if defined(OS_ENABLE_HW_BOUND_CHECK) && WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + /* + * wasm_runtime_detect_native_stack_overflow is done by + * call_wasm_with_hw_bound_check. + */ +#else + if (!wasm_runtime_detect_native_stack_overflow(exec_env)) { + return; + } +#endif + + if (!(frame = + ALLOC_FRAME(exec_env, frame_size, (WASMInterpFrame *)prev_frame))) + return; + + outs_area = wasm_exec_env_wasm_stack_top(exec_env); + frame->function = NULL; + frame->ip = NULL; + /* There is no local variable. */ + frame->lp = frame->operand + 0; +#if WASM_ENABLE_GC != 0 + frame->frame_ref = + (uint8 *)(frame->lp + + (function->ret_cell_num > 2 ? function->ret_cell_num : 2)); +#endif + frame->ret_offset = 0; + + if ((uint8 *)(outs_area->operand + function->const_cell_num + argc) + > exec_env->wasm_stack.top_boundary) { + wasm_set_exception((WASMModuleInstance *)exec_env->module_inst, + "wasm operand stack overflow"); + return; + } + + if (argc > 0) + word_copy(outs_area->operand + function->const_cell_num, argv, argc); + + wasm_exec_env_set_cur_frame(exec_env, frame); + +#if defined(os_writegsbase) + { + WASMMemoryInstance *memory_inst = wasm_get_default_memory(module_inst); + if (memory_inst) + /* write base addr of linear memory to GS segment register */ + os_writegsbase(memory_inst->memory_data); + } +#endif + + if (function->is_import_func) { +#if WASM_ENABLE_MULTI_MODULE != 0 + if (function->import_module_inst) { + LOG_DEBUG("it is a function of a sub module"); + wasm_interp_call_func_import(module_inst, exec_env, function, + frame); + } + else +#endif + { + LOG_DEBUG("it is an native function"); + wasm_interp_call_func_native(module_inst, exec_env, function, + frame); + } + } + else { + wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame); + } + + /* Output the return value to the caller */ + if (!wasm_copy_exception(module_inst, NULL)) { + for (i = 0; i < function->ret_cell_num; i++) + argv[i] = *(frame->lp + i); + } + else { +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + if (wasm_interp_create_call_stack(exec_env)) { + wasm_interp_dump_call_stack(exec_env, true, NULL, 0); + } +#endif + } + + wasm_exec_env_set_cur_frame(exec_env, prev_frame); + FREE_FRAME(exec_env, frame); +#if WASM_ENABLE_OPCODE_COUNTER != 0 + wasm_interp_dump_op_count(); +#endif +} diff --git a/wamr/src/wasm_loader.c b/wamr/src/wasm_loader.c new file mode 100644 index 0000000..e119a65 --- /dev/null +++ b/wamr/src/wasm_loader.c @@ -0,0 +1,16983 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_loader.h" // wasm_loader.h +#include "bh_platform.h" // bh_platform.h +#include "wasm.h" // wasm.h +#include "wasm_opcode.h" // wasm_opcode.h +#include "wasm_runtime.h" // wasm_runtime.h +#include "wasm_loader_common.h" // wasm_loader_common.h +#include "wasm_native.h" // ../common/wasm_native.h +#include "wasm_memory.h" // ../common/wasm_memory.h +#if WASM_ENABLE_GC != 0 + + +#endif +#if WASM_ENABLE_DEBUG_INTERP != 0 +#include "debug_engine.h" // ../libraries/debug-engine/debug_engine.h +#endif +#if WASM_ENABLE_FAST_JIT != 0 + + +#endif + + +#ifndef TRACE_WASM_LOADER +#define TRACE_WASM_LOADER 0 +#endif + +/* Read a value of given type from the address pointed to by the given + pointer and increase the pointer to the position just after the + value being read. */ +#define TEMPLATE_READ_VALUE(Type, p) \ + (p += sizeof(Type), *(Type *)(p - sizeof(Type))) + +#if WASM_ENABLE_MEMORY64 != 0 +static bool +has_module_memory64(WASMModule *module) +{ + /* TODO: multi-memories for now assuming the memory idx type is consistent + * across multi-memories */ + if (module->import_memory_count > 0) + return !!(module->import_memories[0].u.memory.mem_type.flags + & MEMORY64_FLAG); + else if (module->memory_count > 0) + return !!(module->memories[0].flags & MEMORY64_FLAG); + + return false; +} + +static bool +is_table_64bit(WASMModule *module, uint32 table_idx) +{ + if (table_idx < module->import_table_count) + return !!(module->import_tables[table_idx].u.table.table_type.flags + & TABLE64_FLAG); + else + return !!(module->tables[table_idx - module->import_table_count] + .table_type.flags + & TABLE64_FLAG); + + return false; +} +#endif + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + wasm_loader_set_error_buf(error_buf, error_buf_size, string, false); +} + +#if WASM_ENABLE_MEMORY64 != 0 +static void +set_error_buf_mem_offset_out_of_range(char *error_buf, uint32 error_buf_size) +{ + if (error_buf != NULL) { + snprintf(error_buf, error_buf_size, "offset out of range"); + } +} +#endif + +static void +set_error_buf_v(char *error_buf, uint32 error_buf_size, const char *format, ...) +{ + va_list args; + char buf[128]; + + if (error_buf != NULL) { + va_start(args, format); + vsnprintf(buf, sizeof(buf), format, args); + va_end(args); + snprintf(error_buf, error_buf_size, "WASM module load failed: %s", buf); + } +} + +static bool +check_buf(const uint8 *buf, const uint8 *buf_end, uint32 length, + char *error_buf, uint32 error_buf_size) +{ + if ((uintptr_t)buf + length < (uintptr_t)buf + || (uintptr_t)buf + length > (uintptr_t)buf_end) { + set_error_buf(error_buf, error_buf_size, + "unexpected end of section or function"); + return false; + } + return true; +} + +static bool +check_buf1(const uint8 *buf, const uint8 *buf_end, uint32 length, + char *error_buf, uint32 error_buf_size) +{ + if ((uintptr_t)buf + length < (uintptr_t)buf + || (uintptr_t)buf + length > (uintptr_t)buf_end) { + set_error_buf(error_buf, error_buf_size, "unexpected end"); + return false; + } + return true; +} + +#define CHECK_BUF(buf, buf_end, length) \ + do { \ + if (!check_buf(buf, buf_end, length, error_buf, error_buf_size)) { \ + goto fail; \ + } \ + } while (0) + +#define CHECK_BUF1(buf, buf_end, length) \ + do { \ + if (!check_buf1(buf, buf_end, length, error_buf, error_buf_size)) { \ + goto fail; \ + } \ + } while (0) + +#define skip_leb(p) while (*p++ & 0x80) +#define skip_leb_int64(p, p_end) skip_leb(p) +#define skip_leb_uint32(p, p_end) skip_leb(p) +#define skip_leb_int32(p, p_end) skip_leb(p) +#define skip_leb_mem_offset(p, p_end) skip_leb(p) +#define skip_leb_memidx(p, p_end) skip_leb(p) +#if WASM_ENABLE_MULTI_MEMORY == 0 +#define skip_leb_align(p, p_end) skip_leb(p) +#else +/* Skip the following memidx if applicable */ +#define skip_leb_align(p, p_end) \ + do { \ + if (*p++ & OPT_MEMIDX_FLAG) \ + skip_leb_uint32(p, p_end); \ + } while (0) +#endif + +#define read_uint8(p) TEMPLATE_READ_VALUE(uint8, p) +#define read_uint32(p) TEMPLATE_READ_VALUE(uint32, p) + +#define read_leb_int64(p, p_end, res) \ + do { \ + uint64 res64; \ + if (!read_leb((uint8 **)&p, p_end, 64, true, &res64, error_buf, \ + error_buf_size)) \ + goto fail; \ + res = (int64)res64; \ + } while (0) + +#if WASM_ENABLE_MEMORY64 != 0 +#define read_leb_mem_offset(p, p_end, res) \ + do { \ + uint64 res64; \ + if (!read_leb((uint8 **)&p, p_end, is_memory64 ? 64 : 32, false, \ + &res64, error_buf, error_buf_size)) { \ + set_error_buf_mem_offset_out_of_range(error_buf, error_buf_size); \ + goto fail; \ + } \ + res = (mem_offset_t)res64; \ + } while (0) +#else +#define read_leb_mem_offset(p, p_end, res) read_leb_uint32(p, p_end, res) +#endif + +#define read_leb_uint32(p, p_end, res) \ + do { \ + uint64 res64; \ + if (!read_leb((uint8 **)&p, p_end, 32, false, &res64, error_buf, \ + error_buf_size)) \ + goto fail; \ + res = (uint32)res64; \ + } while (0) + +#define read_leb_int32(p, p_end, res) \ + do { \ + uint64 res64; \ + if (!read_leb((uint8 **)&p, p_end, 32, true, &res64, error_buf, \ + error_buf_size)) \ + goto fail; \ + res = (int32)res64; \ + } while (0) + +#if WASM_ENABLE_MULTI_MEMORY != 0 +#define check_memidx(module, memidx) \ + do { \ + if (memidx >= module->import_memory_count + module->memory_count) { \ + set_error_buf_v(error_buf, error_buf_size, "unknown memory %d", \ + memidx); \ + goto fail; \ + } \ + } while (0) +/* Bit 6(0x40) indicating the optional memidx, and reset bit 6 for + * alignment check */ +#define read_leb_memarg(p, p_end, res) \ + do { \ + read_leb_uint32(p, p_end, res); \ + if (res & OPT_MEMIDX_FLAG) { \ + res &= ~OPT_MEMIDX_FLAG; \ + read_leb_uint32(p, p_end, memidx); /* memidx */ \ + check_memidx(module, memidx); \ + } \ + } while (0) +#else +/* reserved byte 0x00 */ +#define check_memidx(module, memidx) \ + do { \ + (void)module; \ + if (memidx != 0) { \ + set_error_buf(error_buf, error_buf_size, "zero byte expected"); \ + goto fail; \ + } \ + } while (0) +#define read_leb_memarg(p, p_end, res) read_leb_uint32(p, p_end, res) +#endif + +static char * +type2str(uint8 type) +{ + char *type_str[] = { "v128", "f64", "f32", "i64", "i32" }; +#if WASM_ENABLE_GC != 0 + char *type_str_ref[] = { "stringview_iter", + "stringview_wtf16", + "(ref null ht)", + "(ref ht)", + "", /* reserved */ + "stringview_wtf8", + "stringref", + "", /* reserved */ + "", /* reserved */ + "arrayref", + "structref", + "i31ref", + "eqref", + "anyref", + "externref", + "funcref", + "nullref", + "nullexternref", + "nullfuncref" }; +#endif + + if (type >= VALUE_TYPE_V128 && type <= VALUE_TYPE_I32) + return type_str[type - VALUE_TYPE_V128]; +#if WASM_ENABLE_GC != 0 + else if (wasm_is_type_reftype(type)) + return type_str_ref[type - REF_TYPE_STRINGVIEWITER]; +#endif + else if (type == VALUE_TYPE_FUNCREF) + return "funcref"; + else if (type == VALUE_TYPE_EXTERNREF) + return "externref"; + else + return "unknown type"; +} + +static bool +is_32bit_type(uint8 type) +{ + if (type == VALUE_TYPE_I32 + || type == VALUE_TYPE_F32 + /* the operand stack is in polymorphic state */ + || type == VALUE_TYPE_ANY +#if WASM_ENABLE_GC != 0 + || (sizeof(uintptr_t) == 4 && wasm_is_type_reftype(type)) +#elif WASM_ENABLE_REF_TYPES != 0 + /* For reference types, we use uint32 index to represent + the funcref and externref */ + || type == VALUE_TYPE_FUNCREF || type == VALUE_TYPE_EXTERNREF +#endif + ) + return true; + return false; +} + +static bool +is_64bit_type(uint8 type) +{ + if (type == VALUE_TYPE_I64 || type == VALUE_TYPE_F64 +#if WASM_ENABLE_GC != 0 + || (sizeof(uintptr_t) == 8 && wasm_is_type_reftype(type)) +#endif + ) + return true; + return false; +} + +#if WASM_ENABLE_GC != 0 +static bool +is_packed_type(uint8 type) +{ + return (type == PACKED_TYPE_I8 || type == PACKED_TYPE_I16) ? true : false; +} +#endif + +static bool +is_byte_a_type(uint8 type) +{ + return (is_valid_value_type_for_interpreter(type) + || (type == VALUE_TYPE_VOID)) + ? true + : false; +} + +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) \ + || (WASM_ENABLE_FAST_INTERP != 0) +static V128 +read_i8x16(uint8 *p_buf, char *error_buf, uint32 error_buf_size) +{ + V128 result; + uint8 i; + + for (i = 0; i != 16; ++i) { + result.i8x16[i] = read_uint8(p_buf); + } + + return result; +} +#endif /* end of (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) || \ + (WASM_ENABLE_FAST_INTERP != 0) */ +#endif /* end of WASM_ENABLE_SIMD */ + +static void * +loader_malloc(uint64 size, char *error_buf, uint32 error_buf_size) +{ + void *mem; + + if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + return NULL; + } + + memset(mem, 0, (uint32)size); + return mem; +} + +static void * +memory_realloc(void *mem_old, uint32 size_old, uint32 size_new, char *error_buf, + uint32 error_buf_size) +{ + uint8 *mem_new; + bh_assert(size_new > size_old); + + if ((mem_new = wasm_runtime_realloc(mem_old, size_new))) { + memset(mem_new + size_old, 0, size_new - size_old); + return mem_new; + } + + if ((mem_new = loader_malloc(size_new, error_buf, error_buf_size))) { + bh_memcpy_s(mem_new, size_new, mem_old, size_old); + wasm_runtime_free(mem_old); + } + return mem_new; +} + +#define MEM_REALLOC(mem, size_old, size_new) \ + do { \ + void *mem_new = memory_realloc(mem, size_old, size_new, error_buf, \ + error_buf_size); \ + if (!mem_new) \ + goto fail; \ + mem = mem_new; \ + } while (0) + +static bool +check_type_index(const WASMModule *module, uint32 type_count, uint32 type_index, + char *error_buf, uint32 error_buf_size) +{ + if (type_index >= type_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown type %d", + type_index); + return false; + } + return true; +} + +#if WASM_ENABLE_GC != 0 +static bool +check_array_type(const WASMModule *module, uint32 type_index, char *error_buf, + uint32 error_buf_size) +{ + if (!check_type_index(module, module->type_count, type_index, error_buf, + error_buf_size)) { + return false; + } + if (module->types[type_index]->type_flag != WASM_TYPE_ARRAY) { + set_error_buf(error_buf, error_buf_size, "unknown array type"); + return false; + } + + return true; +} +#endif + +/* + * if no GC is enabled, an valid type is always a function type. + * but if GC is enabled, we need to check the type flag + */ +static bool +check_function_type(const WASMModule *module, uint32 type_index, + char *error_buf, uint32 error_buf_size) +{ + if (!check_type_index(module, module->type_count, type_index, error_buf, + error_buf_size)) { + return false; + } + +#if WASM_ENABLE_GC != 0 + if (module->types[type_index]->type_flag != WASM_TYPE_FUNC) { + set_error_buf(error_buf, error_buf_size, "unknown function type"); + return false; + } +#endif + + return true; +} + +static bool +check_function_index(const WASMModule *module, uint32 function_index, + char *error_buf, uint32 error_buf_size) +{ + if (function_index + >= module->import_function_count + module->function_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown function %u", + function_index); + return false; + } + return true; +} + +typedef struct InitValue { + uint8 type; + uint8 flag; +#if WASM_ENABLE_GC != 0 + uint8 gc_opcode; + WASMRefType ref_type; +#endif + WASMValue value; +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + InitializerExpression *expr; +#endif +} InitValue; + +typedef struct ConstExprContext { + uint32 sp; + uint32 size; + WASMModule *module; + InitValue *stack; + InitValue data[WASM_CONST_EXPR_STACK_SIZE]; +} ConstExprContext; + +static void +init_const_expr_stack(ConstExprContext *ctx, WASMModule *module) +{ + ctx->sp = 0; + ctx->module = module; + ctx->stack = ctx->data; + ctx->size = WASM_CONST_EXPR_STACK_SIZE; +} + +static bool +push_const_expr_stack(ConstExprContext *ctx, uint8 flag, uint8 type, +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type, uint8 gc_opcode, +#endif + WASMValue *value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + InitializerExpression *expr, +#endif + char *error_buf, uint32 error_buf_size) +{ + InitValue *cur_value; + + if (ctx->sp >= ctx->size) { + if (ctx->stack != ctx->data) { + MEM_REALLOC(ctx->stack, ctx->size * sizeof(InitValue), + (ctx->size + 4) * sizeof(InitValue)); + } + else { + if (!(ctx->stack = + loader_malloc((ctx->size + 4) * (uint64)sizeof(InitValue), + error_buf, error_buf_size))) { + goto fail; + } + bh_memcpy_s(ctx->stack, (ctx->size + 4) * (uint32)sizeof(InitValue), + ctx->data, ctx->size * (uint32)sizeof(InitValue)); + } + ctx->size += 4; + } + + cur_value = &ctx->stack[ctx->sp++]; + cur_value->type = type; + cur_value->flag = flag; + cur_value->value = *value; + +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + cur_value->expr = expr; +#endif + +#if WASM_ENABLE_GC != 0 + cur_value->gc_opcode = gc_opcode; + if (wasm_is_type_multi_byte_type(type)) { + bh_memcpy_s(&cur_value->ref_type, wasm_reftype_struct_size(ref_type), + ref_type, wasm_reftype_struct_size(ref_type)); + } +#endif + + return true; +fail: + return false; +} + +#if WASM_ENABLE_GC != 0 +static void +destroy_init_expr_data_recursive(WASMModule *module, void *data) +{ + WASMStructNewInitValues *struct_init_values = + (WASMStructNewInitValues *)data; + WASMArrayNewInitValues *array_init_values = (WASMArrayNewInitValues *)data; + WASMType *wasm_type; + uint32 i; + + if (!data) + return; + + wasm_type = module->types[struct_init_values->type_idx]; + + /* The data can only be type of `WASMStructNewInitValues *` + or `WASMArrayNewInitValues *` */ + bh_assert(wasm_type->type_flag == WASM_TYPE_STRUCT + || wasm_type->type_flag == WASM_TYPE_ARRAY); + + if (wasm_type->type_flag == WASM_TYPE_STRUCT) { + WASMStructType *struct_type = (WASMStructType *)wasm_type; + WASMRefType *ref_type; + uint8 field_type; + + uint16 ref_type_map_index = 0; + for (i = 0; i < struct_init_values->count; i++) { + field_type = struct_type->fields[i].field_type; + if (wasm_is_type_multi_byte_type(field_type)) + ref_type = + struct_type->ref_type_maps[ref_type_map_index++].ref_type; + else + ref_type = NULL; + if (wasm_reftype_is_subtype_of(field_type, ref_type, + REF_TYPE_STRUCTREF, NULL, + module->types, module->type_count) + || wasm_reftype_is_subtype_of( + field_type, ref_type, REF_TYPE_ARRAYREF, NULL, + module->types, module->type_count)) { + destroy_init_expr_data_recursive( + module, struct_init_values->fields[i].data); + } + } + } + else if (wasm_type->type_flag == WASM_TYPE_ARRAY) { + WASMArrayType *array_type = (WASMArrayType *)wasm_type; + WASMRefType *elem_ref_type = array_type->elem_ref_type; + uint8 elem_type = array_type->elem_type; + + for (i = 0; i < array_init_values->length; i++) { + if (wasm_reftype_is_subtype_of(elem_type, elem_ref_type, + REF_TYPE_STRUCTREF, NULL, + module->types, module->type_count) + || wasm_reftype_is_subtype_of( + elem_type, elem_ref_type, REF_TYPE_ARRAYREF, NULL, + module->types, module->type_count)) { + destroy_init_expr_data_recursive( + module, array_init_values->elem_data[i].data); + } + } + } + + wasm_runtime_free(data); +} +#endif + +static bool +pop_const_expr_stack(ConstExprContext *ctx, uint8 *p_flag, uint8 type, +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type, uint8 *p_gc_opcode, +#endif + WASMValue *p_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + InitializerExpression **p_expr, +#endif + char *error_buf, uint32 error_buf_size) +{ + InitValue *cur_value; + + if (ctx->sp == 0) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: const expr stack underflow"); + return false; + } + + cur_value = &ctx->stack[--ctx->sp]; + +#if WASM_ENABLE_GC == 0 + if (cur_value->type != type) { + set_error_buf(error_buf, error_buf_size, "type mismatch"); + return false; + } +#else + if (!wasm_reftype_is_subtype_of(cur_value->type, &cur_value->ref_type, type, + ref_type, ctx->module->types, + ctx->module->type_count)) { + set_error_buf_v(error_buf, error_buf_size, "%s%s%s", + "type mismatch: expect ", type2str(type), + " but got other"); + goto fail; + } +#endif + + if (p_flag) + *p_flag = cur_value->flag; + if (p_value) + *p_value = cur_value->value; +#if WASM_ENABLE_GC != 0 + if (p_gc_opcode) + *p_gc_opcode = cur_value->gc_opcode; +#endif +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + if (p_expr) + *p_expr = cur_value->expr; +#endif + return true; + +#if WASM_ENABLE_GC != 0 +fail: + if ((cur_value->flag == WASM_OP_GC_PREFIX) + && (cur_value->gc_opcode == WASM_OP_STRUCT_NEW + || cur_value->gc_opcode == WASM_OP_ARRAY_NEW + || cur_value->gc_opcode == WASM_OP_ARRAY_NEW_FIXED)) { + destroy_init_expr_data_recursive(ctx->module, cur_value->value.data); + } + return false; +#endif +} + +static void +destroy_const_expr_stack(ConstExprContext *ctx, bool free_exprs) +{ +#if WASM_ENABLE_GC != 0 + uint32 i; + + for (i = 0; i < ctx->sp; i++) { + if ((ctx->stack[i].flag == WASM_OP_GC_PREFIX) + && (ctx->stack[i].gc_opcode == WASM_OP_STRUCT_NEW + || ctx->stack[i].gc_opcode == WASM_OP_ARRAY_NEW + || ctx->stack[i].gc_opcode == WASM_OP_ARRAY_NEW_FIXED)) { + destroy_init_expr_data_recursive(ctx->module, + ctx->stack[i].value.data); + } + } +#endif +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + if (free_exprs) { + for (uint32 j = 0; j < ctx->sp; j++) { + if (is_expr_binary_op(ctx->stack[j].expr->init_expr_type)) { + destroy_init_expr_recursive(ctx->stack[j].expr); + ctx->stack[j].expr = NULL; + } + } + } +#endif + + if (ctx->stack != ctx->data) { + wasm_runtime_free(ctx->stack); + } +} + +#if WASM_ENABLE_GC != 0 || WASM_ENABLE_EXTENDED_CONST_EXPR != 0 +static void +destroy_init_expr(WASMModule *module, InitializerExpression *expr) +{ +#if WASM_ENABLE_GC != 0 + if (expr->init_expr_type == INIT_EXPR_TYPE_STRUCT_NEW + || expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW + || expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) { + destroy_init_expr_data_recursive(module, expr->u.unary.v.data); + } +#endif + +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + // free left expr and right exprs for binary oprand + if (!is_expr_binary_op(expr->init_expr_type)) { + return; + } + if (expr->u.binary.l_expr) { + destroy_init_expr_recursive(expr->u.binary.l_expr); + } + if (expr->u.binary.r_expr) { + destroy_init_expr_recursive(expr->u.binary.r_expr); + } + expr->u.binary.l_expr = expr->u.binary.r_expr = NULL; +#endif +} +#endif + +/* for init expr + * (data (i32.add (i32.const 0) (i32.sub (i32.const 1) (i32.const 2)))), + * the binary format is + * 0x11: 41 00 ; i32.const 0 + * 0x13: 41 01 ; i32.const 1 + * 0x15: 41 02 ; i32.const 2 + * 0x17: 6b ; i32.sub + * 0x18: 6a ; i32.add + * for traversal: read opcodes and push them onto the stack. When encountering + * a binary opcode, pop two values from the stack which become the left and + * right child nodes of this binary operation node. + */ +static bool +load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, + InitializerExpression *init_expr, uint8 type, void *ref_type, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint8 flag, *p_float; + uint32 i; + ConstExprContext const_expr_ctx = { 0 }; + WASMValue cur_value; +#if WASM_ENABLE_GC != 0 + uint32 opcode1, type_idx; + uint8 opcode; + WASMRefType cur_ref_type = { 0 }; +#endif +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + InitializerExpression *cur_expr = NULL; +#endif + + init_const_expr_stack(&const_expr_ctx, module); + + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + + while (flag != WASM_OP_END) { + switch (flag) { + /* i32.const */ + case INIT_EXPR_TYPE_I32_CONST: + read_leb_int32(p, p_end, cur_value.i32); + + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_I32, +#if WASM_ENABLE_GC != 0 + NULL, 0, +#endif + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + break; + /* i64.const */ + case INIT_EXPR_TYPE_I64_CONST: + read_leb_int64(p, p_end, cur_value.i64); + + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_I64, +#if WASM_ENABLE_GC != 0 + NULL, 0, +#endif + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + break; + /* f32.const */ + case INIT_EXPR_TYPE_F32_CONST: + CHECK_BUF(p, p_end, 4); + p_float = (uint8 *)&cur_value.f32; + for (i = 0; i < sizeof(float32); i++) + *p_float++ = *p++; + + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_F32, +#if WASM_ENABLE_GC != 0 + NULL, 0, +#endif + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + break; + /* f64.const */ + case INIT_EXPR_TYPE_F64_CONST: + CHECK_BUF(p, p_end, 8); + p_float = (uint8 *)&cur_value.f64; + for (i = 0; i < sizeof(float64); i++) + *p_float++ = *p++; + + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_F64, +#if WASM_ENABLE_GC != 0 + NULL, 0, +#endif + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + break; +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) \ + || (WASM_ENABLE_FAST_INTERP != 0) + /* v128.const */ + case INIT_EXPR_TYPE_V128_CONST: + { + uint64 high, low; + + CHECK_BUF(p, p_end, 1); + (void)read_uint8(p); + + CHECK_BUF(p, p_end, 16); + wasm_runtime_read_v128(p, &high, &low); + p += 16; + + cur_value.v128.i64x2[0] = high; + cur_value.v128.i64x2[1] = low; + + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_V128, +#if WASM_ENABLE_GC != 0 + NULL, 0, +#endif + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) + goto fail; +#if WASM_ENABLE_WAMR_COMPILER != 0 + /* If any init_expr is v128.const, mark SIMD used */ + module->is_simd_used = true; +#endif + break; + } +#endif /* end of (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) || \ + (WASM_ENABLE_FAST_INTERP != 0) */ +#endif /* end of WASM_ENABLE_SIMD */ +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + case INIT_EXPR_TYPE_I32_ADD: + case INIT_EXPR_TYPE_I32_SUB: + case INIT_EXPR_TYPE_I32_MUL: + case INIT_EXPR_TYPE_I64_ADD: + case INIT_EXPR_TYPE_I64_SUB: + case INIT_EXPR_TYPE_I64_MUL: + { + + InitializerExpression *l_expr, *r_expr; + WASMValue l_value, r_value; + uint8 l_flag, r_flag; + uint8 value_type; + + if (flag == INIT_EXPR_TYPE_I32_ADD + || flag == INIT_EXPR_TYPE_I32_SUB + || flag == INIT_EXPR_TYPE_I32_MUL) { + value_type = VALUE_TYPE_I32; + } + else { + value_type = VALUE_TYPE_I64; + } + + /* If right flag indicates a binary operation, right expr will + * be popped from stack. Otherwise, allocate a new expr for + * right expr. Same for left expr. + */ + if (!(pop_const_expr_stack(&const_expr_ctx, &r_flag, value_type, +#if WASM_ENABLE_GC != 0 + NULL, NULL, +#endif + &r_value, &r_expr, error_buf, + error_buf_size))) { + goto fail; + } + if (!is_expr_binary_op(r_flag)) { + if (!(r_expr = loader_malloc(sizeof(InitializerExpression), + error_buf, error_buf_size))) { + goto fail; + } + r_expr->init_expr_type = r_flag; + r_expr->u.unary.v = r_value; + } + + if (!(pop_const_expr_stack(&const_expr_ctx, &l_flag, value_type, +#if WASM_ENABLE_GC != 0 + NULL, NULL, +#endif + &l_value, &l_expr, error_buf, + error_buf_size))) { + destroy_init_expr_recursive(r_expr); + goto fail; + } + if (!is_expr_binary_op(l_flag)) { + if (!(l_expr = loader_malloc(sizeof(InitializerExpression), + error_buf, error_buf_size))) { + destroy_init_expr_recursive(r_expr); + goto fail; + } + l_expr->init_expr_type = l_flag; + l_expr->u.unary.v = l_value; + } + + if (!(cur_expr = loader_malloc(sizeof(InitializerExpression), + error_buf, error_buf_size))) { + destroy_init_expr_recursive(l_expr); + destroy_init_expr_recursive(r_expr); + goto fail; + } + cur_expr->init_expr_type = flag; + cur_expr->u.binary.l_expr = l_expr; + cur_expr->u.binary.r_expr = r_expr; + + if (!push_const_expr_stack(&const_expr_ctx, flag, value_type, +#if WASM_ENABLE_GC != 0 + NULL, 0, +#endif + &cur_value, cur_expr, error_buf, + error_buf_size)) { + destroy_init_expr_recursive(cur_expr); + goto fail; + } + + break; + } +#endif /* end of WASM_ENABLE_EXTENDED_CONST_EXPR */ +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + /* ref.func */ + case INIT_EXPR_TYPE_FUNCREF_CONST: + { + uint32 func_idx; + read_leb_uint32(p, p_end, func_idx); + cur_value.ref_index = func_idx; + if (!check_function_index(module, func_idx, error_buf, + error_buf_size)) { + goto fail; + } + +#if WASM_ENABLE_GC == 0 + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_FUNCREF, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) + goto fail; +#else + if (func_idx < module->import_function_count) { + type_idx = + module->import_functions[func_idx].u.function.type_idx; + } + else { + type_idx = module + ->functions[func_idx + - module->import_function_count] + ->type_idx; + } + wasm_set_refheaptype_typeidx(&cur_ref_type.ref_ht_typeidx, + false, type_idx); + if (!push_const_expr_stack(&const_expr_ctx, flag, + cur_ref_type.ref_type, &cur_ref_type, + 0, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) + goto fail; +#endif +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } + + /* ref.null */ + case INIT_EXPR_TYPE_REFNULL_CONST: + { +#if WASM_ENABLE_GC == 0 + uint8 type1; + CHECK_BUF(p, p_end, 1); + type1 = read_uint8(p); + cur_value.ref_index = NULL_REF; + if (!push_const_expr_stack(&const_expr_ctx, flag, type1, + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) + goto fail; +#else + /* + * According to the current GC SPEC rules, the heap_type must be + * validated when ref.null is used. It can be an absheaptype, + * or the type C.types[type_idx] must be defined in the context. + */ + int32 heap_type; + read_leb_int32(p, p_end, heap_type); + cur_value.gc_obj = NULL_REF; + + /* + * The current check of heap_type can deterministically infer + * the result of the previous condition + * `(!is_byte_a_type(type1) || + * wasm_is_type_multi_byte_type(type1))`. Therefore, the + * original condition is redundant and has been removed. + * + * This logic is consistent with the implementation of the + * `WASM_OP_REF_NULL` case in the `wasm_loader_prepare_bytecode` + * function. + */ + + if (heap_type >= 0) { + if (!check_type_index(module, module->type_count, heap_type, + error_buf, error_buf_size)) { + goto fail; + } + wasm_set_refheaptype_typeidx(&cur_ref_type.ref_ht_typeidx, + true, heap_type); + if (!push_const_expr_stack(&const_expr_ctx, flag, + cur_ref_type.ref_type, + &cur_ref_type, 0, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + } + else { + if (!wasm_is_valid_heap_type(heap_type)) { + set_error_buf_v(error_buf, error_buf_size, + "unknown type %d", heap_type); + goto fail; + } + cur_ref_type.ref_ht_common.ref_type = + (uint8)((int32)0x80 + heap_type); + if (!push_const_expr_stack(&const_expr_ctx, flag, + cur_ref_type.ref_type, NULL, 0, + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + } +#endif +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + + /* get_global */ + case INIT_EXPR_TYPE_GET_GLOBAL: + { + uint32 global_idx; + uint8 global_type; + + read_leb_uint32(p, p_end, cur_value.global_index); + global_idx = cur_value.global_index; + + /* + * Currently, constant expressions occurring as initializers + * of globals are further constrained in that contained + * global.get instructions are + * only allowed to refer to imported globals. + * + * https://webassembly.github.io/spec/core/valid/instructions.html#constant-expressions + */ + if (global_idx >= module->import_global_count + /* make spec test happy */ +#if WASM_ENABLE_GC != 0 + + module->global_count +#endif + ) { + set_error_buf_v(error_buf, error_buf_size, + "unknown global %u", global_idx); + goto fail; + } + if ( + /* make spec test happy */ +#if WASM_ENABLE_GC != 0 + global_idx < module->import_global_count && +#endif + module->import_globals[global_idx] + .u.global.type.is_mutable) { + set_error_buf_v(error_buf, error_buf_size, + "constant expression required"); + goto fail; + } + + if (global_idx < module->import_global_count) { + global_type = module->import_globals[global_idx] + .u.global.type.val_type; +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(global_type)) { + WASMRefType *global_ref_type = + module->import_globals[global_idx] + .u.global.ref_type; + bh_memcpy_s(&cur_ref_type, + wasm_reftype_struct_size(global_ref_type), + global_ref_type, + wasm_reftype_struct_size(global_ref_type)); + } +#endif + } + else { + global_type = + module + ->globals[global_idx - module->import_global_count] + .type.val_type; +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(global_type)) { + WASMRefType *global_ref_type = + module + ->globals[global_idx + - module->import_global_count] + .ref_type; + bh_memcpy_s(&cur_ref_type, + wasm_reftype_struct_size(global_ref_type), + global_ref_type, + wasm_reftype_struct_size(global_ref_type)); + } +#endif + } + + if (!push_const_expr_stack(&const_expr_ctx, flag, global_type, +#if WASM_ENABLE_GC != 0 + &cur_ref_type, 0, +#endif + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + + break; + } + +#if WASM_ENABLE_GC != 0 + /* struct.new and array.new */ + case WASM_OP_GC_PREFIX: + { + read_leb_uint32(p, p_end, opcode1); + + switch (opcode1) { + case WASM_OP_STRUCT_NEW: + { + WASMStructType *struct_type; + WASMStructNewInitValues *struct_init_values = NULL; + uint32 field_count; + read_leb_uint32(p, p_end, type_idx); + + if (!check_type_index(module, module->type_count, + type_idx, error_buf, + error_buf_size)) { + goto fail; + } + + struct_type = (WASMStructType *)module->types[type_idx]; + if (struct_type->base_type.type_flag + != WASM_TYPE_STRUCT) { + set_error_buf(error_buf, error_buf_size, + "unknown struct type"); + goto fail; + } + field_count = struct_type->field_count; + + if (!(struct_init_values = loader_malloc( + offsetof(WASMStructNewInitValues, fields) + + (uint64)field_count * sizeof(WASMValue), + error_buf, error_buf_size))) { + goto fail; + } + struct_init_values->type_idx = type_idx; + struct_init_values->count = field_count; + + for (i = field_count; i > 0; i--) { + WASMRefType *field_ref_type = NULL; + uint32 field_idx = i - 1; + uint8 field_type = + struct_type->fields[field_idx].field_type; + if (wasm_is_type_multi_byte_type(field_type)) { + field_ref_type = wasm_reftype_map_find( + struct_type->ref_type_maps, + struct_type->ref_type_map_count, field_idx); + } + + if (is_packed_type(field_type)) { + field_type = VALUE_TYPE_I32; + } + + if (!pop_const_expr_stack( + &const_expr_ctx, NULL, field_type, + field_ref_type, NULL, + &struct_init_values->fields[field_idx], +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) { + destroy_init_expr_data_recursive( + module, struct_init_values); + goto fail; + } + } + + cur_value.data = struct_init_values; + wasm_set_refheaptype_typeidx( + &cur_ref_type.ref_ht_typeidx, false, type_idx); + if (!push_const_expr_stack( + &const_expr_ctx, flag, cur_ref_type.ref_type, + &cur_ref_type, (uint8)opcode1, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) { + destroy_init_expr_data_recursive( + module, struct_init_values); + goto fail; + } + break; + } + case WASM_OP_STRUCT_NEW_DEFAULT: + { + read_leb_uint32(p, p_end, cur_value.type_index); + type_idx = cur_value.type_index; + + if (!check_type_index(module, module->type_count, + type_idx, error_buf, + error_buf_size)) { + goto fail; + } + if (module->types[type_idx]->type_flag + != WASM_TYPE_STRUCT) { + set_error_buf(error_buf, error_buf_size, + "unknown struct type"); + goto fail; + } + + cur_value.type_index = type_idx; + cur_value.data = NULL; + wasm_set_refheaptype_typeidx( + &cur_ref_type.ref_ht_typeidx, false, type_idx); + if (!push_const_expr_stack( + &const_expr_ctx, flag, cur_ref_type.ref_type, + &cur_ref_type, (uint8)opcode1, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) { + goto fail; + } + break; + } + case WASM_OP_ARRAY_NEW: + case WASM_OP_ARRAY_NEW_DEFAULT: + case WASM_OP_ARRAY_NEW_FIXED: + { + WASMArrayNewInitValues *array_init_values = NULL; + WASMArrayType *array_type = NULL; + WASMRefType *elem_ref_type = NULL; + uint64 total_size; + uint8 elem_type; + + read_leb_uint32(p, p_end, cur_value.type_index); + type_idx = cur_value.type_index; + + if (!check_type_index(module, module->type_count, + type_idx, error_buf, + error_buf_size)) { + goto fail; + } + + array_type = (WASMArrayType *)module->types[type_idx]; + if (array_type->base_type.type_flag + != WASM_TYPE_ARRAY) { + set_error_buf(error_buf, error_buf_size, + "unknown array type"); + goto fail; + } + + if (opcode1 != WASM_OP_ARRAY_NEW_DEFAULT) { + elem_type = array_type->elem_type; + if (wasm_is_type_multi_byte_type(elem_type)) { + elem_ref_type = array_type->elem_ref_type; + } + + if (is_packed_type(elem_type)) { + elem_type = VALUE_TYPE_I32; + } + + if (opcode1 == WASM_OP_ARRAY_NEW) { + WASMValue len_val = { 0 }; + uint64 size = 0; + + if (!pop_const_expr_stack( + &const_expr_ctx, NULL, VALUE_TYPE_I32, + NULL, NULL, &len_val, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) { + goto fail; + } + + size = + sizeof(WASMArrayNewInitValues) + + sizeof(WASMValue) * (uint64)len_val.i32; + if (!(array_init_values = loader_malloc( + size, error_buf, error_buf_size))) { + goto fail; + } + + array_init_values->type_idx = type_idx; + array_init_values->length = len_val.i32; + + if (!pop_const_expr_stack( + &const_expr_ctx, NULL, elem_type, + elem_ref_type, NULL, + &array_init_values->elem_data[0], +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) { + destroy_init_expr_data_recursive( + module, array_init_values); + goto fail; + } + + cur_value.data = array_init_values; + } + else { + /* WASM_OP_ARRAY_NEW_FIXED */ + uint32 len; + read_leb_uint32(p, p_end, len); + + total_size = + (uint64)offsetof(WASMArrayNewInitValues, + elem_data) + + (uint64)sizeof(WASMValue) * len; + if (!(array_init_values = + loader_malloc(total_size, error_buf, + error_buf_size))) { + goto fail; + } + + array_init_values->type_idx = type_idx; + array_init_values->length = len; + + for (i = len; i > 0; i--) { + if (!pop_const_expr_stack( + &const_expr_ctx, NULL, elem_type, + elem_ref_type, NULL, + &array_init_values + ->elem_data[i - 1], +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) { + destroy_init_expr_data_recursive( + module, array_init_values); + goto fail; + } + } + + cur_value.data = array_init_values; + } + } + else { + /* WASM_OP_ARRAY_NEW_DEFAULT */ + WASMValue len_val; + uint32 len; + + /* POP(i32) */ + if (!pop_const_expr_stack( + &const_expr_ctx, NULL, VALUE_TYPE_I32, NULL, + NULL, &len_val, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) { + goto fail; + } + len = len_val.i32; + + cur_value.array_new_default.type_index = type_idx; + cur_value.array_new_default.length = len; + } + + wasm_set_refheaptype_typeidx( + &cur_ref_type.ref_ht_typeidx, false, type_idx); + if (!push_const_expr_stack( + &const_expr_ctx, flag, cur_ref_type.ref_type, + &cur_ref_type, (uint8)opcode1, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) { + if (array_init_values) { + destroy_init_expr_data_recursive( + module, array_init_values); + } + goto fail; + } + break; + } + case WASM_OP_ANY_CONVERT_EXTERN: + { + set_error_buf(error_buf, error_buf_size, + "unsupported constant expression of " + "extern.internalize"); + goto fail; + } + case WASM_OP_EXTERN_CONVERT_ANY: + { + set_error_buf(error_buf, error_buf_size, + "unsupported constant expression of " + "extern.externalize"); + goto fail; + } + case WASM_OP_REF_I31: + { + /* POP(i32) */ + if (!pop_const_expr_stack(&const_expr_ctx, NULL, + VALUE_TYPE_I32, NULL, NULL, + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) { + goto fail; + } + + wasm_set_refheaptype_common(&cur_ref_type.ref_ht_common, + false, HEAP_TYPE_I31); + if (!push_const_expr_stack( + &const_expr_ctx, flag, cur_ref_type.ref_type, + &cur_ref_type, (uint8)opcode1, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) { + goto fail; + } + break; + } + default: + set_error_buf( + error_buf, error_buf_size, + "type mismatch or constant expression required"); + goto fail; + } + + break; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + default: + { + set_error_buf(error_buf, error_buf_size, + "illegal opcode " + "or constant expression required " + "or type mismatch"); + goto fail; + } + } + + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + } + + /* There should be only one value left on the init value stack */ + if (!pop_const_expr_stack(&const_expr_ctx, &flag, type, +#if WASM_ENABLE_GC != 0 + ref_type, &opcode, +#endif + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + &cur_expr, +#endif + error_buf, error_buf_size)) { + goto fail; + } + + if (const_expr_ctx.sp != 0) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: illegal constant opcode sequence"); + goto fail; + } + +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + if (cur_expr != NULL) { + bh_memcpy_s(init_expr, sizeof(InitializerExpression), cur_expr, + sizeof(InitializerExpression)); + wasm_runtime_free(cur_expr); + } + else { + init_expr->init_expr_type = flag; + init_expr->u.unary.v = cur_value; + } + +#else + init_expr->init_expr_type = flag; + init_expr->u.unary.v = cur_value; +#endif /* end of WASM_ENABLE_EXTENDED_CONST_EXPR != 0 */ + +#if WASM_ENABLE_GC != 0 + if (init_expr->init_expr_type == WASM_OP_GC_PREFIX) { + switch (opcode) { + case WASM_OP_STRUCT_NEW: + init_expr->init_expr_type = INIT_EXPR_TYPE_STRUCT_NEW; + break; + case WASM_OP_STRUCT_NEW_DEFAULT: + init_expr->init_expr_type = INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT; + break; + case WASM_OP_ARRAY_NEW: + init_expr->init_expr_type = INIT_EXPR_TYPE_ARRAY_NEW; + break; + case WASM_OP_ARRAY_NEW_DEFAULT: + init_expr->init_expr_type = INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT; + break; + case WASM_OP_ARRAY_NEW_FIXED: + init_expr->init_expr_type = INIT_EXPR_TYPE_ARRAY_NEW_FIXED; + break; + case WASM_OP_REF_I31: + init_expr->init_expr_type = INIT_EXPR_TYPE_I31_NEW; + break; + default: + bh_assert(0); + break; + } + } +#endif /* end of WASM_ENABLE_GC != 0 */ + + *p_buf = p; + destroy_const_expr_stack(&const_expr_ctx, false); + return true; + +fail: + destroy_const_expr_stack(&const_expr_ctx, true); + return false; +} + +static bool +check_mutability(uint8 mutable, char *error_buf, uint32 error_buf_size) +{ + if (mutable >= 2) { + set_error_buf(error_buf, error_buf_size, "invalid mutability"); + return false; + } + return true; +} + +#if WASM_ENABLE_GC != 0 +static void +destroy_func_type(WASMFuncType *type) +{ + /* Destroy the reference type hash set */ + if (type->ref_type_maps) + wasm_runtime_free(type->ref_type_maps); + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + if (type->call_to_llvm_jit_from_fast_jit) + jit_code_cache_free(type->call_to_llvm_jit_from_fast_jit); +#endif + /* Free the type */ + wasm_runtime_free(type); +} + +static void +destroy_struct_type(WASMStructType *type) +{ + if (type->ref_type_maps) + wasm_runtime_free(type->ref_type_maps); + + wasm_runtime_free(type); +} + +static void +destroy_array_type(WASMArrayType *type) +{ + wasm_runtime_free(type); +} + +static void +destroy_wasm_type(WASMType *type) +{ + if (type->ref_count > 1) { + /* The type is referenced by other types + of current wasm module */ + type->ref_count--; + return; + } + + if (type->type_flag == WASM_TYPE_FUNC) + destroy_func_type((WASMFuncType *)type); + else if (type->type_flag == WASM_TYPE_STRUCT) + destroy_struct_type((WASMStructType *)type); + else if (type->type_flag == WASM_TYPE_ARRAY) + destroy_array_type((WASMArrayType *)type); + else { + bh_assert(0); + } +} + +/* Resolve (ref null ht) or (ref ht) */ +static bool +resolve_reftype_htref(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *module, uint32 type_count, bool nullable, + WASMRefType *ref_type, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + + ref_type->ref_type = + nullable ? REF_TYPE_HT_NULLABLE : REF_TYPE_HT_NON_NULLABLE; + ref_type->ref_ht_common.nullable = nullable; + read_leb_int32(p, p_end, ref_type->ref_ht_common.heap_type); + + if (wasm_is_refheaptype_typeidx(&ref_type->ref_ht_common)) { + /* heap type is (type i), i : typeidx, >= 0 */ + if (!check_type_index(module, type_count, + ref_type->ref_ht_typeidx.type_idx, error_buf, + error_buf_size)) { + return false; + } + } + else if (!wasm_is_refheaptype_common(&ref_type->ref_ht_common)) { + /* heap type is func, extern, any, eq, i31 or data */ + set_error_buf(error_buf, error_buf_size, "unknown heap type"); + return false; + } + + *p_buf = p; + return true; +fail: + return false; +} + +static bool +resolve_value_type(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *module, uint32 type_count, + bool *p_need_ref_type_map, WASMRefType *ref_type, + bool allow_packed_type, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint8 type; + + memset(ref_type, 0, sizeof(WASMRefType)); + + CHECK_BUF(p, p_end, 1); + type = read_uint8(p); + + if (wasm_is_reftype_htref_nullable(type)) { + /* (ref null ht) */ + if (!resolve_reftype_htref(&p, p_end, module, type_count, true, + ref_type, error_buf, error_buf_size)) + return false; + if (!wasm_is_refheaptype_common(&ref_type->ref_ht_common)) + *p_need_ref_type_map = true; + else { + /* For (ref null func/extern/any/eq/i31/data), they are same as + funcref/externref/anyref/eqref/i31ref/dataref, we convert the + multi-byte type to one-byte type to reduce the footprint and + the complexity of type equal/subtype checking */ + ref_type->ref_type = + (uint8)((int32)0x80 + ref_type->ref_ht_common.heap_type); + *p_need_ref_type_map = false; + } + } + else if (wasm_is_reftype_htref_non_nullable(type)) { + /* (ref ht) */ + if (!resolve_reftype_htref(&p, p_end, module, type_count, false, + ref_type, error_buf, error_buf_size)) + return false; + *p_need_ref_type_map = true; +#if WASM_ENABLE_STRINGREF != 0 + /* covert (ref string) to stringref */ + if (wasm_is_refheaptype_stringrefs(&ref_type->ref_ht_common)) { + ref_type->ref_type = + (uint8)((int32)0x80 + ref_type->ref_ht_common.heap_type); + *p_need_ref_type_map = false; + } +#endif + } + else { + /* type which can be represented by one byte */ + if (!is_valid_value_type_for_interpreter(type) + && !(allow_packed_type && is_packed_type(type))) { + set_error_buf(error_buf, error_buf_size, "type mismatch"); + return false; + } + ref_type->ref_type = type; + *p_need_ref_type_map = false; +#if WASM_ENABLE_WAMR_COMPILER != 0 + /* If any value's type is v128, mark the module as SIMD used */ + if (type == VALUE_TYPE_V128) + module->is_simd_used = true; +#endif + } + + *p_buf = p; + return true; +fail: + return false; +} + +static WASMRefType * +reftype_set_insert(HashMap *ref_type_set, const WASMRefType *ref_type, + char *error_buf, uint32 error_buf_size) +{ + WASMRefType *ret = wasm_reftype_set_insert(ref_type_set, ref_type); + + if (!ret) { + set_error_buf(error_buf, error_buf_size, + "insert ref type to hash set failed"); + } + return ret; +} + +static bool +resolve_func_type(const uint8 **p_buf, const uint8 *buf_end, WASMModule *module, + uint32 type_count, uint32 type_idx, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; + uint32 param_count, result_count, i, j = 0; + uint32 param_cell_num, ret_cell_num; + uint32 ref_type_map_count = 0, result_ref_type_map_count = 0; + uint64 total_size; + bool need_ref_type_map; + WASMRefType ref_type; + WASMFuncType *type = NULL; + + /* Parse first time to resolve param count, result count and + ref type map count */ + read_leb_uint32(p, p_end, param_count); + p_org = p; + for (i = 0; i < param_count; i++) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { + return false; + } + if (need_ref_type_map) + ref_type_map_count++; + } + + read_leb_uint32(p, p_end, result_count); + for (i = 0; i < result_count; i++) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { + return false; + } + if (need_ref_type_map) { + ref_type_map_count++; + result_ref_type_map_count++; + } + } + + LOG_VERBOSE("type %u: func, param count: %d, result count: %d, " + "ref type map count: %d", + type_idx, param_count, result_count, ref_type_map_count); + + /* Parse second time to resolve param types, result types and + ref type map info */ + p = p_org; + + total_size = offsetof(WASMFuncType, types) + + sizeof(uint8) * (uint64)(param_count + result_count); + if (!(type = loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + if (ref_type_map_count > 0) { + if (ref_type_map_count > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "ref type count too large"); + return false; + } + total_size = sizeof(WASMRefTypeMap) * (uint64)ref_type_map_count; + if (!(type->ref_type_maps = + loader_malloc(total_size, error_buf, error_buf_size))) { + goto fail; + } + } + + type->base_type.type_flag = WASM_TYPE_FUNC; + type->param_count = param_count; + type->result_count = result_count; + type->ref_type_map_count = ref_type_map_count; + if (ref_type_map_count > 0) { + type->result_ref_type_maps = type->ref_type_maps + ref_type_map_count + - result_ref_type_map_count; + } + + for (i = 0; i < param_count; i++) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { + goto fail; + } + type->types[i] = ref_type.ref_type; + if (need_ref_type_map) { + type->ref_type_maps[j].index = i; + if (!(type->ref_type_maps[j++].ref_type = + reftype_set_insert(module->ref_type_set, &ref_type, + error_buf, error_buf_size))) { + goto fail; + } + } + } + + read_leb_uint32(p, p_end, result_count); + for (i = 0; i < result_count; i++) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { + goto fail; + } + type->types[param_count + i] = ref_type.ref_type; + if (need_ref_type_map) { + type->ref_type_maps[j].index = param_count + i; + if (!(type->ref_type_maps[j++].ref_type = + reftype_set_insert(module->ref_type_set, &ref_type, + error_buf, error_buf_size))) { + goto fail; + } + } + } + + bh_assert(j == type->ref_type_map_count); +#if TRACE_WASM_LOADER != 0 + os_printf("type %d = ", type_idx); + wasm_dump_func_type(type); +#endif + + param_cell_num = wasm_get_cell_num(type->types, param_count); + ret_cell_num = wasm_get_cell_num(type->types + param_count, result_count); + if (param_cell_num > UINT16_MAX || ret_cell_num > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "param count or result count too large"); + goto fail; + } + type->param_cell_num = (uint16)param_cell_num; + type->ret_cell_num = (uint16)ret_cell_num; + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + type->quick_aot_entry = wasm_native_lookup_quick_aot_entry(type); +#endif + +#if WASM_ENABLE_WAMR_COMPILER != 0 + for (i = 0; i < (uint32)(type->param_count + type->result_count); i++) { + if (type->types[i] == VALUE_TYPE_V128) + module->is_simd_used = true; + } +#endif + + *p_buf = p; + + module->types[type_idx] = (WASMType *)type; + return true; + +fail: + if (type) + destroy_func_type(type); + return false; +} + +static bool +resolve_struct_type(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *module, uint32 type_count, uint32 type_idx, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; + uint32 field_count, ref_type_map_count = 0, ref_field_count = 0; + uint32 i, j = 0, offset; + uint16 *reference_table; + uint64 total_size; + uint8 mutable; + bool need_ref_type_map; + WASMRefType ref_type; + WASMStructType *type = NULL; + + /* Parse first time to resolve field count and ref type map count */ + read_leb_uint32(p, p_end, field_count); + p_org = p; + for (i = 0; i < field_count; i++) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, true, error_buf, + error_buf_size)) { + return false; + } + if (need_ref_type_map) + ref_type_map_count++; + + if (wasm_is_type_reftype(ref_type.ref_type)) + ref_field_count++; + + CHECK_BUF(p, p_end, 1); + mutable = read_uint8(p); + if (!check_mutability(mutable, error_buf, error_buf_size)) { + return false; + } + } + + LOG_VERBOSE("type %u: struct, field count: %d, ref type map count: %d", + type_idx, field_count, ref_type_map_count); + + /* Parse second time to resolve field types and ref type map info */ + p = p_org; + + total_size = offsetof(WASMStructType, fields) + + sizeof(WASMStructFieldType) * (uint64)field_count + + sizeof(uint16) * (uint64)(ref_field_count + 1); + if (!(type = loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + if (ref_type_map_count > 0) { + if (ref_type_map_count > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "ref type count too large"); + return false; + } + total_size = sizeof(WASMRefTypeMap) * (uint64)ref_type_map_count; + if (!(type->ref_type_maps = + loader_malloc(total_size, error_buf, error_buf_size))) { + goto fail; + } + } + + type->reference_table = reference_table = + (uint16 *)((uint8 *)type + offsetof(WASMStructType, fields) + + sizeof(WASMStructFieldType) * field_count); + *reference_table++ = ref_field_count; + + type->base_type.type_flag = WASM_TYPE_STRUCT; + type->field_count = field_count; + type->ref_type_map_count = ref_type_map_count; + + offset = (uint32)offsetof(WASMStructObject, field_data); + for (i = 0; i < field_count; i++) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, true, error_buf, + error_buf_size)) { + goto fail; + } + if (!is_valid_field_type(ref_type.ref_type)) { + set_error_buf(error_buf, error_buf_size, "invalid field type"); + goto fail; + } + type->fields[i].field_type = ref_type.ref_type; + if (need_ref_type_map) { + type->ref_type_maps[j].index = i; + if (!(type->ref_type_maps[j++].ref_type = + reftype_set_insert(module->ref_type_set, &ref_type, + error_buf, error_buf_size))) { + goto fail; + } + } + + CHECK_BUF(p, p_end, 1); + type->fields[i].field_flags = read_uint8(p); + type->fields[i].field_size = + (uint8)wasm_reftype_size(ref_type.ref_type); +#if !(defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_X86_32)) + if (type->fields[i].field_size == 2) + offset = align_uint(offset, 2); + else if (type->fields[i].field_size >= 4) /* field size is 4 or 8 */ + offset = align_uint(offset, 4); +#endif + type->fields[i].field_offset = offset; + if (wasm_is_type_reftype(ref_type.ref_type)) + *reference_table++ = offset; + offset += type->fields[i].field_size; + + LOG_VERBOSE(" field: %d, flags: %d, type: %d", i, + type->fields[i].field_flags, type->fields[i].field_type); + } + type->total_size = offset; + + bh_assert(j == type->ref_type_map_count); +#if TRACE_WASM_LOADER != 0 + os_printf("type %d = ", type_idx); + wasm_dump_struct_type(type); +#endif + + *p_buf = p; + + module->types[type_idx] = (WASMType *)type; + return true; + +fail: + if (type) + destroy_struct_type(type); + return false; +} + +static bool +resolve_array_type(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *module, uint32 type_count, uint32 type_idx, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint8 mutable; + bool need_ref_type_map; + WASMRefType ref_type; + WASMArrayType *type = NULL; + + if (!resolve_value_type(&p, p_end, module, type_count, &need_ref_type_map, + &ref_type, true, error_buf, error_buf_size)) { + return false; + } + + CHECK_BUF(p, p_end, 1); + mutable = read_uint8(p); + if (!check_mutability(mutable, error_buf, error_buf_size)) { + return false; + } + + LOG_VERBOSE("type %u: array", type_idx); + + if (!(type = loader_malloc(sizeof(WASMArrayType), error_buf, + error_buf_size))) { + return false; + } + + type->base_type.type_flag = WASM_TYPE_ARRAY; + type->elem_flags = mutable; + type->elem_type = ref_type.ref_type; + if (need_ref_type_map) { + if (!(type->elem_ref_type = + reftype_set_insert(module->ref_type_set, &ref_type, error_buf, + error_buf_size))) { + goto fail; + } + } + +#if TRACE_WASM_LOADER != 0 + os_printf("type %d = ", type_idx); + wasm_dump_array_type(type); +#endif + + *p_buf = p; + + module->types[type_idx] = (WASMType *)type; + return true; + +fail: + if (type) + destroy_array_type(type); + return false; +} + +static bool +init_ref_type(WASMModule *module, WASMRefType *ref_type, bool nullable, + int32 heap_type, char *error_buf, uint32 error_buf_size) +{ + if (heap_type >= 0) { + if (!check_type_index(module, module->type_count, heap_type, error_buf, + error_buf_size)) { + return false; + } + wasm_set_refheaptype_typeidx(&ref_type->ref_ht_typeidx, nullable, + heap_type); + } + else { + if (!wasm_is_valid_heap_type(heap_type)) { + set_error_buf(error_buf, error_buf_size, "unknown type"); + return false; + } + wasm_set_refheaptype_common(&ref_type->ref_ht_common, nullable, + heap_type); + if (nullable) { + /* For (ref null func/extern/any/eq/i31/data), + they are same as + funcref/externref/anyref/eqref/i31ref/dataref, + we convert the multi-byte type to one-byte + type to reduce the footprint and the + complexity of type equal/subtype checking */ + ref_type->ref_type = + (uint8)((int32)0x80 + ref_type->ref_ht_common.heap_type); + } + } + return true; +} + +static void +calculate_reftype_diff(WASMRefType *ref_type_diff, WASMRefType *ref_type1, + WASMRefType *ref_type2) +{ + /** + * The difference rt1 ∖ rt2 between two reference types is defined as + * follows: + * (ref null?1 ht1) ∖ (ref null ht2) = (ref ht1) (ref null?1 ht1) ∖ + * (ref ht2) = (ref null?1 ht1) + */ + if (wasm_is_type_multi_byte_type(ref_type1->ref_type)) { + bh_memcpy_s(ref_type_diff, wasm_reftype_struct_size(ref_type1), + ref_type1, wasm_reftype_struct_size(ref_type1)); + } + else { + ref_type_diff->ref_type = ref_type1->ref_type; + } + + if (ref_type2->ref_ht_common.nullable) { + if (wasm_is_type_reftype(ref_type_diff->ref_type) + && !(wasm_is_type_multi_byte_type(ref_type_diff->ref_type))) { + wasm_set_refheaptype_typeidx(&ref_type_diff->ref_ht_typeidx, false, + (int32)ref_type_diff->ref_type - 0x80); + } + else { + ref_type_diff->ref_ht_typeidx.nullable = false; + } + } +} +#else /* else of WASM_ENABLE_GC != 0 */ +static void +destroy_wasm_type(WASMType *type) +{ + if (type->ref_count > 1) { + /* The type is referenced by other types + of current wasm module */ + type->ref_count--; + return; + } + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + if (type->call_to_llvm_jit_from_fast_jit) + jit_code_cache_free(type->call_to_llvm_jit_from_fast_jit); +#endif + + wasm_runtime_free(type); +} +#endif /* end of WASM_ENABLE_GC != 0 */ + +static bool +load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 type_count, i; + uint64 total_size; + uint8 flag; +#if WASM_ENABLE_GC != 0 + uint32 processed_type_count = 0; +#endif + + read_leb_uint32(p, p_end, type_count); + + if (type_count) { + module->type_count = type_count; + total_size = sizeof(WASMType *) * (uint64)type_count; + if (!(module->types = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + +#if WASM_ENABLE_GC == 0 + for (i = 0; i < type_count; i++) { + WASMFuncType *type; + const uint8 *p_org; + uint32 param_count, result_count, j; + uint32 param_cell_num, ret_cell_num; + + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + if (flag != 0x60) { + set_error_buf(error_buf, error_buf_size, "invalid type flag"); + return false; + } + + read_leb_uint32(p, p_end, param_count); + + /* Resolve param count and result count firstly */ + p_org = p; + CHECK_BUF(p, p_end, param_count); + p += param_count; + read_leb_uint32(p, p_end, result_count); + CHECK_BUF(p, p_end, result_count); + p = p_org; + + if (param_count > UINT16_MAX || result_count > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "param count or result count too large"); + return false; + } + + total_size = offsetof(WASMFuncType, types) + + sizeof(uint8) * (uint64)(param_count + result_count); + if (!(type = module->types[i] = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* Resolve param types and result types */ + type->ref_count = 1; + type->param_count = (uint16)param_count; + type->result_count = (uint16)result_count; + for (j = 0; j < param_count; j++) { + CHECK_BUF(p, p_end, 1); + type->types[j] = read_uint8(p); + } + read_leb_uint32(p, p_end, result_count); + for (j = 0; j < result_count; j++) { + CHECK_BUF(p, p_end, 1); + type->types[param_count + j] = read_uint8(p); + } + for (j = 0; j < param_count + result_count; j++) { + if (!is_valid_value_type_for_interpreter(type->types[j])) { + set_error_buf(error_buf, error_buf_size, + "unknown value type"); + return false; + } + } + + param_cell_num = wasm_get_cell_num(type->types, param_count); + ret_cell_num = + wasm_get_cell_num(type->types + param_count, result_count); + if (param_cell_num > UINT16_MAX || ret_cell_num > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "param count or result count too large"); + return false; + } + type->param_cell_num = (uint16)param_cell_num; + type->ret_cell_num = (uint16)ret_cell_num; + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + type->quick_aot_entry = wasm_native_lookup_quick_aot_entry(type); +#endif + +#if WASM_ENABLE_WAMR_COMPILER != 0 + for (j = 0; j < type->param_count + type->result_count; j++) { + if (type->types[j] == VALUE_TYPE_V128) + module->is_simd_used = true; + else if (type->types[j] == VALUE_TYPE_FUNCREF + || type->types[j] == VALUE_TYPE_EXTERNREF) + module->is_ref_types_used = true; + } +#endif + + /* If there is already a same type created, use it instead */ + for (j = 0; j < i; j++) { + if (wasm_type_equal(type, module->types[j], module->types, i)) { + if (module->types[j]->ref_count == UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "wasm type's ref count too large"); + return false; + } + destroy_wasm_type(type); + module->types[i] = module->types[j]; + module->types[j]->ref_count++; + break; + } + } + } +#else /* else of WASM_ENABLE_GC == 0 */ + for (i = 0; i < type_count; i++) { + uint32 super_type_count = 0, parent_type_idx = (uint32)-1; + uint32 rec_count = 1, j; + bool is_sub_final = true; + + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + + if (flag == DEFINED_TYPE_REC) { + read_leb_uint32(p, p_end, rec_count); + + if (rec_count > 1) { + uint64 new_total_size; + + /* integer overflow */ + if (rec_count - 1 > UINT32_MAX - module->type_count) { + set_error_buf(error_buf, error_buf_size, + "recursive type count too large"); + return false; + } + new_total_size = + sizeof(WASMFuncType *) + * (uint64)(module->type_count + rec_count - 1); + if (new_total_size > UINT32_MAX) { + set_error_buf(error_buf, error_buf_size, + "allocate memory failed"); + return false; + } + MEM_REALLOC(module->types, (uint32)total_size, + (uint32)new_total_size); + module->type_count += rec_count - 1; + total_size = new_total_size; + } + + if (rec_count < 1) { + LOG_VERBOSE("Processing 0-entry rec group"); + } + else { + LOG_VERBOSE("Processing rec group [%d-%d]", + processed_type_count, + processed_type_count + rec_count - 1); + } + } + else { + p--; + } + + for (j = 0; j < rec_count; j++) { + WASMType *cur_type = NULL; + + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + + parent_type_idx = -1; + + if (flag == DEFINED_TYPE_SUB + || flag == DEFINED_TYPE_SUB_FINAL) { + read_leb_uint32(p, p_end, super_type_count); + if (super_type_count > 1) { + set_error_buf(error_buf, error_buf_size, + "super type count too large"); + return false; + } + + if (super_type_count > 0) { + read_leb_uint32(p, p_end, parent_type_idx); + if (parent_type_idx >= processed_type_count + j) { + set_error_buf_v(error_buf, error_buf_size, + "unknown type %d", parent_type_idx); + return false; + } + if (module->types[parent_type_idx]->is_sub_final) { + set_error_buf(error_buf, error_buf_size, + "sub type can not inherit from " + "a final super type"); + return false; + } + } + + if (flag == DEFINED_TYPE_SUB) + is_sub_final = false; + + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + } + + if (flag == DEFINED_TYPE_FUNC) { + if (!resolve_func_type(&p, buf_end, module, + processed_type_count + rec_count, + processed_type_count + j, error_buf, + error_buf_size)) { + return false; + } + } + else if (flag == DEFINED_TYPE_STRUCT) { + if (!resolve_struct_type(&p, buf_end, module, + processed_type_count + rec_count, + processed_type_count + j, + error_buf, error_buf_size)) { + return false; + } + } + else if (flag == DEFINED_TYPE_ARRAY) { + if (!resolve_array_type(&p, buf_end, module, + processed_type_count + rec_count, + processed_type_count + j, error_buf, + error_buf_size)) { + return false; + } + } + else { + set_error_buf(error_buf, error_buf_size, + "invalid type flag"); + return false; + } + + cur_type = module->types[processed_type_count + j]; + + cur_type->ref_count = 1; + cur_type->parent_type_idx = parent_type_idx; + cur_type->is_sub_final = is_sub_final; + + cur_type->rec_count = rec_count; + cur_type->rec_idx = j; + cur_type->rec_begin_type_idx = processed_type_count; + } + + /* resolve subtyping relationship in current rec group */ + for (j = 0; j < rec_count; j++) { + WASMType *cur_type = module->types[processed_type_count + j]; + + if (cur_type->parent_type_idx != (uint32)-1) { /* has parent */ + WASMType *parent_type = + module->types[cur_type->parent_type_idx]; + cur_type->parent_type = parent_type; + cur_type->root_type = parent_type->root_type; + if (parent_type->inherit_depth == UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "parent type's inherit depth too large"); + return false; + } + cur_type->inherit_depth = parent_type->inherit_depth + 1; + } + else { + cur_type->parent_type = NULL; + cur_type->root_type = cur_type; + cur_type->inherit_depth = 0; + } + } + + for (j = 0; j < rec_count; j++) { + WASMType *cur_type = module->types[processed_type_count + j]; + + if (cur_type->parent_type_idx != (uint32)-1) { /* has parent */ + WASMType *parent_type = + module->types[cur_type->parent_type_idx]; + if (!wasm_type_is_subtype_of(cur_type, parent_type, + module->types, + module->type_count)) { + set_error_buf_v(error_buf, error_buf_size, + "sub type %u does not match super type", + processed_type_count + j); + return false; + } + } + } + + /* If there is already an equivalence type or a group of equivalence + recursive types created, use it or them instead */ + for (j = 0; j < processed_type_count;) { + WASMType *src_type = module->types[j]; + WASMType *cur_type = module->types[processed_type_count]; + uint32 k, src_rec_count; + + src_rec_count = src_type->rec_count; + if (src_rec_count != rec_count) { + /* no type equivalence */ + j += src_rec_count; + continue; + } + + for (k = 0; k < rec_count; k++) { + src_type = module->types[j + k]; + cur_type = module->types[processed_type_count + k]; + if (!wasm_type_equal(src_type, cur_type, module->types, + module->type_count)) { + break; + } + } + if (k < rec_count) { + /* no type equivalence */ + j += src_rec_count; + continue; + } + + /* type equivalence */ + for (k = 0; k < rec_count; k++) { + if (module->types[j + k]->ref_count == UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "wasm type's ref count too large"); + return false; + } + destroy_wasm_type(module->types[processed_type_count + k]); + module->types[processed_type_count + k] = + module->types[j + k]; + module->types[j + k]->ref_count++; + } + break; + } + + if (rec_count > 1) { + LOG_VERBOSE("Finished processing rec group [%d-%d]", + processed_type_count, + processed_type_count + rec_count - 1); + } + + processed_type_count += rec_count; + } + + if (!(module->rtt_types = loader_malloc((uint64)sizeof(WASMRttType *) + * module->type_count, + error_buf, error_buf_size))) { + return false; + } +#endif /* end of WASM_ENABLE_GC == 0 */ + } + + for (i = 0; i < module->type_count; i++) { + if (module->types[i] == NULL) { + set_error_buf_v(error_buf, error_buf_size, "unknown type %d", i); + return false; + } + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load type section success.\n"); + return true; +fail: + return false; +} + +static void +adjust_table_max_size(bool is_table64, uint32 init_size, uint32 max_size_flag, + uint32 *max_size) +{ + uint32 default_max_size; + + /* TODO: current still use UINT32_MAX as upper limit for table size to keep + * ABI unchanged */ + (void)is_table64; + if (UINT32_MAX / 2 > init_size) + default_max_size = init_size * 2; + else + default_max_size = UINT32_MAX; + + if (default_max_size < WASM_TABLE_MAX_SIZE) + default_max_size = WASM_TABLE_MAX_SIZE; + + if (max_size_flag) { + /* module defines the table limitation */ + bh_assert(init_size <= *max_size); + + if (init_size < *max_size) { + *max_size = + *max_size < default_max_size ? *max_size : default_max_size; + } + } + else { + /* partial defined table limitation, gives a default value */ + *max_size = default_max_size; + } +} + +#if WASM_ENABLE_LIBC_WASI != 0 || WASM_ENABLE_MULTI_MODULE != 0 +/** + * Find export item of a module with export info: + * module name, field name and export kind + */ +static WASMExport * +wasm_loader_find_export(const WASMModule *module, const char *module_name, + const char *field_name, uint8 export_kind, + char *error_buf, uint32 error_buf_size) +{ + WASMExport *export = + loader_find_export((WASMModuleCommon *)module, module_name, field_name, + export_kind, error_buf, error_buf_size); + return export; +} +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 +static WASMTable * +wasm_loader_resolve_table(const char *module_name, const char *table_name, + uint32 init_size, uint32 max_size, char *error_buf, + uint32 error_buf_size) +{ + WASMModuleCommon *module_reg; + WASMTable *table = NULL; + WASMExport *export = NULL; + WASMModule *module = NULL; + + module_reg = wasm_runtime_find_module_registered(module_name); + if (!module_reg || module_reg->module_type != Wasm_Module_Bytecode) { + LOG_DEBUG("can not find a module named %s for table", module_name); + set_error_buf(error_buf, error_buf_size, "unknown import"); + return NULL; + } + + module = (WASMModule *)module_reg; + export = + wasm_loader_find_export(module, module_name, table_name, + EXPORT_KIND_TABLE, error_buf, error_buf_size); + if (!export) { + return NULL; + } + + /* resolve table and check the init/max size */ + if (export->index < module->import_table_count) { + table = + module->import_tables[export->index].u.table.import_table_linked; + } + else { + table = &(module->tables[export->index - module->import_table_count]); + } + if (table->table_type.init_size < init_size + || table->table_type.max_size > max_size) { + LOG_DEBUG("%s,%s failed type check(%d-%d), expected(%d-%d)", + module_name, table_name, table->table_type.init_size, + table->table_type.max_size, init_size, max_size); + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return NULL; + } + + return table; +} + +static WASMMemory * +wasm_loader_resolve_memory(const char *module_name, const char *memory_name, + uint32 init_page_count, uint32 max_page_count, + char *error_buf, uint32 error_buf_size) +{ + WASMModuleCommon *module_reg; + WASMMemory *memory = NULL; + WASMExport *export = NULL; + WASMModule *module = NULL; + + module_reg = wasm_runtime_find_module_registered(module_name); + if (!module_reg || module_reg->module_type != Wasm_Module_Bytecode) { + LOG_DEBUG("can not find a module named %s for memory", module_name); + set_error_buf(error_buf, error_buf_size, "unknown import"); + return NULL; + } + + module = (WASMModule *)module_reg; + export = + wasm_loader_find_export(module, module_name, memory_name, + EXPORT_KIND_MEMORY, error_buf, error_buf_size); + if (!export) { + return NULL; + } + + /* resolve memory and check the init/max page count */ + if (export->index < module->import_memory_count) { + memory = module->import_memories[export->index] + .u.memory.import_memory_linked; + } + else { + memory = + &(module->memories[export->index - module->import_memory_count]); + } + if (memory->init_page_count < init_page_count + || memory->max_page_count > max_page_count) { + LOG_DEBUG("%s,%s failed type check(%d-%d), expected(%d-%d)", + module_name, memory_name, memory->init_page_count, + memory->max_page_count, init_page_count, max_page_count); + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return NULL; + } + return memory; +} + +static WASMGlobal * +wasm_loader_resolve_global(const char *module_name, const char *global_name, + uint8 type, bool is_mutable, char *error_buf, + uint32 error_buf_size) +{ + WASMModuleCommon *module_reg; + WASMGlobal *global = NULL; + WASMExport *export = NULL; + WASMModule *module = NULL; + + module_reg = wasm_runtime_find_module_registered(module_name); + if (!module_reg || module_reg->module_type != Wasm_Module_Bytecode) { + LOG_DEBUG("can not find a module named %s for global", module_name); + set_error_buf(error_buf, error_buf_size, "unknown import"); + return NULL; + } + + module = (WASMModule *)module_reg; + export = + wasm_loader_find_export(module, module_name, global_name, + EXPORT_KIND_GLOBAL, error_buf, error_buf_size); + if (!export) { + return NULL; + } + + /* resolve and check the global */ + if (export->index < module->import_global_count) { + global = + module->import_globals[export->index].u.global.import_global_linked; + } + else { + global = + &(module->globals[export->index - module->import_global_count]); + } + if (global->type.val_type != type + || global->type.is_mutable != is_mutable) { + LOG_DEBUG("%s,%s failed type check(%d, %d), expected(%d, %d)", + module_name, global_name, global->type.val_type, + global->type.is_mutable, type, is_mutable); + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return NULL; + } + return global; +} + +#if WASM_ENABLE_TAGS != 0 +static WASMTag * +wasm_loader_resolve_tag(const char *module_name, const char *tag_name, + const WASMType *expected_tag_type, + uint32 *linked_tag_index, char *error_buf, + uint32 error_buf_size) +{ + WASMModuleCommon *module_reg; + WASMTag *tag = NULL; + WASMExport *export = NULL; + WASMModule *module = NULL; + + module_reg = wasm_runtime_find_module_registered(module_name); + if (!module_reg || module_reg->module_type != Wasm_Module_Bytecode) { + LOG_DEBUG("can not find a module named %s for tag %s", module_name, + tag_name); + set_error_buf(error_buf, error_buf_size, "unknown import"); + return NULL; + } + + module = (WASMModule *)module_reg; + export = + wasm_loader_find_export(module, module_name, tag_name, EXPORT_KIND_TAG, + error_buf, error_buf_size); + if (!export) { + return NULL; + } + + /* resolve tag type and tag */ + if (export->index < module->import_tag_count) { + /* importing an imported tag from the submodule */ + tag = module->import_tags[export->index].u.tag.import_tag_linked; + } + else { + /* importing an section tag from the submodule */ + tag = module->tags[export->index - module->import_tag_count]; + } + + /* check function type */ + if (!wasm_type_equal(expected_tag_type, tag->tag_type, module->types, + module->type_count)) { + LOG_DEBUG("%s.%s failed the type check", module_name, tag_name); + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return NULL; + } + + if (linked_tag_index != NULL) { + *linked_tag_index = export->index; + } + + return tag; +} +#endif /* end of WASM_ENABLE_TAGS != 0 */ +#endif /* end of WASM_ENABLE_MULTI_MODULE */ + +static bool +load_function_import(const uint8 **p_buf, const uint8 *buf_end, + const WASMModule *parent_module, + const char *sub_module_name, const char *function_name, + WASMFunctionImport *function, bool no_resolve, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint32 declare_type_index = 0; + + read_leb_uint32(p, p_end, declare_type_index); + *p_buf = p; + + if (!check_function_type(parent_module, declare_type_index, error_buf, + error_buf_size)) { + return false; + } + +#if WASM_ENABLE_GC != 0 + function->type_idx = declare_type_index; +#endif + +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) + declare_type_index = wasm_get_smallest_type_idx( + parent_module->types, parent_module->type_count, declare_type_index); +#endif + + function->func_type = + (WASMFuncType *)parent_module->types[declare_type_index]; + + function->module_name = (char *)sub_module_name; + function->field_name = (char *)function_name; + function->attachment = NULL; + function->signature = NULL; +#if WASM_ENABLE_INVOKE_NATIVE != 0 + function->call_conv_raw = false; + + /* lookup registered native symbols first */ + + if (!no_resolve) { + wasm_resolve_import_func(parent_module, function); + } +#endif + return true; +fail: + return false; +} + +static bool +check_table_max_size(uint32 init_size, uint32 max_size, char *error_buf, + uint32 error_buf_size) +{ + if (max_size < init_size) { + set_error_buf(error_buf, error_buf_size, + "size minimum must not be greater than maximum"); + return false; + } + return true; +} + +static bool +load_table_import(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *parent_module, const char *sub_module_name, + const char *table_name, WASMTableImport *table, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; + uint32 declare_elem_type = 0, table_flag = 0, declare_init_size = 0, + declare_max_size = 0; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *sub_module = NULL; + WASMTable *linked_table = NULL; +#endif +#if WASM_ENABLE_GC != 0 + WASMRefType ref_type; + bool need_ref_type_map; +#endif + bool is_table64 = false; + +#if WASM_ENABLE_GC == 0 + CHECK_BUF(p, p_end, 1); + /* 0x70 or 0x6F */ + declare_elem_type = read_uint8(p); + if (VALUE_TYPE_FUNCREF != declare_elem_type +#if WASM_ENABLE_REF_TYPES != 0 + && VALUE_TYPE_EXTERNREF != declare_elem_type +#endif + ) { + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return false; + } +#else /* else of WASM_ENABLE_GC == 0 */ + if (!resolve_value_type(&p, p_end, parent_module, parent_module->type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { + return false; + } + if (!wasm_is_type_reftype(ref_type.ref_type) + || wasm_is_reftype_htref_non_nullable(ref_type.ref_type)) { + set_error_buf(error_buf, error_buf_size, "type mismatch"); + return false; + } + declare_elem_type = ref_type.ref_type; + if (need_ref_type_map) { + if (!(table->table_type.elem_ref_type = + reftype_set_insert(parent_module->ref_type_set, &ref_type, + error_buf, error_buf_size))) { + return false; + } + } +#if TRACE_WASM_LOADER != 0 + os_printf("import table type: "); + wasm_dump_value_type(declare_elem_type, table->table_type.elem_ref_type); + os_printf("\n"); +#endif +#endif /* end of WASM_ENABLE_GC == 0 */ + + p_org = p; + read_leb_uint32(p, p_end, table_flag); + is_table64 = table_flag & TABLE64_FLAG; + if (p - p_org > 1) { + LOG_VERBOSE("integer representation too long(import table)"); + set_error_buf(error_buf, error_buf_size, "invalid limits flags"); + return false; + } + + if (!wasm_table_check_flags(table_flag, error_buf, error_buf_size, false)) { + return false; + } + + read_leb_uint32(p, p_end, declare_init_size); + if (table_flag & MAX_TABLE_SIZE_FLAG) { + read_leb_uint32(p, p_end, declare_max_size); + if (!check_table_max_size(declare_init_size, declare_max_size, + error_buf, error_buf_size)) + return false; + } + + adjust_table_max_size(is_table64, declare_init_size, + table_flag & MAX_TABLE_SIZE_FLAG, &declare_max_size); + + *p_buf = p; + +#if WASM_ENABLE_MULTI_MODULE != 0 + if (!wasm_runtime_is_built_in_module(sub_module_name)) { + sub_module = (WASMModule *)wasm_runtime_load_depended_module( + (WASMModuleCommon *)parent_module, sub_module_name, error_buf, + error_buf_size); + if (sub_module) { + linked_table = wasm_loader_resolve_table( + sub_module_name, table_name, declare_init_size, + declare_max_size, error_buf, error_buf_size); + if (linked_table) { + /* reset with linked table limit */ + declare_elem_type = linked_table->table_type.elem_type; + declare_init_size = linked_table->table_type.init_size; + declare_max_size = linked_table->table_type.max_size; + table_flag = linked_table->table_type.flags; + table->import_table_linked = linked_table; + table->import_module = sub_module; + } + } + } +#endif /* WASM_ENABLE_MULTI_MODULE != 0 */ + + /* (table (export "table") 10 20 funcref) */ + /* (table (export "table64") 10 20 funcref) */ + /* we need this section working in wamrc */ + if (!strcmp("spectest", sub_module_name)) { + const uint32 spectest_table_init_size = 10; + const uint32 spectest_table_max_size = 20; + + if (strcmp("table", table_name) +#if WASM_ENABLE_MEMORY64 != 0 + && strcmp("table64", table_name) +#endif + ) { + set_error_buf(error_buf, error_buf_size, + "incompatible import type or unknown import"); + return false; + } + + if (declare_init_size > spectest_table_init_size + || declare_max_size < spectest_table_max_size) { + set_error_buf(error_buf, error_buf_size, + "incompatible import type"); + return false; + } + + declare_init_size = spectest_table_init_size; + declare_max_size = spectest_table_max_size; + } + + /* now we believe all declaration are ok */ + table->table_type.elem_type = declare_elem_type; + table->table_type.init_size = declare_init_size; + table->table_type.flags = table_flag; + table->table_type.max_size = declare_max_size; + +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (table->table_type.elem_type == VALUE_TYPE_EXTERNREF) + parent_module->is_ref_types_used = true; +#endif + (void)parent_module; + return true; +fail: + return false; +} + +static bool +check_memory_init_size(bool is_memory64, uint32 init_size, char *error_buf, + uint32 error_buf_size) +{ + uint32 default_max_size = + is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; + + if (!is_memory64 && init_size > default_max_size) { + set_error_buf(error_buf, error_buf_size, + "memory size must be at most 65536 pages (4GiB)"); + return false; + } +#if WASM_ENABLE_MEMORY64 != 0 + else if (is_memory64 && init_size > default_max_size) { + set_error_buf( + error_buf, error_buf_size, + "memory size must be at most 4,294,967,295 pages (274 Terabyte)"); + return false; + } +#endif + return true; +} + +static bool +check_memory_max_size(bool is_memory64, uint32 init_size, uint32 max_size, + char *error_buf, uint32 error_buf_size) +{ + uint32 default_max_size = + is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; + + if (max_size < init_size) { + set_error_buf(error_buf, error_buf_size, + "size minimum must not be greater than maximum"); + return false; + } + + if (!is_memory64 && max_size > default_max_size) { + set_error_buf(error_buf, error_buf_size, + "memory size must be at most 65536 pages (4GiB)"); + return false; + } +#if WASM_ENABLE_MEMORY64 != 0 + else if (is_memory64 && max_size > default_max_size) { + set_error_buf( + error_buf, error_buf_size, + "memory size must be at most 4,294,967,295 pages (274 Terabyte)"); + return false; + } +#endif + + return true; +} + +static bool +load_memory_import(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *parent_module, const char *sub_module_name, + const char *memory_name, WASMMemoryImport *memory, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; +#if WASM_ENABLE_APP_FRAMEWORK != 0 + uint32 pool_size = wasm_runtime_memory_pool_size(); + uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT + / DEFAULT_NUM_BYTES_PER_PAGE; +#else + uint32 max_page_count; +#endif /* WASM_ENABLE_APP_FRAMEWORK */ + uint32 mem_flag = 0; + bool is_memory64 = false; + uint32 declare_init_page_count = 0; + uint32 declare_max_page_count = 0; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *sub_module = NULL; + WASMMemory *linked_memory = NULL; +#endif + + p_org = p; + read_leb_uint32(p, p_end, mem_flag); + is_memory64 = mem_flag & MEMORY64_FLAG; + if (p - p_org > 1) { + LOG_VERBOSE("integer representation too long(import memory)"); + set_error_buf(error_buf, error_buf_size, "invalid limits flags"); + return false; + } + + if (!wasm_memory_check_flags(mem_flag, error_buf, error_buf_size, false)) { + return false; + } + + read_leb_uint32(p, p_end, declare_init_page_count); + if (!check_memory_init_size(is_memory64, declare_init_page_count, error_buf, + error_buf_size)) { + return false; + } + +#if WASM_ENABLE_APP_FRAMEWORK == 0 + max_page_count = is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; +#endif + if (mem_flag & MAX_PAGE_COUNT_FLAG) { + read_leb_uint32(p, p_end, declare_max_page_count); + if (!check_memory_max_size(is_memory64, declare_init_page_count, + declare_max_page_count, error_buf, + error_buf_size)) { + return false; + } + if (declare_max_page_count > max_page_count) { + declare_max_page_count = max_page_count; + } + } + else { + /* Limit the maximum memory size to max_page_count */ + declare_max_page_count = max_page_count; + } + +#if WASM_ENABLE_MULTI_MODULE != 0 + if (!wasm_runtime_is_built_in_module(sub_module_name)) { + sub_module = (WASMModule *)wasm_runtime_load_depended_module( + (WASMModuleCommon *)parent_module, sub_module_name, error_buf, + error_buf_size); + if (sub_module) { + linked_memory = wasm_loader_resolve_memory( + sub_module_name, memory_name, declare_init_page_count, + declare_max_page_count, error_buf, error_buf_size); + if (linked_memory) { + /** + * reset with linked memory limit + */ + memory->import_module = sub_module; + memory->import_memory_linked = linked_memory; + declare_init_page_count = linked_memory->init_page_count; + declare_max_page_count = linked_memory->max_page_count; + } + } + } +#endif + + /* (memory (export "memory") 1 2) */ + if (!strcmp("spectest", sub_module_name)) { + uint32 spectest_memory_init_page = 1; + uint32 spectest_memory_max_page = 2; + + if (strcmp("memory", memory_name)) { + set_error_buf(error_buf, error_buf_size, + "incompatible import type or unknown import"); + return false; + } + + if (declare_init_page_count > spectest_memory_init_page + || declare_max_page_count < spectest_memory_max_page) { + set_error_buf(error_buf, error_buf_size, + "incompatible import type"); + return false; + } + + declare_init_page_count = spectest_memory_init_page; + declare_max_page_count = spectest_memory_max_page; + } +#if WASM_ENABLE_WASI_TEST != 0 + /* a case in wasi-testsuite which imports ("foo" "bar") */ + else if (!strcmp("foo", sub_module_name)) { + uint32 spectest_memory_init_page = 1; + uint32 spectest_memory_max_page = 1; + + if (strcmp("bar", memory_name)) { + set_error_buf(error_buf, error_buf_size, + "incompatible import type or unknown import"); + return false; + } + + if (declare_init_page_count > spectest_memory_init_page + || declare_max_page_count < spectest_memory_max_page) { + set_error_buf(error_buf, error_buf_size, + "incompatible import type"); + return false; + } + + declare_init_page_count = spectest_memory_init_page; + declare_max_page_count = spectest_memory_max_page; + } +#endif + + /* now we believe all declaration are ok */ + memory->mem_type.flags = mem_flag; + memory->mem_type.init_page_count = declare_init_page_count; + memory->mem_type.max_page_count = declare_max_page_count; + memory->mem_type.num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE; + + *p_buf = p; + + (void)parent_module; + return true; +fail: + return false; +} + +#if WASM_ENABLE_TAGS != 0 +static bool +load_tag_import(const uint8 **p_buf, const uint8 *buf_end, + const WASMModule *parent_module, /* this module ! */ + const char *sub_module_name, const char *tag_name, + WASMTagImport *tag, /* structure to fill */ + char *error_buf, uint32 error_buf_size) +{ + /* attribute and type of the import statement */ + uint8 declare_tag_attribute; + uint32 declare_type_index; + const uint8 *p = *p_buf, *p_end = buf_end; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *sub_module = NULL; +#endif + + /* get the one byte attribute */ + CHECK_BUF(p, p_end, 1); + declare_tag_attribute = read_uint8(p); + if (declare_tag_attribute != 0) { + set_error_buf(error_buf, error_buf_size, "unknown tag attribute"); + goto fail; + } + + /* get type */ + read_leb_uint32(p, p_end, declare_type_index); + /* compare against module->types */ + if (!check_function_type(parent_module, declare_type_index, error_buf, + error_buf_size)) { + goto fail; + } + + WASMFuncType *declare_tag_type = + (WASMFuncType *)parent_module->types[declare_type_index]; + + /* check, that the type of the declared tag returns void */ + if (declare_tag_type->result_count != 0) { + set_error_buf(error_buf, error_buf_size, + "tag type signature does not return void"); + + goto fail; + } + +#if WASM_ENABLE_MULTI_MODULE != 0 + if (!wasm_runtime_is_built_in_module(sub_module_name)) { + sub_module = (WASMModule *)wasm_runtime_load_depended_module( + (WASMModuleCommon *)parent_module, sub_module_name, error_buf, + error_buf_size); + if (sub_module) { + /* wasm_loader_resolve_tag checks, that the imported tag + * and the declared tag have the same type + */ + uint32 linked_tag_index = 0; + WASMTag *linked_tag = wasm_loader_resolve_tag( + sub_module_name, tag_name, declare_tag_type, + &linked_tag_index /* out */, error_buf, error_buf_size); + if (linked_tag) { + tag->import_module = sub_module; + tag->import_tag_linked = linked_tag; + tag->import_tag_index_linked = linked_tag_index; + } + } + } +#endif + /* store to module tag declarations */ + tag->attribute = declare_tag_attribute; + tag->type = declare_type_index; + + tag->module_name = (char *)sub_module_name; + tag->field_name = (char *)tag_name; + tag->tag_type = declare_tag_type; + + *p_buf = p; + (void)parent_module; + + LOG_VERBOSE("Load tag import success\n"); + + return true; +fail: + return false; +} +#endif /* end of WASM_ENABLE_TAGS != 0 */ + +static bool +load_global_import(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *parent_module, char *sub_module_name, + char *global_name, WASMGlobalImport *global, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint8 declare_type = 0; + uint8 declare_mutable = 0; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *sub_module = NULL; + WASMGlobal *linked_global = NULL; +#endif +#if WASM_ENABLE_GC != 0 + WASMRefType ref_type; + bool need_ref_type_map; +#endif + bool ret = false; + +#if WASM_ENABLE_GC == 0 + CHECK_BUF(p, p_end, 2); + /* global type */ + declare_type = read_uint8(p); + if (!is_valid_value_type_for_interpreter(declare_type)) { + set_error_buf(error_buf, error_buf_size, "type mismatch"); + return false; + } + declare_mutable = read_uint8(p); +#else + if (!resolve_value_type(&p, p_end, parent_module, parent_module->type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { + return false; + } + declare_type = ref_type.ref_type; + if (need_ref_type_map) { + if (!(global->ref_type = + reftype_set_insert(parent_module->ref_type_set, &ref_type, + error_buf, error_buf_size))) { + return false; + } + } +#if TRACE_WASM_LOADER != 0 + os_printf("import global type: "); + wasm_dump_value_type(declare_type, global->ref_type); + os_printf("\n"); +#endif + CHECK_BUF(p, p_end, 1); + declare_mutable = read_uint8(p); +#endif /* end of WASM_ENABLE_GC == 0 */ + + *p_buf = p; + + if (!check_mutability(declare_mutable, error_buf, error_buf_size)) { + return false; + } + +#if WASM_ENABLE_LIBC_BUILTIN != 0 + ret = wasm_native_lookup_libc_builtin_global(sub_module_name, global_name, + global); + if (ret) { + if (global->type.val_type != declare_type + || global->type.is_mutable != declare_mutable) { + set_error_buf(error_buf, error_buf_size, + "incompatible import type"); + return false; + } + global->is_linked = true; + } +#endif +#if WASM_ENABLE_MULTI_MODULE != 0 + if (!global->is_linked + && !wasm_runtime_is_built_in_module(sub_module_name)) { + sub_module = (WASMModule *)wasm_runtime_load_depended_module( + (WASMModuleCommon *)parent_module, sub_module_name, error_buf, + error_buf_size); + if (sub_module) { + /* check sub modules */ + linked_global = wasm_loader_resolve_global( + sub_module_name, global_name, declare_type, declare_mutable, + error_buf, error_buf_size); + if (linked_global) { + global->import_module = sub_module; + global->import_global_linked = linked_global; + global->is_linked = true; + } + } + } +#endif + + global->module_name = sub_module_name; + global->field_name = global_name; + global->type.val_type = declare_type; + global->type.is_mutable = (declare_mutable == 1); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (global->type.val_type == VALUE_TYPE_V128) + parent_module->is_simd_used = true; + else if (global->type.val_type == VALUE_TYPE_EXTERNREF) + parent_module->is_ref_types_used = true; +#endif + (void)parent_module; + (void)ret; + return true; +fail: + return false; +} + +static bool +load_table(const uint8 **p_buf, const uint8 *buf_end, WASMModule *module, + WASMTable *table, char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; +#if WASM_ENABLE_GC != 0 + WASMRefType ref_type; + bool need_ref_type_map; +#endif + bool is_table64 = false; + +#if WASM_ENABLE_GC == 0 + CHECK_BUF(p, p_end, 1); + /* 0x70 or 0x6F */ + table->table_type.elem_type = read_uint8(p); + if (VALUE_TYPE_FUNCREF != table->table_type.elem_type +#if WASM_ENABLE_REF_TYPES != 0 + && VALUE_TYPE_EXTERNREF != table->table_type.elem_type +#endif + ) { + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return false; + } +#else /* else of WASM_ENABLE_GC == 0 */ + if (!resolve_value_type(&p, p_end, module, module->type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { + return false; + } + /* + * TODO: add this validator + * `wasm_is_reftype_htref_non_nullable(ref_type.ref_type)` + * after sync up with the latest GC spec + */ + if (!wasm_is_type_reftype(ref_type.ref_type)) { + set_error_buf(error_buf, error_buf_size, "type mismatch"); + return false; + } + table->table_type.elem_type = ref_type.ref_type; + if (need_ref_type_map) { + if (!(table->table_type.elem_ref_type = + reftype_set_insert(module->ref_type_set, &ref_type, error_buf, + error_buf_size))) { + return false; + } + } +#if TRACE_WASM_LOADER != 0 + os_printf("table type: "); + wasm_dump_value_type(table->table_type.elem_type, + table->table_type.elem_ref_type); + os_printf("\n"); +#endif +#endif /* end of WASM_ENABLE_GC == 0 */ + + p_org = p; + read_leb_uint32(p, p_end, table->table_type.flags); + is_table64 = table->table_type.flags & TABLE64_FLAG; + if (p - p_org > 1) { + LOG_VERBOSE("integer representation too long(table)"); + set_error_buf(error_buf, error_buf_size, "invalid limits flags"); + return false; + } + + if (!wasm_table_check_flags(table->table_type.flags, error_buf, + error_buf_size, false)) { + return false; + } + + read_leb_uint32(p, p_end, table->table_type.init_size); + if (table->table_type.flags & MAX_TABLE_SIZE_FLAG) { + read_leb_uint32(p, p_end, table->table_type.max_size); + if (!check_table_max_size(table->table_type.init_size, + table->table_type.max_size, error_buf, + error_buf_size)) + return false; + } + + adjust_table_max_size(is_table64, table->table_type.init_size, + table->table_type.flags & MAX_TABLE_SIZE_FLAG, + &table->table_type.max_size); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (table->table_type.elem_type == VALUE_TYPE_EXTERNREF) + module->is_ref_types_used = true; +#endif + + *p_buf = p; + return true; +fail: + return false; +} + +static bool +load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; +#if WASM_ENABLE_APP_FRAMEWORK != 0 + uint32 pool_size = wasm_runtime_memory_pool_size(); + uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT + / DEFAULT_NUM_BYTES_PER_PAGE; +#else + uint32 max_page_count; +#endif + bool is_memory64 = false; + + p_org = p; + read_leb_uint32(p, p_end, memory->flags); + is_memory64 = memory->flags & MEMORY64_FLAG; + if (p - p_org > 1) { + LOG_VERBOSE("integer representation too long(memory)"); + set_error_buf(error_buf, error_buf_size, "invalid limits flags"); + return false; + } + + if (!wasm_memory_check_flags(memory->flags, error_buf, error_buf_size, + false)) { + return false; + } + + read_leb_uint32(p, p_end, memory->init_page_count); + if (!check_memory_init_size(is_memory64, memory->init_page_count, error_buf, + error_buf_size)) + return false; + +#if WASM_ENABLE_APP_FRAMEWORK == 0 + max_page_count = is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; +#endif + if (memory->flags & 1) { + read_leb_uint32(p, p_end, memory->max_page_count); + if (!check_memory_max_size(is_memory64, memory->init_page_count, + memory->max_page_count, error_buf, + error_buf_size)) + return false; + if (memory->max_page_count > max_page_count) + memory->max_page_count = max_page_count; + } + else { + /* Limit the maximum memory size to max_page_count */ + memory->max_page_count = max_page_count; + } + + memory->num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE; + + *p_buf = p; + return true; +fail: + return false; +} + +static int +cmp_export_name(const void *a, const void *b) +{ + return strcmp(*(char **)a, *(char **)b); +} + +static bool +load_import_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + bool is_load_from_file_buf, bool no_resolve, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end, *p_old; + uint32 import_count, name_len, type_index, i, u32, flags; + uint64 total_size; + WASMImport *import; + WASMImport *import_functions = NULL, *import_tables = NULL; + WASMImport *import_memories = NULL, *import_globals = NULL; +#if WASM_ENABLE_TAGS != 0 + WASMImport *import_tags = NULL; +#endif + char *sub_module_name, *field_name; + uint8 u8, kind, global_type; + + read_leb_uint32(p, p_end, import_count); + + if (import_count) { + module->import_count = import_count; + total_size = sizeof(WASMImport) * (uint64)import_count; + if (!(module->imports = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + p_old = p; + + /* Scan firstly to get import count of each type */ + for (i = 0; i < import_count; i++) { + /* module name */ + read_leb_uint32(p, p_end, name_len); + CHECK_BUF(p, p_end, name_len); + p += name_len; + + /* field name */ + read_leb_uint32(p, p_end, name_len); + CHECK_BUF(p, p_end, name_len); + p += name_len; + + CHECK_BUF(p, p_end, 1); + /* 0x00/0x01/0x02/0x03/0x04 */ + kind = read_uint8(p); + + switch (kind) { + case IMPORT_KIND_FUNC: /* import function */ + read_leb_uint32(p, p_end, type_index); + module->import_function_count++; + break; + + case IMPORT_KIND_TABLE: /* import table */ + CHECK_BUF(p, p_end, 1); + /* 0x70 */ + u8 = read_uint8(p); +#if WASM_ENABLE_GC != 0 + if (wasm_is_reftype_htref_nullable(u8)) { + int32 heap_type; + read_leb_int32(p, p_end, heap_type); + (void)heap_type; + } +#endif + read_leb_uint32(p, p_end, flags); + read_leb_uint32(p, p_end, u32); + if (flags & 1) + read_leb_uint32(p, p_end, u32); + module->import_table_count++; + + if (module->import_table_count > 1) { +#if WASM_ENABLE_REF_TYPES == 0 && WASM_ENABLE_GC == 0 + set_error_buf(error_buf, error_buf_size, + "multiple tables"); + return false; +#elif WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + } + break; + + case IMPORT_KIND_MEMORY: /* import memory */ + read_leb_uint32(p, p_end, flags); + read_leb_uint32(p, p_end, u32); + if (flags & 1) + read_leb_uint32(p, p_end, u32); + module->import_memory_count++; +#if WASM_ENABLE_MULTI_MEMORY == 0 + if (module->import_memory_count > 1) { + set_error_buf(error_buf, error_buf_size, + "multiple memories"); + return false; + } +#endif + break; + +#if WASM_ENABLE_TAGS != 0 + case IMPORT_KIND_TAG: /* import tags */ + /* it only counts the number of tags to import */ + module->import_tag_count++; + CHECK_BUF(p, p_end, 1); + u8 = read_uint8(p); + read_leb_uint32(p, p_end, type_index); + break; +#endif + + case IMPORT_KIND_GLOBAL: /* import global */ +#if WASM_ENABLE_GC != 0 + /* valtype */ + CHECK_BUF(p, p_end, 1); + global_type = read_uint8(p); + if (wasm_is_reftype_htref_nullable(global_type) + || wasm_is_reftype_htref_non_nullable(global_type)) { + int32 heap_type; + read_leb_int32(p, p_end, heap_type); + (void)heap_type; + } + + /* mutability */ + CHECK_BUF(p, p_end, 1); + p += 1; +#else + CHECK_BUF(p, p_end, 2); + p += 2; +#endif + + (void)global_type; + module->import_global_count++; + break; + + default: + set_error_buf(error_buf, error_buf_size, + "invalid import kind"); + return false; + } + } + + if (module->import_function_count) + import_functions = module->import_functions = module->imports; + if (module->import_table_count) + import_tables = module->import_tables = + module->imports + module->import_function_count; + if (module->import_memory_count) + import_memories = module->import_memories = + module->imports + module->import_function_count + + module->import_table_count; + +#if WASM_ENABLE_TAGS != 0 + if (module->import_tag_count) + import_tags = module->import_tags = + module->imports + module->import_function_count + + module->import_table_count + module->import_memory_count; + if (module->import_global_count) + import_globals = module->import_globals = + module->imports + module->import_function_count + + module->import_table_count + module->import_memory_count + + module->import_tag_count; +#else + if (module->import_global_count) + import_globals = module->import_globals = + module->imports + module->import_function_count + + module->import_table_count + module->import_memory_count; +#endif + + p = p_old; + + /* Scan again to resolve the data */ + for (i = 0; i < import_count; i++) { + /* load module name */ + read_leb_uint32(p, p_end, name_len); + CHECK_BUF(p, p_end, name_len); + if (!(sub_module_name = wasm_const_str_list_insert( + p, name_len, module, is_load_from_file_buf, error_buf, + error_buf_size))) { + return false; + } + p += name_len; + + /* load field name */ + read_leb_uint32(p, p_end, name_len); + CHECK_BUF(p, p_end, name_len); + if (!(field_name = wasm_const_str_list_insert( + p, name_len, module, is_load_from_file_buf, error_buf, + error_buf_size))) { + return false; + } + p += name_len; + + CHECK_BUF(p, p_end, 1); + /* 0x00/0x01/0x02/0x03/0x4 */ + kind = read_uint8(p); + + switch (kind) { + case IMPORT_KIND_FUNC: /* import function */ + bh_assert(import_functions); + import = import_functions++; + if (!load_function_import(&p, p_end, module, + sub_module_name, field_name, + &import->u.function, no_resolve, + error_buf, error_buf_size)) { + return false; + } + break; + + case IMPORT_KIND_TABLE: /* import table */ + bh_assert(import_tables); + import = import_tables++; + if (!load_table_import(&p, p_end, module, sub_module_name, + field_name, &import->u.table, + error_buf, error_buf_size)) { + LOG_DEBUG("can not import such a table (%s,%s)", + sub_module_name, field_name); + return false; + } + break; + + case IMPORT_KIND_MEMORY: /* import memory */ + bh_assert(import_memories); + import = import_memories++; + if (!load_memory_import(&p, p_end, module, sub_module_name, + field_name, &import->u.memory, + error_buf, error_buf_size)) { + return false; + } + break; + +#if WASM_ENABLE_TAGS != 0 + case IMPORT_KIND_TAG: + bh_assert(import_tags); + import = import_tags++; + if (!load_tag_import(&p, p_end, module, sub_module_name, + field_name, &import->u.tag, error_buf, + error_buf_size)) { + return false; + } + break; +#endif + + case IMPORT_KIND_GLOBAL: /* import global */ + bh_assert(import_globals); + import = import_globals++; + if (!load_global_import(&p, p_end, module, sub_module_name, + field_name, &import->u.global, + error_buf, error_buf_size)) { + return false; + } + break; + + default: + set_error_buf(error_buf, error_buf_size, + "invalid import kind"); + return false; + } + import->kind = kind; + import->u.names.module_name = sub_module_name; + import->u.names.field_name = field_name; + } + +#if WASM_ENABLE_LIBC_WASI != 0 + import = module->import_functions; + for (i = 0; i < module->import_function_count; i++, import++) { + if (!strcmp(import->u.names.module_name, "wasi_unstable") + || !strcmp(import->u.names.module_name, + "wasi_snapshot_preview1")) { + module->import_wasi_api = true; + break; + } + } +#endif + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load import section success.\n"); + (void)u8; + (void)u32; + (void)type_index; + return true; +fail: + return false; +} + +static bool +init_function_local_offsets(WASMFunction *func, char *error_buf, + uint32 error_buf_size) +{ + WASMFuncType *param_type = func->func_type; + uint32 param_count = param_type->param_count; + uint8 *param_types = param_type->types; + uint32 local_count = func->local_count; + uint8 *local_types = func->local_types; + uint32 i, local_offset = 0; + uint64 total_size = sizeof(uint16) * ((uint64)param_count + local_count); + + /* + * Only allocate memory when total_size is not 0, + * or the return value of malloc(0) might be NULL on some platforms, + * which causes wasm loader return false. + */ + if (total_size > 0 + && !(func->local_offsets = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < param_count; i++) { + func->local_offsets[i] = (uint16)local_offset; + local_offset += wasm_value_type_cell_num(param_types[i]); + } + + for (i = 0; i < local_count; i++) { + func->local_offsets[param_count + i] = (uint16)local_offset; + local_offset += wasm_value_type_cell_num(local_types[i]); + } + + bh_assert(local_offset == func->param_cell_num + func->local_cell_num); + return true; +} + +static bool +load_function_section(const uint8 *buf, const uint8 *buf_end, + const uint8 *buf_code, const uint8 *buf_code_end, + WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + const uint8 *p_code = buf_code, *p_code_end, *p_code_save; + uint32 func_count; + uint64 total_size; + uint32 code_count = 0, code_size, type_index, i, j, k, local_type_index; + uint32 local_count, local_set_count, sub_local_count, local_cell_num; + uint8 type; + WASMFunction *func; +#if WASM_ENABLE_GC != 0 + bool need_ref_type_map; + WASMRefType ref_type; + uint32 ref_type_map_count = 0, t = 0, type_index_org; +#endif + + read_leb_uint32(p, p_end, func_count); + + if (buf_code) + read_leb_uint32(p_code, buf_code_end, code_count); + + if (func_count != code_count) { + set_error_buf(error_buf, error_buf_size, + "function and code section have inconsistent lengths or " + "unexpected end"); + return false; + } + + if (is_indices_overflow(module->import_function_count, func_count, + error_buf, error_buf_size)) + return false; + + if (func_count) { + module->function_count = func_count; + total_size = sizeof(WASMFunction *) * (uint64)func_count; + if (!(module->functions = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < func_count; i++) { + /* Resolve function type */ + read_leb_uint32(p, p_end, type_index); + + if (!check_function_type(module, type_index, error_buf, + error_buf_size)) { + return false; + } + +#if WASM_ENABLE_GC != 0 + type_index_org = type_index; +#endif + +#if (WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0) \ + && WASM_ENABLE_GC == 0 + type_index = wasm_get_smallest_type_idx( + module->types, module->type_count, type_index); +#endif + + read_leb_uint32(p_code, buf_code_end, code_size); + if (code_size == 0 || p_code + code_size > buf_code_end) { + set_error_buf(error_buf, error_buf_size, + "invalid function code size"); + return false; + } + + /* Resolve local set count */ + p_code_end = p_code + code_size; +#if WASM_ENABLE_BRANCH_HINTS != 0 + uint8 *p_body_start = (uint8 *)p_code; +#endif + local_count = 0; + read_leb_uint32(p_code, buf_code_end, local_set_count); + p_code_save = p_code; + +#if WASM_ENABLE_GC != 0 + ref_type_map_count = 0; +#endif + + /* Calculate total local count */ + for (j = 0; j < local_set_count; j++) { + read_leb_uint32(p_code, buf_code_end, sub_local_count); + if (sub_local_count > UINT32_MAX - local_count) { + set_error_buf(error_buf, error_buf_size, "too many locals"); + return false; + } +#if WASM_ENABLE_GC == 0 + CHECK_BUF(p_code, buf_code_end, 1); + /* 0x7F/0x7E/0x7D/0x7C */ + type = read_uint8(p_code); + local_count += sub_local_count; +#if WASM_ENABLE_WAMR_COMPILER != 0 + /* If any value's type is v128, mark the module as SIMD used */ + if (type == VALUE_TYPE_V128) + module->is_simd_used = true; +#endif +#else + if (!resolve_value_type(&p_code, buf_code_end, module, + module->type_count, &need_ref_type_map, + &ref_type, false, error_buf, + error_buf_size)) { + return false; + } + local_count += sub_local_count; + if (need_ref_type_map) + ref_type_map_count += sub_local_count; +#endif + } + + /* Code size in code entry can't be smaller than size of vec(locals) + * + expr(at least 1 for opcode end). And expressions are encoded by + * their instruction sequence terminated with an explicit 0x0B + * opcode for end. */ + if (p_code_end <= p_code || *(p_code_end - 1) != WASM_OP_END) { + set_error_buf( + error_buf, error_buf_size, + "section size mismatch: function body END opcode expected"); + return false; + } + + /* Alloc memory, layout: function structure + local types */ + code_size = (uint32)(p_code_end - p_code); + + total_size = sizeof(WASMFunction) + (uint64)local_count; + if (!(func = module->functions[i] = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } +#if WASM_ENABLE_GC != 0 + if (ref_type_map_count > 0) { + if (ref_type_map_count > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "ref type count too large"); + return false; + } + total_size = + sizeof(WASMRefTypeMap) * (uint64)ref_type_map_count; + if (!(func->local_ref_type_maps = loader_malloc( + total_size, error_buf, error_buf_size))) { + return false; + } + func->local_ref_type_map_count = ref_type_map_count; + } +#endif + + /* Set function type, local count, code size and code body */ + func->func_type = (WASMFuncType *)module->types[type_index]; + func->local_count = local_count; + if (local_count > 0) + func->local_types = (uint8 *)func + sizeof(WASMFunction); + func->code_size = code_size; +#if WASM_ENABLE_BRANCH_HINTS != 0 + func->code_body_begin = p_body_start; +#endif + /* + * we shall make a copy of code body [p_code, p_code + code_size] + * when we are worrying about inappropriate releasing behaviour. + * all code bodies are actually in a buffer which user allocates in + * their embedding environment and we don't have power over them. + * it will be like: + * code_body_cp = malloc(code_size); + * memcpy(code_body_cp, p_code, code_size); + * func->code = code_body_cp; + */ + func->code = (uint8 *)p_code; +#if WASM_ENABLE_GC != 0 + func->type_idx = type_index_org; +#endif + +#if WASM_ENABLE_GC != 0 + t = 0; +#endif + + /* Load each local type */ + p_code = p_code_save; + local_type_index = 0; + for (j = 0; j < local_set_count; j++) { + read_leb_uint32(p_code, buf_code_end, sub_local_count); + /* Note: sub_local_count is allowed to be 0 */ + if (local_type_index > UINT32_MAX - sub_local_count + || local_type_index + sub_local_count > local_count) { + set_error_buf(error_buf, error_buf_size, + "invalid local count"); + return false; + } +#if WASM_ENABLE_GC == 0 + CHECK_BUF(p_code, buf_code_end, 1); + /* 0x7F/0x7E/0x7D/0x7C */ + type = read_uint8(p_code); + if (!is_valid_value_type_for_interpreter(type)) { + if (type == VALUE_TYPE_V128) + set_error_buf(error_buf, error_buf_size, + "v128 value type requires simd feature"); + else if (type == VALUE_TYPE_FUNCREF + || type == VALUE_TYPE_EXTERNREF) + set_error_buf(error_buf, error_buf_size, + "ref value type requires " + "reference types feature"); + else + set_error_buf_v(error_buf, error_buf_size, + "invalid local type 0x%02X", type); + return false; + } +#else + if (!resolve_value_type(&p_code, buf_code_end, module, + module->type_count, &need_ref_type_map, + &ref_type, false, error_buf, + error_buf_size)) { + return false; + } + if (need_ref_type_map) { + WASMRefType *ref_type_tmp; + if (!(ref_type_tmp = reftype_set_insert( + module->ref_type_set, &ref_type, error_buf, + error_buf_size))) { + return false; + } + for (k = 0; k < sub_local_count; k++) { + func->local_ref_type_maps[t + k].ref_type = + ref_type_tmp; + func->local_ref_type_maps[t + k].index = + local_type_index + k; + } + t += sub_local_count; + } + type = ref_type.ref_type; +#endif + for (k = 0; k < sub_local_count; k++) { + func->local_types[local_type_index++] = type; + } +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (type == VALUE_TYPE_V128) + module->is_simd_used = true; + else if (type == VALUE_TYPE_FUNCREF + || type == VALUE_TYPE_EXTERNREF) + module->is_ref_types_used = true; +#endif + } + + bh_assert(local_type_index == func->local_count); +#if WASM_ENABLE_GC != 0 + bh_assert(t == func->local_ref_type_map_count); +#if TRACE_WASM_LOADER != 0 + os_printf("func %u, local types: [", i); + k = 0; + for (j = 0; j < func->local_count; j++) { + WASMRefType *ref_type_tmp = NULL; + if (wasm_is_type_multi_byte_type(func->local_types[j])) { + bh_assert(j == func->local_ref_type_maps[k].index); + ref_type_tmp = func->local_ref_type_maps[k++].ref_type; + } + wasm_dump_value_type(func->local_types[j], ref_type_tmp); + if (j < func->local_count - 1) + os_printf(" "); + } + os_printf("]\n"); +#endif +#endif + + func->param_cell_num = func->func_type->param_cell_num; + func->ret_cell_num = func->func_type->ret_cell_num; + local_cell_num = + wasm_get_cell_num(func->local_types, func->local_count); + + if (local_cell_num > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "local count too large"); + return false; + } + + func->local_cell_num = (uint16)local_cell_num; + + if (!init_function_local_offsets(func, error_buf, error_buf_size)) + return false; + + p_code = p_code_end; + } + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load function section success.\n"); + return true; +fail: + return false; +} + +static bool +load_table_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 table_count, i; + uint64 total_size; + WASMTable *table; + + read_leb_uint32(p, p_end, table_count); + if (module->import_table_count + table_count > 1) { +#if WASM_ENABLE_REF_TYPES == 0 && WASM_ENABLE_GC == 0 + /* a total of one table is allowed */ + set_error_buf(error_buf, error_buf_size, "multiple tables"); + return false; +#elif WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + } + + if (table_count) { + module->table_count = table_count; + total_size = sizeof(WASMTable) * (uint64)table_count; + if (!(module->tables = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* load each table */ + table = module->tables; + for (i = 0; i < table_count; i++, table++) { +#if WASM_ENABLE_GC != 0 + uint8 flag; + bool has_init = false; + + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + + if (flag == TABLE_INIT_EXPR_FLAG) { + CHECK_BUF(p, p_end, 1); + flag = read_uint8(p); + + if (flag != 0x00) { + set_error_buf(error_buf, error_buf_size, + "invalid leading bytes for table"); + return false; + } + has_init = true; + } + else { + p--; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + + if (!load_table(&p, p_end, module, table, error_buf, + error_buf_size)) + return false; + +#if WASM_ENABLE_GC != 0 + if (has_init) { + if (!load_init_expr(module, &p, p_end, &table->init_expr, + table->table_type.elem_type, + table->table_type.elem_ref_type, error_buf, + error_buf_size)) + return false; + if (table->init_expr.init_expr_type >= INIT_EXPR_TYPE_STRUCT_NEW + && table->init_expr.init_expr_type + <= INIT_EXPR_TYPE_ARRAY_NEW_FIXED) { + set_error_buf( + error_buf, error_buf_size, + "unsupported initializer expression for table"); + return false; + } + } + else { + if (wasm_is_reftype_htref_non_nullable( + table->table_type.elem_type)) { + set_error_buf( + error_buf, error_buf_size, + "type mismatch: non-nullable table without init expr"); + return false; + } + } +#endif /* end of WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (table->table_type.elem_type == VALUE_TYPE_EXTERNREF) + module->is_ref_types_used = true; +#endif + } + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load table section success.\n"); + return true; +fail: + return false; +} + +static bool +load_memory_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 memory_count, i; + uint64 total_size; + WASMMemory *memory; + + read_leb_uint32(p, p_end, memory_count); + +#if WASM_ENABLE_MULTI_MEMORY == 0 + /* a total of one memory is allowed */ + if (module->import_memory_count + memory_count > 1) { + set_error_buf(error_buf, error_buf_size, "multiple memories"); + return false; + } +#endif + + if (memory_count) { + module->memory_count = memory_count; + total_size = sizeof(WASMMemory) * (uint64)memory_count; + if (!(module->memories = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* load each memory */ + memory = module->memories; + for (i = 0; i < memory_count; i++, memory++) + if (!load_memory(&p, p_end, memory, error_buf, error_buf_size)) + return false; + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load memory section success.\n"); + return true; +fail: + return false; +} + +static bool +load_global_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 global_count, i; + uint64 total_size; + WASMGlobal *global; + uint8 mutable; +#if WASM_ENABLE_GC != 0 + bool need_ref_type_map; + WASMRefType ref_type; +#endif + + read_leb_uint32(p, p_end, global_count); + if (is_indices_overflow(module->import_global_count, global_count, + error_buf, error_buf_size)) + return false; + + module->global_count = 0; + if (global_count) { + total_size = sizeof(WASMGlobal) * (uint64)global_count; + if (!(module->globals = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + global = module->globals; + + for (i = 0; i < global_count; i++, global++) { +#if WASM_ENABLE_GC == 0 + CHECK_BUF(p, p_end, 2); + /* global type */ + global->type.val_type = read_uint8(p); + if (!is_valid_value_type_for_interpreter(global->type.val_type)) { + set_error_buf(error_buf, error_buf_size, "type mismatch"); + return false; + } + mutable = read_uint8(p); +#else + if (!resolve_value_type(&p, p_end, module, module->type_count, + &need_ref_type_map, &ref_type, false, + error_buf, error_buf_size)) { + return false; + } + global->type.val_type = ref_type.ref_type; + CHECK_BUF(p, p_end, 1); + mutable = read_uint8(p); +#endif /* end of WASM_ENABLE_GC */ + +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (global->type.val_type == VALUE_TYPE_V128) + module->is_simd_used = true; + else if (global->type.val_type == VALUE_TYPE_FUNCREF + || global->type.val_type == VALUE_TYPE_EXTERNREF) + module->is_ref_types_used = true; +#endif + + if (!check_mutability(mutable, error_buf, error_buf_size)) { + return false; + } + global->type.is_mutable = mutable ? true : false; + + /* initialize expression */ + if (!load_init_expr(module, &p, p_end, &(global->init_expr), + global->type.val_type, +#if WASM_ENABLE_GC == 0 + NULL, +#else + &ref_type, +#endif + error_buf, error_buf_size)) + return false; + +#if WASM_ENABLE_GC != 0 + if (global->init_expr.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { + uint8 global_type; + WASMRefType *global_ref_type; + uint32 global_idx = global->init_expr.u.unary.v.global_index; + + if (global->init_expr.u.unary.v.global_index + >= module->import_global_count + i) { + set_error_buf(error_buf, error_buf_size, "unknown global"); + return false; + } + + if (global_idx < module->import_global_count) { + global_type = module->import_globals[global_idx] + .u.global.type.val_type; + global_ref_type = + module->import_globals[global_idx].u.global.ref_type; + } + else { + global_type = + module + ->globals[global_idx - module->import_global_count] + .type.val_type; + global_ref_type = + module + ->globals[global_idx - module->import_global_count] + .ref_type; + } + if (!wasm_reftype_is_subtype_of( + global_type, global_ref_type, global->type.val_type, + global->ref_type, module->types, module->type_count)) { + set_error_buf(error_buf, error_buf_size, "type mismatch"); + return false; + } + } + + if (need_ref_type_map) { + if (!(global->ref_type = + reftype_set_insert(module->ref_type_set, &ref_type, + error_buf, error_buf_size))) { + return false; + } + } +#if TRACE_WASM_LOADER != 0 + os_printf("global type: "); + wasm_dump_value_type(global->type, global->ref_type); + os_printf("\n"); +#endif +#endif + module->global_count++; + } + bh_assert(module->global_count == global_count); + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load global section success.\n"); + return true; +fail: + return false; +} + +static bool +check_duplicate_exports(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + uint32 i; + bool result = false; + char *names_buf[32], **names = names_buf; + + if (module->export_count > 32) { + names = loader_malloc(module->export_count * sizeof(char *), error_buf, + error_buf_size); + if (!names) { + return result; + } + } + + for (i = 0; i < module->export_count; i++) { + names[i] = module->exports[i].name; + } + + qsort(names, module->export_count, sizeof(char *), cmp_export_name); + + for (i = 1; i < module->export_count; i++) { + if (!strcmp(names[i], names[i - 1])) { + set_error_buf(error_buf, error_buf_size, "duplicate export name"); + goto cleanup; + } + } + + result = true; +cleanup: + if (module->export_count > 32) { + wasm_runtime_free(names); + } + return result; +} + +static bool +load_export_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 export_count, i, index; + uint64 total_size; + uint32 str_len; + WASMExport *export; + + read_leb_uint32(p, p_end, export_count); + + if (export_count) { + module->export_count = export_count; + total_size = sizeof(WASMExport) * (uint64)export_count; + if (!(module->exports = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + export = module->exports; + for (i = 0; i < export_count; i++, export ++) { +#if WASM_ENABLE_THREAD_MGR == 0 + if (p == p_end) { + /* export section with inconsistent count: + n export declared, but less than n given */ + set_error_buf(error_buf, error_buf_size, + "length out of bounds"); + return false; + } +#endif + read_leb_uint32(p, p_end, str_len); + CHECK_BUF(p, p_end, str_len); + + if (!(export->name = wasm_const_str_list_insert( + p, str_len, module, is_load_from_file_buf, error_buf, + error_buf_size))) { + return false; + } + + p += str_len; + CHECK_BUF(p, p_end, 1); + export->kind = read_uint8(p); + read_leb_uint32(p, p_end, index); + export->index = index; + + switch (export->kind) { + /* function index */ + case EXPORT_KIND_FUNC: + if (index >= module->function_count + + module->import_function_count) { + set_error_buf(error_buf, error_buf_size, + "unknown function"); + return false; + } +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) \ + || (WASM_ENABLE_FAST_INTERP != 0) + /* TODO: check func type, if it has v128 param or result, + report error */ +#endif +#endif + break; + /* table index */ + case EXPORT_KIND_TABLE: + if (index + >= module->table_count + module->import_table_count) { + set_error_buf(error_buf, error_buf_size, + "unknown table"); + return false; + } + break; + /* memory index */ + case EXPORT_KIND_MEMORY: + if (index + >= module->memory_count + module->import_memory_count) { + set_error_buf(error_buf, error_buf_size, + "unknown memory"); + return false; + } + break; +#if WASM_ENABLE_TAGS != 0 + /* export tag */ + case EXPORT_KIND_TAG: + if (index >= module->tag_count + module->import_tag_count) { + set_error_buf(error_buf, error_buf_size, "unknown tag"); + return false; + } + break; +#endif + + /* global index */ + case EXPORT_KIND_GLOBAL: + if (index + >= module->global_count + module->import_global_count) { + set_error_buf(error_buf, error_buf_size, + "unknown global"); + return false; + } + break; + + default: + set_error_buf(error_buf, error_buf_size, + "invalid export kind"); + return false; + } + } + + if (!check_duplicate_exports(module, error_buf, error_buf_size)) { + return false; + } + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load export section success.\n"); + return true; +fail: + return false; +} + +static bool +check_table_index(const WASMModule *module, uint32 table_index, char *error_buf, + uint32 error_buf_size) +{ +#if WASM_ENABLE_REF_TYPES == 0 && WASM_ENABLE_GC == 0 + if (table_index != 0) { + set_error_buf( + error_buf, error_buf_size, + "zero byte expected. The module uses reference types feature " + "which is disabled in the runtime."); + return false; + } +#endif + + if (table_index >= module->import_table_count + module->table_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown table %d", + table_index); + return false; + } + return true; +} + +static bool +load_table_index(const uint8 **p_buf, const uint8 *buf_end, WASMModule *module, + uint32 *p_table_index, char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint32 table_index; + + read_leb_uint32(p, p_end, table_index); + if (!check_table_index(module, table_index, error_buf, error_buf_size)) { + return false; + } + + *p_table_index = table_index; + *p_buf = p; + return true; +fail: + return false; +} + +/* Element segments must match element type of table */ +static bool +check_table_elem_type(WASMModule *module, uint32 table_index, + uint32 type_from_elem_seg, char *error_buf, + uint32 error_buf_size) +{ + uint32 table_declared_elem_type; + + if (table_index < module->import_table_count) + table_declared_elem_type = + module->import_tables[table_index].u.table.table_type.elem_type; + else + table_declared_elem_type = + module->tables[table_index - module->import_table_count] + .table_type.elem_type; + + if (table_declared_elem_type == type_from_elem_seg) + return true; + +#if WASM_ENABLE_GC != 0 + /* + * balance in: anyref, funcref, (ref.null func) and (ref.func) + */ + if (table_declared_elem_type == REF_TYPE_ANYREF) + return true; + + if (table_declared_elem_type == VALUE_TYPE_FUNCREF + && type_from_elem_seg == REF_TYPE_HT_NON_NULLABLE) + return true; + + if (table_declared_elem_type == REF_TYPE_HT_NULLABLE + && type_from_elem_seg == REF_TYPE_HT_NON_NULLABLE) + return true; +#endif + + set_error_buf(error_buf, error_buf_size, "type mismatch"); + return false; +} + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 +static bool +load_elem_type(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, + uint32 *p_elem_type, +#if WASM_ENABLE_GC != 0 + WASMRefType **p_elem_ref_type, +#endif + bool elemkind_zero, char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint8 elem_type; +#if WASM_ENABLE_GC != 0 + WASMRefType elem_ref_type; + bool need_ref_type_map; +#endif + + CHECK_BUF(p, p_end, 1); + elem_type = read_uint8(p); + if (elemkind_zero) { + if (elem_type != 0) { + set_error_buf(error_buf, error_buf_size, + "invalid reference type or unknown type"); + return false; + } + else { + *p_elem_type = VALUE_TYPE_FUNCREF; + *p_buf = p; + return true; + } + } + +#if WASM_ENABLE_GC == 0 + if (elem_type != VALUE_TYPE_FUNCREF && elem_type != VALUE_TYPE_EXTERNREF) { + set_error_buf(error_buf, error_buf_size, + "invalid reference type or unknown type"); + return false; + } + *p_elem_type = elem_type; +#else + p--; + if (!resolve_value_type((const uint8 **)&p, p_end, module, + module->type_count, &need_ref_type_map, + &elem_ref_type, false, error_buf, error_buf_size)) { + return false; + } + if (!wasm_is_type_reftype(elem_ref_type.ref_type)) { + set_error_buf(error_buf, error_buf_size, + "invalid reference type or unknown type"); + return false; + } + *p_elem_type = elem_ref_type.ref_type; + if (need_ref_type_map) { + if (!(*p_elem_ref_type = + reftype_set_insert(module->ref_type_set, &elem_ref_type, + error_buf, error_buf_size))) { + return false; + } + } +#endif + + *p_buf = p; + return true; +fail: + return false; +} +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +static bool +load_func_index_vec(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *module, WASMTableSeg *table_segment, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint32 function_count, function_index = 0, i; + uint64 total_size; + + read_leb_uint32(p, p_end, function_count); + table_segment->value_count = function_count; + total_size = sizeof(InitializerExpression) * (uint64)function_count; + if (total_size > 0 + && !(table_segment->init_values = + (InitializerExpression *)loader_malloc(total_size, error_buf, + error_buf_size))) { + return false; + } + + for (i = 0; i < function_count; i++) { + InitializerExpression *init_expr = &table_segment->init_values[i]; + + read_leb_uint32(p, p_end, function_index); + if (!check_function_index(module, function_index, error_buf, + error_buf_size)) { + return false; + } + + init_expr->init_expr_type = INIT_EXPR_TYPE_FUNCREF_CONST; + init_expr->u.unary.v.ref_index = function_index; + } + + *p_buf = p; + return true; +fail: + return false; +} + +#if (WASM_ENABLE_GC != 0) || (WASM_ENABLE_REF_TYPES != 0) +static bool +load_init_expr_vec(const uint8 **p_buf, const uint8 *buf_end, + WASMModule *module, WASMTableSeg *table_segment, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = *p_buf, *p_end = buf_end; + uint32 ref_count, i; + uint64 total_size; + + read_leb_uint32(p, p_end, ref_count); + table_segment->value_count = ref_count; + total_size = sizeof(InitializerExpression) * (uint64)ref_count; + if (total_size > 0 + && !(table_segment->init_values = + (InitializerExpression *)loader_malloc(total_size, error_buf, + error_buf_size))) { + return false; + } + + for (i = 0; i < ref_count; i++) { + InitializerExpression *init_expr = &table_segment->init_values[i]; + + if (!load_init_expr(module, &p, p_end, init_expr, + table_segment->elem_type, +#if WASM_ENABLE_GC == 0 + NULL, +#else + table_segment->elem_ref_type, +#endif + error_buf, error_buf_size)) + return false; + + bh_assert((init_expr->init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) + || (init_expr->init_expr_type == INIT_EXPR_TYPE_REFNULL_CONST) + || (init_expr->init_expr_type >= INIT_EXPR_TYPE_FUNCREF_CONST + && init_expr->init_expr_type + <= INIT_EXPR_TYPE_ARRAY_NEW_FIXED)); + } + + *p_buf = p; + return true; +fail: + return false; +} +#endif /* end of (WASM_ENABLE_GC != 0) || (WASM_ENABLE_REF_TYPES != 0) */ + +static bool +load_table_segment_section(const uint8 *buf, const uint8 *buf_end, + WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint8 table_elem_idx_type; + uint32 table_segment_count, i; + uint64 total_size; + WASMTableSeg *table_segment; + + read_leb_uint32(p, p_end, table_segment_count); + + if (table_segment_count) { + module->table_seg_count = table_segment_count; + total_size = sizeof(WASMTableSeg) * (uint64)table_segment_count; + if (!(module->table_segments = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + table_segment = module->table_segments; + for (i = 0; i < table_segment_count; i++, table_segment++) { + if (p >= p_end) { + set_error_buf(error_buf, error_buf_size, + "invalid value type or " + "invalid elements segment kind"); + return false; + } + table_elem_idx_type = VALUE_TYPE_I32; + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + read_leb_uint32(p, p_end, table_segment->mode); + /* last three bits */ + table_segment->mode = table_segment->mode & 0x07; + switch (table_segment->mode) { + /* elemkind/elemtype + active */ + case 0: + case 4: + { +#if WASM_ENABLE_GC != 0 + if (table_segment->mode == 0) { + /* vec(funcidx), set elem type to (ref func) */ + WASMRefType elem_ref_type = { 0 }; + table_segment->elem_type = REF_TYPE_HT_NON_NULLABLE; + wasm_set_refheaptype_common( + &elem_ref_type.ref_ht_common, false, + HEAP_TYPE_FUNC); + if (!(table_segment->elem_ref_type = reftype_set_insert( + module->ref_type_set, &elem_ref_type, + error_buf, error_buf_size))) + return false; + } + else { + /* vec(expr), set elem type to funcref */ + table_segment->elem_type = VALUE_TYPE_FUNCREF; + } +#else + table_segment->elem_type = VALUE_TYPE_FUNCREF; +#endif + table_segment->table_index = 0; + + if (!check_table_index(module, table_segment->table_index, + error_buf, error_buf_size)) + return false; + +#if WASM_ENABLE_MEMORY64 != 0 + table_elem_idx_type = + is_table_64bit(module, table_segment->table_index) + ? VALUE_TYPE_I64 + : VALUE_TYPE_I32; +#endif + if (!load_init_expr(module, &p, p_end, + &table_segment->base_offset, + table_elem_idx_type, NULL, error_buf, + error_buf_size)) + return false; + + if (table_segment->mode == 0) { + /* vec(funcidx) */ + if (!load_func_index_vec(&p, p_end, module, + table_segment, error_buf, + error_buf_size)) + return false; + } + else { + /* vec(expr) */ + if (!load_init_expr_vec(&p, p_end, module, + table_segment, error_buf, + error_buf_size)) + return false; + } + + if (!check_table_elem_type(module, + table_segment->table_index, + table_segment->elem_type, + error_buf, error_buf_size)) + return false; + + break; + } + /* elemkind + passive/declarative */ + case 1: + case 3: + if (!load_elem_type(module, &p, p_end, + &table_segment->elem_type, +#if WASM_ENABLE_GC != 0 + &table_segment->elem_ref_type, +#endif + true, error_buf, error_buf_size)) + return false; + /* vec(funcidx) */ + if (!load_func_index_vec(&p, p_end, module, table_segment, + error_buf, error_buf_size)) + return false; + break; + /* elemkind/elemtype + table_idx + active */ + case 2: + case 6: + if (!load_table_index(&p, p_end, module, + &table_segment->table_index, + error_buf, error_buf_size)) + return false; +#if WASM_ENABLE_MEMORY64 != 0 + table_elem_idx_type = + is_table_64bit(module, table_segment->table_index) + ? VALUE_TYPE_I64 + : VALUE_TYPE_I32; +#endif + if (!load_init_expr(module, &p, p_end, + &table_segment->base_offset, + table_elem_idx_type, NULL, error_buf, + error_buf_size)) + return false; + if (!load_elem_type(module, &p, p_end, + &table_segment->elem_type, +#if WASM_ENABLE_GC != 0 + &table_segment->elem_ref_type, +#endif + table_segment->mode == 2 ? true : false, + error_buf, error_buf_size)) + return false; + + if (table_segment->mode == 2) { + /* vec(funcidx) */ + if (!load_func_index_vec(&p, p_end, module, + table_segment, error_buf, + error_buf_size)) + return false; + } + else { + /* vec(expr) */ + if (!load_init_expr_vec(&p, p_end, module, + table_segment, error_buf, + error_buf_size)) + return false; + } + + if (!check_table_elem_type(module, + table_segment->table_index, + table_segment->elem_type, + error_buf, error_buf_size)) + return false; + + break; + case 5: + case 7: + if (!load_elem_type(module, &p, p_end, + &table_segment->elem_type, +#if WASM_ENABLE_GC != 0 + &table_segment->elem_ref_type, +#endif + false, error_buf, error_buf_size)) + return false; + /* vec(expr) */ + if (!load_init_expr_vec(&p, p_end, module, table_segment, + error_buf, error_buf_size)) + return false; + break; + default: + set_error_buf(error_buf, error_buf_size, + "unknown element segment kind"); + return false; + } +#else /* else of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + /* + * like: 00 41 05 0b 04 00 01 00 01 + * for: (elem 0 (offset (i32.const 5)) $f1 $f2 $f1 $f2) + */ + if (!load_table_index(&p, p_end, module, + &table_segment->table_index, error_buf, + error_buf_size)) + return false; +#if WASM_ENABLE_MEMORY64 != 0 + table_elem_idx_type = + is_table_64bit(module, table_segment->table_index) + ? VALUE_TYPE_I64 + : VALUE_TYPE_I32; +#endif + if (!load_init_expr(module, &p, p_end, &table_segment->base_offset, + table_elem_idx_type, NULL, error_buf, + error_buf_size)) + return false; + if (!load_func_index_vec(&p, p_end, module, table_segment, + error_buf, error_buf_size)) + return false; + + table_segment->elem_type = VALUE_TYPE_FUNCREF; + + if (!check_table_elem_type(module, table_segment->table_index, + table_segment->elem_type, error_buf, + error_buf_size)) + return false; +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_MEMORY64 != 0 + if (table_elem_idx_type == VALUE_TYPE_I64 + && table_segment->base_offset.u.unary.v.u64 > UINT32_MAX) { + set_error_buf(error_buf, error_buf_size, + "In table64, table base offset can't be " + "larger than UINT32_MAX"); + return false; + } +#endif + +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (table_segment->elem_type == VALUE_TYPE_EXTERNREF) + module->is_ref_types_used = true; +#endif + } + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load table segment section success.\n"); + return true; +fail: + return false; +} + +#if WASM_ENABLE_BULK_MEMORY != 0 +static bool +check_data_count_consistency(bool has_datacount_section, int datacount_len, + int data_seg_len, char *error_buf, + uint32 error_buf_size) +{ + if (has_datacount_section && datacount_len != data_seg_len) { + set_error_buf(error_buf, error_buf_size, + "data count and data section have inconsistent lengths"); + return false; + } + return true; +} +#endif + +static bool +load_data_segment_section(const uint8 *buf, const uint8 *buf_end, + WASMModule *module, +#if WASM_ENABLE_BULK_MEMORY != 0 + bool has_datacount_section, +#endif + bool clone_data_seg, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 data_seg_count, i, mem_index, data_seg_len; + uint64 total_size; + WASMDataSeg *dataseg; + InitializerExpression init_expr; +#if WASM_ENABLE_BULK_MEMORY != 0 + bool is_passive = false; + uint32 mem_flag; +#endif + uint8 mem_offset_type = VALUE_TYPE_I32; + + read_leb_uint32(p, p_end, data_seg_count); + +#if WASM_ENABLE_BULK_MEMORY != 0 + if (!check_data_count_consistency(has_datacount_section, + module->data_seg_count1, data_seg_count, + error_buf, error_buf_size)) { + return false; + } +#endif + + if (data_seg_count) { + module->data_seg_count = data_seg_count; + total_size = sizeof(WASMDataSeg *) * (uint64)data_seg_count; + if (!(module->data_segments = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < data_seg_count; i++) { + read_leb_uint32(p, p_end, mem_index); +#if WASM_ENABLE_BULK_MEMORY != 0 + is_passive = false; + mem_flag = mem_index & 0x03; + switch (mem_flag) { + case 0x01: + is_passive = true; +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_bulk_memory_used = true; +#endif + break; + case 0x00: + /* no memory index, treat index as 0 */ + mem_index = 0; + goto check_mem_index; + case 0x02: + /* read following memory index */ + read_leb_uint32(p, p_end, mem_index); +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_bulk_memory_used = true; +#endif + check_mem_index: + if (mem_index + >= module->import_memory_count + module->memory_count) { + set_error_buf_v(error_buf, error_buf_size, + "unknown memory %d", mem_index); + return false; + } + break; + case 0x03: + default: + set_error_buf(error_buf, error_buf_size, "unknown memory"); + return false; + break; + } +#else + if (mem_index + >= module->import_memory_count + module->memory_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown memory %d", + mem_index); + return false; + } +#endif /* WASM_ENABLE_BULK_MEMORY */ + +#if WASM_ENABLE_BULK_MEMORY != 0 + if (!is_passive) +#endif + { +#if WASM_ENABLE_MEMORY64 != 0 + /* This memory_flag is from memory instead of data segment */ + uint8 memory_flag; + if (module->import_memory_count > 0) { + memory_flag = module->import_memories[mem_index] + .u.memory.mem_type.flags; + } + else { + memory_flag = + module + ->memories[mem_index - module->import_memory_count] + .flags; + } + mem_offset_type = memory_flag & MEMORY64_FLAG ? VALUE_TYPE_I64 + : VALUE_TYPE_I32; +#else + mem_offset_type = VALUE_TYPE_I32; +#endif + } + +#if WASM_ENABLE_BULK_MEMORY != 0 + if (!is_passive) +#endif + if (!load_init_expr(module, &p, p_end, &init_expr, + mem_offset_type, NULL, error_buf, + error_buf_size)) + return false; + + read_leb_uint32(p, p_end, data_seg_len); + + if (!(dataseg = module->data_segments[i] = loader_malloc( + sizeof(WASMDataSeg), error_buf, error_buf_size))) { +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + destroy_init_expr(module, &init_expr); +#endif + return false; + } + +#if WASM_ENABLE_BULK_MEMORY != 0 + dataseg->is_passive = is_passive; + if (!is_passive) +#endif + { + bh_memcpy_s(&dataseg->base_offset, + sizeof(InitializerExpression), &init_expr, + sizeof(InitializerExpression)); + + dataseg->memory_index = mem_index; + } + + dataseg->data_length = data_seg_len; + CHECK_BUF(p, p_end, data_seg_len); + if (clone_data_seg) { + if (!(dataseg->data = loader_malloc( + dataseg->data_length, error_buf, error_buf_size))) { + return false; + } + + bh_memcpy_s(dataseg->data, dataseg->data_length, p, + data_seg_len); + } + else { + dataseg->data = (uint8 *)p; + } + dataseg->is_data_cloned = clone_data_seg; + p += data_seg_len; + } + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load data segment section success.\n"); + return true; +fail: + return false; +} + +#if WASM_ENABLE_BULK_MEMORY != 0 +static bool +load_datacount_section(const uint8 *buf, const uint8 *buf_end, + WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 data_seg_count1 = 0; + + read_leb_uint32(p, p_end, data_seg_count1); + module->data_seg_count1 = data_seg_count1; + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_bulk_memory_used = true; +#endif + LOG_VERBOSE("Load datacount section success.\n"); + return true; +fail: + return false; +} +#endif + +#if WASM_ENABLE_TAGS != 0 +static bool +load_tag_section(const uint8 *buf, const uint8 *buf_end, const uint8 *buf_code, + const uint8 *buf_code_end, WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + (void)buf_code; + (void)buf_code_end; + + const uint8 *p = buf, *p_end = buf_end; + size_t total_size = 0; + /* number of tags defined in the section */ + uint32 section_tag_count = 0; + uint8 tag_attribute; + uint32 tag_type; + WASMTag *tag = NULL; + + /* get tag count */ + read_leb_uint32(p, p_end, section_tag_count); + if (is_indices_overflow(module->import_tag_count, section_tag_count, + error_buf, error_buf_size)) + return false; + + module->tag_count = section_tag_count; + + if (section_tag_count) { + total_size = sizeof(WASMTag *) * module->tag_count; + if (!(module->tags = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + /* load each tag, imported tags precede the tags */ + uint32 tag_index; + for (tag_index = 0; tag_index < section_tag_count; tag_index++) { + + /* get the one byte attribute */ + CHECK_BUF(p, p_end, 1); + tag_attribute = read_uint8(p); + + /* get type */ + read_leb_uint32(p, p_end, tag_type); + /* compare against module->types */ + if (!check_function_type(module, tag_type, error_buf, + error_buf_size)) { + return false; + } + + /* get return type (must be 0) */ + /* check, that the type of the referred tag returns void */ + WASMFuncType *func_type = (WASMFuncType *)module->types[tag_type]; + if (func_type->result_count != 0) { + set_error_buf(error_buf, error_buf_size, + "non-empty tag result type"); + + goto fail; + } + + if (!(tag = module->tags[tag_index] = loader_malloc( + sizeof(WASMTag), error_buf, error_buf_size))) { + return false; + } + + /* store to module tag declarations */ + tag->attribute = tag_attribute; + tag->type = tag_type; + tag->tag_type = func_type; + } + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load tag section success.\n"); + return true; +fail: + return false; +} +#endif /* end of WASM_ENABLE_TAGS != 0 */ + +static bool +load_code_section(const uint8 *buf, const uint8 *buf_end, const uint8 *buf_func, + const uint8 *buf_func_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + const uint8 *p_func = buf_func; + uint32 func_count = 0, code_count; + + /* code has been loaded in function section, so pass it here, just check + * whether function and code section have inconsistent lengths */ + read_leb_uint32(p, p_end, code_count); + + if (buf_func) + read_leb_uint32(p_func, buf_func_end, func_count); + + if (func_count != code_count) { + set_error_buf(error_buf, error_buf_size, + "function and code section have inconsistent lengths"); + return false; + } + + LOG_VERBOSE("Load code segment section success.\n"); + (void)module; + return true; +fail: + return false; +} + +static bool +load_start_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + WASMFuncType *type; + uint32 start_function; + + read_leb_uint32(p, p_end, start_function); + + if (start_function + >= module->function_count + module->import_function_count) { + set_error_buf(error_buf, error_buf_size, "unknown function"); + return false; + } + + if (start_function < module->import_function_count) + type = module->import_functions[start_function].u.function.func_type; + else + type = module->functions[start_function - module->import_function_count] + ->func_type; + if (type->param_count != 0 || type->result_count != 0) { + set_error_buf(error_buf, error_buf_size, "invalid start function"); + return false; + } + + module->start_function = start_function; + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + return false; + } + + LOG_VERBOSE("Load start section success.\n"); + return true; +fail: + return false; +} + +#if WASM_ENABLE_STRINGREF != 0 +static bool +load_stringref_section(const uint8 *buf, const uint8 *buf_end, + WASMModule *module, bool is_load_from_file_buf, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + int32 deferred_count, immediate_count, string_length, i; + uint64 total_size; + + read_leb_uint32(p, p_end, deferred_count); + read_leb_uint32(p, p_end, immediate_count); + + /* proposal set deferred_count for future extension */ + if (deferred_count != 0) { + goto fail; + } + + if (immediate_count > 0) { + total_size = sizeof(char *) * (uint64)immediate_count; + if (!(module->string_literal_ptrs = + loader_malloc(total_size, error_buf, error_buf_size))) { + goto fail; + } + module->string_literal_count = immediate_count; + + total_size = sizeof(uint32) * (uint64)immediate_count; + if (!(module->string_literal_lengths = + loader_malloc(total_size, error_buf, error_buf_size))) { + goto fail; + } + + for (i = 0; i < immediate_count; i++) { + read_leb_uint32(p, p_end, string_length); + + CHECK_BUF(p, p_end, string_length); + module->string_literal_ptrs[i] = p; + module->string_literal_lengths[i] = string_length; + p += string_length; + } + } + + if (p != p_end) { + set_error_buf(error_buf, error_buf_size, "section size mismatch"); + goto fail; + } + + LOG_VERBOSE("Load stringref section success.\n"); + return true; + +fail: + return false; +} +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ + +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 +static bool +handle_name_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + uint32 name_type, subsection_size; + uint32 previous_name_type = 0; + uint32 num_func_name; + uint32 func_index; + uint32 previous_func_index = ~0U; + uint32 func_name_len; + uint32 name_index; + int i = 0; + + if (p >= p_end) { + set_error_buf(error_buf, error_buf_size, "unexpected end"); + return false; + } + + while (p < p_end) { + read_leb_uint32(p, p_end, name_type); + if (i != 0) { + if (name_type == previous_name_type) { + set_error_buf(error_buf, error_buf_size, + "duplicate sub-section"); + return false; + } + if (name_type < previous_name_type) { + set_error_buf(error_buf, error_buf_size, + "out-of-order sub-section"); + return false; + } + } + previous_name_type = name_type; + read_leb_uint32(p, p_end, subsection_size); + CHECK_BUF(p, p_end, subsection_size); + switch (name_type) { + case SUB_SECTION_TYPE_FUNC: + if (subsection_size) { + read_leb_uint32(p, p_end, num_func_name); + for (name_index = 0; name_index < num_func_name; + name_index++) { + read_leb_uint32(p, p_end, func_index); + if (func_index == previous_func_index) { + set_error_buf(error_buf, error_buf_size, + "duplicate function name"); + return false; + } + if (func_index < previous_func_index + && previous_func_index != ~0U) { + set_error_buf(error_buf, error_buf_size, + "out-of-order function index "); + return false; + } + previous_func_index = func_index; + read_leb_uint32(p, p_end, func_name_len); + CHECK_BUF(p, p_end, func_name_len); + /* Skip the import functions */ + if (func_index >= module->import_function_count) { + func_index -= module->import_function_count; + if (func_index >= module->function_count) { + set_error_buf(error_buf, error_buf_size, + "out-of-range function index"); + return false; + } + if (!(module->functions[func_index]->field_name = + wasm_const_str_list_insert( + p, func_name_len, module, +#if WASM_ENABLE_WAMR_COMPILER != 0 + false, +#else + is_load_from_file_buf, +#endif + error_buf, error_buf_size))) { + return false; + } + } + p += func_name_len; + } + } + break; + case SUB_SECTION_TYPE_MODULE: /* TODO: Parse for module subsection + */ + case SUB_SECTION_TYPE_LOCAL: /* TODO: Parse for local subsection */ + default: + p = p + subsection_size; + break; + } + i++; + } + + return true; +fail: + return false; +} +#endif + +#if WASM_ENABLE_BRANCH_HINTS != 0 +static bool +handle_branch_hint_section(const uint8 *buf, const uint8 *buf_end, + WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + if (module->function_hints == NULL) { + module->function_hints = loader_malloc( + sizeof(struct WASMCompilationHint) * module->function_count, + error_buf, error_buf_size); + } + uint32 numFunctionHints = 0; + read_leb_uint32(buf, buf_end, numFunctionHints); + for (uint32 i = 0; i < numFunctionHints; ++i) { + uint32 func_idx; + read_leb_uint32(buf, buf_end, func_idx); + if (!check_function_index(module, func_idx, error_buf, + error_buf_size)) { + goto fail; + } + if (func_idx < module->import_function_count) { + set_error_buf(error_buf, error_buf_size, + "branch hint for imported function is not allowed"); + goto fail; + } + + struct WASMCompilationHint *current_hint = + (struct WASMCompilationHint *)&module + ->function_hints[func_idx - module->import_function_count]; + while (current_hint->next != NULL) { + current_hint = current_hint->next; + } + + uint32 num_hints; + read_leb_uint32(buf, buf_end, num_hints); + struct WASMCompilationHintBranchHint *new_hints = loader_malloc( + sizeof(struct WASMCompilationHintBranchHint) * num_hints, error_buf, + error_buf_size); + for (uint32 j = 0; j < num_hints; ++j) { + struct WASMCompilationHintBranchHint *new_hint = &new_hints[j]; + new_hint->next = NULL; + new_hint->type = WASM_COMPILATION_BRANCH_HINT; + read_leb_uint32(buf, buf_end, new_hint->offset); + + uint32 size; + read_leb_uint32(buf, buf_end, size); + if (size != 1) { + set_error_buf_v(error_buf, error_buf_size, + "invalid branch hint size, expected 1, got %d.", + size); + wasm_runtime_free(new_hint); + goto fail; + } + + uint8 data = *buf++; + if (data == 0x00) + new_hint->is_likely = false; + else if (data == 0x01) + new_hint->is_likely = true; + else { + set_error_buf_v(error_buf, error_buf_size, + "invalid branch hint, expected 0 or 1, got %d", + data); + wasm_runtime_free(new_hint); + goto fail; + } + + current_hint->next = (struct WASMCompilationHint *)new_hint; + current_hint = (struct WASMCompilationHint *)new_hint; + } + } + if (buf != buf_end) { + set_error_buf(error_buf, error_buf_size, + "invalid branch hint section, not filled until end"); + goto fail; + } + return true; +fail: + return false; +} +#endif + +static bool +load_user_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *p = buf, *p_end = buf_end; + char section_name[32]; + uint32 name_len, buffer_len; + + if (p >= p_end) { + set_error_buf(error_buf, error_buf_size, "unexpected end"); + return false; + } + + read_leb_uint32(p, p_end, name_len); + + if (p + name_len > p_end) { + set_error_buf(error_buf, error_buf_size, "unexpected end"); + return false; + } + + if (!wasm_check_utf8_str(p, name_len)) { + set_error_buf(error_buf, error_buf_size, "invalid UTF-8 encoding"); + return false; + } + + buffer_len = sizeof(section_name); + memset(section_name, 0, buffer_len); + if (name_len < buffer_len) { + bh_memcpy_s(section_name, buffer_len, p, name_len); + } + else { + bh_memcpy_s(section_name, buffer_len, p, buffer_len - 4); + memset(section_name + buffer_len - 4, '.', 3); + } + +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 + if (name_len == 4 && memcmp(p, "name", 4) == 0) { + module->name_section_buf = buf; + module->name_section_buf_end = buf_end; + p += name_len; + if (!handle_name_section(p, p_end, module, is_load_from_file_buf, + error_buf, error_buf_size)) { + return false; + } + LOG_VERBOSE("Load custom name section success."); + } +#endif + +#if WASM_ENABLE_BRANCH_HINTS != 0 + if (name_len == 25 + && memcmp((const char *)p, "metadata.code.branch_hint", 25) == 0) { + p += name_len; + if (!handle_branch_hint_section(p, p_end, module, error_buf, + error_buf_size)) { + return false; + } + LOG_VERBOSE("Load branch hint section success."); + } +#else + if (name_len == 25 + && memcmp((const char *)p, "metadata.code.branch_hint", 25) == 0) { + LOG_VERBOSE("Found branch hint section, but branch hints are disabled " + "in this build, skipping."); + } +#endif + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 + { + WASMCustomSection *section = + loader_malloc(sizeof(WASMCustomSection), error_buf, error_buf_size); + + if (!section) { + return false; + } + + section->name_addr = (char *)p; + section->name_len = name_len; + section->content_addr = (uint8 *)(p + name_len); + section->content_len = (uint32)(p_end - p - name_len); + + section->next = module->custom_section_list; + module->custom_section_list = section; + LOG_VERBOSE("Load custom section [%s] success.", section_name); + return true; + } +#endif + + LOG_VERBOSE("Ignore custom section [%s].", section_name); + + (void)is_load_from_file_buf; + (void)module; + return true; +fail: + return false; +} + +static void +calculate_global_data_offset(WASMModule *module) +{ + uint32 i, data_offset; + + data_offset = 0; + for (i = 0; i < module->import_global_count; i++) { + WASMGlobalImport *import_global = + &((module->import_globals + i)->u.global); +#if WASM_ENABLE_FAST_JIT != 0 + import_global->data_offset = data_offset; +#endif + data_offset += wasm_value_type_size(import_global->type.val_type); + } + + for (i = 0; i < module->global_count; i++) { + WASMGlobal *global = module->globals + i; +#if WASM_ENABLE_FAST_JIT != 0 + global->data_offset = data_offset; +#endif + data_offset += wasm_value_type_size(global->type.val_type); + } + + module->global_data_size = data_offset; +} + +#if WASM_ENABLE_FAST_JIT != 0 +static bool +init_fast_jit_functions(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ +#if WASM_ENABLE_LAZY_JIT != 0 + JitGlobals *jit_globals = jit_compiler_get_jit_globals(); +#endif + uint32 i; + + if (!module->function_count) + return true; + + if (!(module->fast_jit_func_ptrs = + loader_malloc(sizeof(void *) * module->function_count, error_buf, + error_buf_size))) { + return false; + } + +#if WASM_ENABLE_LAZY_JIT != 0 + for (i = 0; i < module->function_count; i++) { + module->fast_jit_func_ptrs[i] = + jit_globals->compile_fast_jit_and_then_call; + } +#endif + + for (i = 0; i < WASM_ORC_JIT_BACKEND_THREAD_NUM; i++) { + if (os_mutex_init(&module->fast_jit_thread_locks[i]) != 0) { + set_error_buf(error_buf, error_buf_size, + "init fast jit thread lock failed"); + return false; + } + module->fast_jit_thread_locks_inited[i] = true; + } + + return true; +} +#endif /* end of WASM_ENABLE_FAST_JIT != 0 */ + +#if WASM_ENABLE_JIT != 0 +static bool +init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + LLVMJITOptions *llvm_jit_options = wasm_runtime_get_llvm_jit_options(); + AOTCompOption option = { 0 }; + char *aot_last_error; + uint64 size; +#if WASM_ENABLE_GC != 0 + bool gc_enabled = true; +#else + bool gc_enabled = false; +#endif + + if (module->function_count == 0) + return true; + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + if (os_mutex_init(&module->tierup_wait_lock) != 0) { + set_error_buf(error_buf, error_buf_size, "init jit tierup lock failed"); + return false; + } + if (os_cond_init(&module->tierup_wait_cond) != 0) { + set_error_buf(error_buf, error_buf_size, "init jit tierup cond failed"); + os_mutex_destroy(&module->tierup_wait_lock); + return false; + } + module->tierup_wait_lock_inited = true; +#endif + + size = sizeof(void *) * (uint64)module->function_count + + sizeof(bool) * (uint64)module->function_count; + if (!(module->func_ptrs = loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + module->func_ptrs_compiled = + (bool *)((uint8 *)module->func_ptrs + + sizeof(void *) * module->function_count); + + module->comp_data = aot_create_comp_data(module, NULL, gc_enabled); + if (!module->comp_data) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + + option.is_jit_mode = true; + + option.opt_level = llvm_jit_options->opt_level; + option.size_level = llvm_jit_options->size_level; + option.segue_flags = llvm_jit_options->segue_flags; + option.quick_invoke_c_api_import = + llvm_jit_options->quick_invoke_c_api_import; + +#if WASM_ENABLE_BULK_MEMORY != 0 + option.enable_bulk_memory = true; +#endif +#if WASM_ENABLE_BULK_MEMORY_OPT != 0 + option.enable_bulk_memory_opt = true; +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + option.enable_thread_mgr = true; +#endif +#if WASM_ENABLE_TAIL_CALL != 0 + option.enable_tail_call = true; +#endif +#if WASM_ENABLE_SIMD != 0 + option.enable_simd = true; +#endif +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + option.enable_ref_types = true; +#elif WASM_ENABLE_GC != 0 + option.enable_gc = true; +#endif +#if WASM_ENABLE_CALL_INDIRECT_OVERLONG != 0 + option.enable_call_indirect_overlong = true; +#endif + option.enable_aux_stack_check = true; +#if WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0 \ + || WASM_ENABLE_AOT_STACK_FRAME != 0 + option.aux_stack_frame_type = AOT_STACK_FRAME_TYPE_STANDARD; + aot_call_stack_features_init_default(&option.call_stack_features); +#endif +#if WASM_ENABLE_PERF_PROFILING != 0 + option.enable_perf_profiling = true; +#endif +#if WASM_ENABLE_MEMORY_PROFILING != 0 + option.enable_memory_profiling = true; + option.enable_stack_estimation = true; +#endif +#if WASM_ENABLE_SHARED_HEAP != 0 + option.enable_shared_heap = true; +#endif + + module->comp_ctx = aot_create_comp_context(module->comp_data, &option); + if (!module->comp_ctx) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + + return true; +} + +static bool +init_llvm_jit_functions_stage2(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + char *aot_last_error; + uint32 i; + + if (module->function_count == 0) + return true; + + if (!aot_compile_wasm(module->comp_ctx)) { + aot_last_error = aot_get_last_error(); + bh_assert(aot_last_error != NULL); + set_error_buf(error_buf, error_buf_size, aot_last_error); + return false; + } + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + if (module->orcjit_stop_compiling) + return false; +#endif + + bh_print_time("Begin to lookup llvm jit functions"); + + for (i = 0; i < module->function_count; i++) { + LLVMOrcJITTargetAddress func_addr = 0; + LLVMErrorRef error; + char func_name[48]; + + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); + error = LLVMOrcLLLazyJITLookup(module->comp_ctx->orc_jit, &func_addr, + func_name); + if (error != LLVMErrorSuccess) { + char *err_msg = LLVMGetErrorMessage(error); + set_error_buf_v(error_buf, error_buf_size, + "failed to compile llvm jit function: %s", err_msg); + LLVMDisposeErrorMessage(err_msg); + return false; + } + + /** + * No need to lock the func_ptr[func_idx] here as it is basic + * data type, the load/store for it can be finished by one cpu + * instruction, and there can be only one cpu instruction + * loading/storing at the same time. + */ + module->func_ptrs[i] = (void *)func_addr; + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + module->functions[i]->llvm_jit_func_ptr = (void *)func_addr; + + if (module->orcjit_stop_compiling) + return false; +#endif + } + + bh_print_time("End lookup llvm jit functions"); + + return true; +} +#endif /* end of WASM_ENABLE_JIT != 0 */ + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 +static void * +init_llvm_jit_functions_stage2_callback(void *arg) +{ + WASMModule *module = (WASMModule *)arg; + char error_buf[128]; + uint32 error_buf_size = (uint32)sizeof(error_buf); + + if (!init_llvm_jit_functions_stage2(module, error_buf, error_buf_size)) { + module->orcjit_stop_compiling = true; + return NULL; + } + + os_mutex_lock(&module->tierup_wait_lock); + module->llvm_jit_inited = true; + os_cond_broadcast(&module->tierup_wait_cond); + os_mutex_unlock(&module->tierup_wait_lock); + + return NULL; +} +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 +/* The callback function to compile jit functions */ +static void * +orcjit_thread_callback(void *arg) +{ + OrcJitThreadArg *thread_arg = (OrcJitThreadArg *)arg; +#if WASM_ENABLE_JIT != 0 + AOTCompContext *comp_ctx = thread_arg->comp_ctx; +#endif + WASMModule *module = thread_arg->module; + uint32 group_idx = thread_arg->group_idx; + uint32 group_stride = WASM_ORC_JIT_BACKEND_THREAD_NUM; + uint32 func_count = module->function_count; + uint32 i; + +#if WASM_ENABLE_FAST_JIT != 0 + /* Compile fast jit functions of this group */ + for (i = group_idx; i < func_count; i += group_stride) { + if (!jit_compiler_compile(module, i + module->import_function_count)) { + LOG_ERROR("failed to compile fast jit function %u\n", i); + break; + } + + if (module->orcjit_stop_compiling) { + return NULL; + } + } +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + os_mutex_lock(&module->tierup_wait_lock); + module->fast_jit_ready_groups++; + os_mutex_unlock(&module->tierup_wait_lock); +#endif +#endif + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + /* For JIT tier-up, set each llvm jit func to call_to_fast_jit */ + for (i = group_idx; i < func_count; + i += group_stride * WASM_ORC_JIT_COMPILE_THREAD_NUM) { + uint32 j; + + for (j = 0; j < WASM_ORC_JIT_COMPILE_THREAD_NUM; j++) { + if (i + j * group_stride < func_count) { + if (!jit_compiler_set_call_to_fast_jit( + module, + i + j * group_stride + module->import_function_count)) { + LOG_ERROR( + "failed to compile call_to_fast_jit for func %u\n", + i + j * group_stride + module->import_function_count); + module->orcjit_stop_compiling = true; + return NULL; + } + } + if (module->orcjit_stop_compiling) { + return NULL; + } + } + } + + /* Wait until init_llvm_jit_functions_stage2 finishes and all + fast jit functions are compiled */ + os_mutex_lock(&module->tierup_wait_lock); + while (!(module->llvm_jit_inited && module->enable_llvm_jit_compilation + && module->fast_jit_ready_groups >= group_stride)) { + os_cond_reltimedwait(&module->tierup_wait_cond, + &module->tierup_wait_lock, 10000); + if (module->orcjit_stop_compiling) { + /* init_llvm_jit_functions_stage2 failed */ + os_mutex_unlock(&module->tierup_wait_lock); + return NULL; + } + } + os_mutex_unlock(&module->tierup_wait_lock); +#endif + +#if WASM_ENABLE_JIT != 0 + /* Compile llvm jit functions of this group */ + for (i = group_idx; i < func_count; + i += group_stride * WASM_ORC_JIT_COMPILE_THREAD_NUM) { + LLVMOrcJITTargetAddress func_addr = 0; + LLVMErrorRef error; + char func_name[48]; + typedef void (*F)(void); + union { + F f; + void *v; + } u; + uint32 j; + + snprintf(func_name, sizeof(func_name), "%s%d%s", AOT_FUNC_PREFIX, i, + "_wrapper"); + LOG_DEBUG("compile llvm jit func %s", func_name); + error = + LLVMOrcLLLazyJITLookup(comp_ctx->orc_jit, &func_addr, func_name); + if (error != LLVMErrorSuccess) { + char *err_msg = LLVMGetErrorMessage(error); + LOG_ERROR("failed to compile llvm jit function %u: %s", i, err_msg); + LLVMDisposeErrorMessage(err_msg); + break; + } + + /* Call the jit wrapper function to trigger its compilation, so as + to compile the actual jit functions, since we add the latter to + function list in the PartitionFunction callback */ + u.v = (void *)func_addr; + u.f(); + + for (j = 0; j < WASM_ORC_JIT_COMPILE_THREAD_NUM; j++) { + if (i + j * group_stride < func_count) { + module->func_ptrs_compiled[i + j * group_stride] = true; +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, + i + j * group_stride); + error = LLVMOrcLLLazyJITLookup(comp_ctx->orc_jit, &func_addr, + func_name); + if (error != LLVMErrorSuccess) { + char *err_msg = LLVMGetErrorMessage(error); + LOG_ERROR("failed to compile llvm jit function %u: %s", i, + err_msg); + LLVMDisposeErrorMessage(err_msg); + /* Ignore current llvm jit func, as its func ptr is + previous set to call_to_fast_jit, which also works */ + continue; + } + + jit_compiler_set_llvm_jit_func_ptr( + module, + i + j * group_stride + module->import_function_count, + (void *)func_addr); + + /* Try to switch to call this llvm jit function instead of + fast jit function from fast jit jitted code */ + jit_compiler_set_call_to_llvm_jit( + module, + i + j * group_stride + module->import_function_count); +#endif + } + } + + if (module->orcjit_stop_compiling) { + break; + } + } +#endif + + return NULL; +} + +static void +orcjit_stop_compile_threads(WASMModule *module) +{ +#if WASM_ENABLE_LAZY_JIT != 0 + uint32 i, thread_num = (uint32)(sizeof(module->orcjit_thread_args) + / sizeof(OrcJitThreadArg)); + + module->orcjit_stop_compiling = true; + for (i = 0; i < thread_num; i++) { + if (module->orcjit_threads[i]) + os_thread_join(module->orcjit_threads[i], NULL); + } +#endif +} + +static bool +compile_jit_functions(WASMModule *module, char *error_buf, + uint32 error_buf_size) +{ + uint32 thread_num = + (uint32)(sizeof(module->orcjit_thread_args) / sizeof(OrcJitThreadArg)); + uint32 i, j; + + bh_print_time("Begin to compile jit functions"); + + /* Create threads to compile the jit functions */ + for (i = 0; i < thread_num && i < module->function_count; i++) { +#if WASM_ENABLE_JIT != 0 + module->orcjit_thread_args[i].comp_ctx = module->comp_ctx; +#endif + module->orcjit_thread_args[i].module = module; + module->orcjit_thread_args[i].group_idx = i; + + if (os_thread_create(&module->orcjit_threads[i], orcjit_thread_callback, + (void *)&module->orcjit_thread_args[i], + APP_THREAD_STACK_SIZE_DEFAULT) + != 0) { + set_error_buf(error_buf, error_buf_size, + "create orcjit compile thread failed"); + /* Terminate the threads created */ + module->orcjit_stop_compiling = true; + for (j = 0; j < i; j++) { + os_thread_join(module->orcjit_threads[j], NULL); + } + return false; + } + } + +#if WASM_ENABLE_LAZY_JIT == 0 + /* Wait until all jit functions are compiled for eager mode */ + for (i = 0; i < thread_num; i++) { + if (module->orcjit_threads[i]) + os_thread_join(module->orcjit_threads[i], NULL); + } + +#if WASM_ENABLE_FAST_JIT != 0 + /* Ensure all the fast-jit functions are compiled */ + for (i = 0; i < module->function_count; i++) { + if (!jit_compiler_is_compiled(module, + i + module->import_function_count)) { + set_error_buf(error_buf, error_buf_size, + "failed to compile fast jit function"); + return false; + } + } +#endif + +#if WASM_ENABLE_JIT != 0 + /* Ensure all the llvm-jit functions are compiled */ + for (i = 0; i < module->function_count; i++) { + if (!module->func_ptrs_compiled[i]) { + set_error_buf(error_buf, error_buf_size, + "failed to compile llvm jit function"); + return false; + } + } +#endif +#endif /* end of WASM_ENABLE_LAZY_JIT == 0 */ + + bh_print_time("End compile jit functions"); + + return true; +} +#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 */ + +static bool +wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, + uint32 cur_func_idx, char *error_buf, + uint32 error_buf_size); + +#if WASM_ENABLE_FAST_INTERP != 0 && WASM_ENABLE_LABELS_AS_VALUES != 0 +void ** +wasm_interp_get_handle_table(void); + +static void **handle_table; +#endif + +static bool +load_from_sections(WASMModule *module, WASMSection *sections, + bool is_load_from_file_buf, bool wasm_binary_freeable, + bool no_resolve, char *error_buf, uint32 error_buf_size) +{ + WASMExport *export; + WASMSection *section = sections; + const uint8 *buf, *buf_end, *buf_code = NULL, *buf_code_end = NULL, + *buf_func = NULL, *buf_func_end = NULL; + WASMGlobal *aux_data_end_global = NULL, *aux_heap_base_global = NULL; + WASMGlobal *aux_stack_top_global = NULL, *global; + uint64 aux_data_end = (uint64)-1LL, aux_heap_base = (uint64)-1LL, + aux_stack_top = (uint64)-1LL; + uint32 global_index, func_index, i; + uint32 aux_data_end_global_index = (uint32)-1; + uint32 aux_heap_base_global_index = (uint32)-1; + WASMFuncType *func_type; + uint8 malloc_free_io_type = VALUE_TYPE_I32; + bool reuse_const_strings = is_load_from_file_buf && !wasm_binary_freeable; + bool clone_data_seg = is_load_from_file_buf && wasm_binary_freeable; +#if WASM_ENABLE_BULK_MEMORY != 0 + bool has_datacount_section = false; +#endif + + /* Find code and function sections if have */ + while (section) { + if (section->section_type == SECTION_TYPE_CODE) { + buf_code = section->section_body; + buf_code_end = buf_code + section->section_body_size; +#if WASM_ENABLE_DEBUG_INTERP != 0 || WASM_ENABLE_DEBUG_AOT != 0 + module->buf_code = (uint8 *)buf_code; + module->buf_code_size = section->section_body_size; +#endif + } + else if (section->section_type == SECTION_TYPE_FUNC) { + buf_func = section->section_body; + buf_func_end = buf_func + section->section_body_size; + } + section = section->next; + } + + section = sections; + while (section) { + buf = section->section_body; + buf_end = buf + section->section_body_size; + switch (section->section_type) { + case SECTION_TYPE_USER: + /* unsupported user section, ignore it. */ + if (!load_user_section(buf, buf_end, module, + reuse_const_strings, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_TYPE: + if (!load_type_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_IMPORT: + if (!load_import_section(buf, buf_end, module, + reuse_const_strings, no_resolve, + error_buf, error_buf_size)) + return false; + break; + case SECTION_TYPE_FUNC: + if (!load_function_section(buf, buf_end, buf_code, buf_code_end, + module, error_buf, error_buf_size)) + return false; + break; + case SECTION_TYPE_TABLE: + if (!load_table_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_MEMORY: + if (!load_memory_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; +#if WASM_ENABLE_TAGS != 0 + case SECTION_TYPE_TAG: + /* load tag declaration section */ + if (!load_tag_section(buf, buf_end, buf_code, buf_code_end, + module, error_buf, error_buf_size)) + return false; + break; +#endif + case SECTION_TYPE_GLOBAL: + if (!load_global_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_EXPORT: + if (!load_export_section(buf, buf_end, module, + reuse_const_strings, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_START: + if (!load_start_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_ELEM: + if (!load_table_segment_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + break; + case SECTION_TYPE_CODE: + if (!load_code_section(buf, buf_end, buf_func, buf_func_end, + module, error_buf, error_buf_size)) + return false; + break; + case SECTION_TYPE_DATA: + if (!load_data_segment_section(buf, buf_end, module, +#if WASM_ENABLE_BULK_MEMORY != 0 + has_datacount_section, +#endif + clone_data_seg, error_buf, + error_buf_size)) + return false; + break; +#if WASM_ENABLE_BULK_MEMORY != 0 + case SECTION_TYPE_DATACOUNT: + if (!load_datacount_section(buf, buf_end, module, error_buf, + error_buf_size)) + return false; + has_datacount_section = true; + break; +#endif +#if WASM_ENABLE_STRINGREF != 0 + case SECTION_TYPE_STRINGREF: + if (!load_stringref_section(buf, buf_end, module, + reuse_const_strings, error_buf, + error_buf_size)) + return false; + break; +#endif + default: + set_error_buf(error_buf, error_buf_size, "invalid section id"); + return false; + } + + section = section->next; + } + +#if WASM_ENABLE_BULK_MEMORY != 0 + if (!check_data_count_consistency( + has_datacount_section, module->data_seg_count1, + module->data_seg_count, error_buf, error_buf_size)) { + return false; + } +#endif + + module->aux_data_end_global_index = (uint32)-1; + module->aux_heap_base_global_index = (uint32)-1; + module->aux_stack_top_global_index = (uint32)-1; + + /* Resolve auxiliary data/stack/heap info and reset memory info */ + export = module->exports; + for (i = 0; i < module->export_count; i++, export ++) { + if (export->kind == EXPORT_KIND_GLOBAL) { + if (!strcmp(export->name, "__heap_base")) { + if (export->index < module->import_global_count) { + LOG_DEBUG("Skip the process if __heap_base is imported " + "instead of being a local global"); + continue; + } + + /* only process linker-generated symbols */ + global_index = export->index - module->import_global_count; + global = module->globals + global_index; + if (global->type.val_type == VALUE_TYPE_I32 + && !global->type.is_mutable + && global->init_expr.init_expr_type + == INIT_EXPR_TYPE_I32_CONST) { + aux_heap_base_global = global; + aux_heap_base = + (uint64)(uint32)global->init_expr.u.unary.v.i32; + aux_heap_base_global_index = export->index; + LOG_VERBOSE("Found aux __heap_base global, value: %" PRIu64, + aux_heap_base); + } + } + else if (!strcmp(export->name, "__data_end")) { + if (export->index < module->import_global_count) { + LOG_DEBUG("Skip the process if __data_end is imported " + "instead of being a local global"); + continue; + } + + /* only process linker-generated symbols */ + global_index = export->index - module->import_global_count; + global = module->globals + global_index; + if (global->type.val_type == VALUE_TYPE_I32 + && !global->type.is_mutable + && global->init_expr.init_expr_type + == INIT_EXPR_TYPE_I32_CONST) { + aux_data_end_global = global; + aux_data_end = + (uint64)(uint32)global->init_expr.u.unary.v.i32; + aux_data_end_global_index = export->index; + LOG_VERBOSE("Found aux __data_end global, value: %" PRIu64, + aux_data_end); + + aux_data_end = align_uint64(aux_data_end, 16); + } + } + + /* For module compiled with -pthread option, the global is: + [0] stack_top <-- 0 + [1] tls_pointer + [2] tls_size + [3] data_end <-- 3 + [4] global_base + [5] heap_base <-- 5 + [6] dso_handle + + For module compiled without -pthread option: + [0] stack_top <-- 0 + [1] data_end <-- 1 + [2] global_base + [3] heap_base <-- 3 + [4] dso_handle + */ + if (aux_data_end_global && aux_heap_base_global + && aux_data_end <= aux_heap_base) { + module->aux_data_end_global_index = aux_data_end_global_index; + module->aux_data_end = aux_data_end; + module->aux_heap_base_global_index = aux_heap_base_global_index; + module->aux_heap_base = aux_heap_base; + + /* Resolve aux stack top global */ + for (global_index = 0; global_index < module->global_count; + global_index++) { + global = module->globals + global_index; + if (global->type.is_mutable /* heap_base and data_end is + not mutable */ + && global->type.val_type == VALUE_TYPE_I32 + && global->init_expr.init_expr_type + == INIT_EXPR_TYPE_I32_CONST + && (uint64)(uint32)global->init_expr.u.unary.v.i32 + <= aux_heap_base) { + aux_stack_top_global = global; + aux_stack_top = + (uint64)(uint32)global->init_expr.u.unary.v.i32; + module->aux_stack_top_global_index = + module->import_global_count + global_index; + module->aux_stack_bottom = aux_stack_top; + module->aux_stack_size = + aux_stack_top > aux_data_end + ? (uint32)(aux_stack_top - aux_data_end) + : (uint32)aux_stack_top; + LOG_VERBOSE( + "Found aux stack top global, value: %" PRIu64 ", " + "global index: %d, stack size: %d", + aux_stack_top, global_index, + module->aux_stack_size); + break; + } + } + if (!aux_stack_top_global) { + /* Auxiliary stack global isn't found, it must be unused + in the wasm app, as if it is used, the global must be + defined. Here we set it to __heap_base global and set + its size to 0. */ + aux_stack_top_global = aux_heap_base_global; + aux_stack_top = aux_heap_base; + module->aux_stack_top_global_index = + module->aux_heap_base_global_index; + module->aux_stack_bottom = aux_stack_top; + module->aux_stack_size = 0; + } + break; + } + } + } + + module->malloc_function = (uint32)-1; + module->free_function = (uint32)-1; + module->retain_function = (uint32)-1; + + /* Resolve malloc/free function exported by wasm module */ +#if WASM_ENABLE_MEMORY64 != 0 + if (has_module_memory64(module)) + malloc_free_io_type = VALUE_TYPE_I64; +#endif + export = module->exports; + for (i = 0; i < module->export_count; i++, export ++) { + if (export->kind == EXPORT_KIND_FUNC) { + if (!strcmp(export->name, "malloc") + && export->index >= module->import_function_count) { + func_index = export->index - module->import_function_count; + func_type = module->functions[func_index]->func_type; + if (func_type->param_count == 1 && func_type->result_count == 1 + && func_type->types[0] == malloc_free_io_type + && func_type->types[1] == malloc_free_io_type) { + bh_assert(module->malloc_function == (uint32)-1); + module->malloc_function = export->index; + LOG_VERBOSE("Found malloc function, name: %s, index: %u", + export->name, export->index); + } + } + else if (!strcmp(export->name, "__new") + && export->index >= module->import_function_count) { + /* __new && __pin for AssemblyScript */ + func_index = export->index - module->import_function_count; + func_type = module->functions[func_index]->func_type; + if (func_type->param_count == 2 && func_type->result_count == 1 + && func_type->types[0] == malloc_free_io_type + && func_type->types[1] == VALUE_TYPE_I32 + && func_type->types[2] == malloc_free_io_type) { + uint32 j; + WASMExport *export_tmp; + + bh_assert(module->malloc_function == (uint32)-1); + module->malloc_function = export->index; + LOG_VERBOSE("Found malloc function, name: %s, index: %u", + export->name, export->index); + + /* resolve retain function. + If not found, reset malloc function index */ + export_tmp = module->exports; + for (j = 0; j < module->export_count; j++, export_tmp++) { + if ((export_tmp->kind == EXPORT_KIND_FUNC) + && (!strcmp(export_tmp->name, "__retain") + || (!strcmp(export_tmp->name, "__pin"))) + && (export_tmp->index + >= module->import_function_count)) { + func_index = export_tmp->index + - module->import_function_count; + func_type = + module->functions[func_index]->func_type; + if (func_type->param_count == 1 + && func_type->result_count == 1 + && func_type->types[0] == malloc_free_io_type + && func_type->types[1] == malloc_free_io_type) { + bh_assert(module->retain_function + == (uint32)-1); + module->retain_function = export_tmp->index; + LOG_VERBOSE("Found retain function, name: %s, " + "index: %u", + export_tmp->name, + export_tmp->index); + break; + } + } + } + if (j == module->export_count) { + module->malloc_function = (uint32)-1; + LOG_VERBOSE("Can't find retain function," + "reset malloc function index to -1"); + } + } + } + else if (((!strcmp(export->name, "free")) + || (!strcmp(export->name, "__release")) + || (!strcmp(export->name, "__unpin"))) + && export->index >= module->import_function_count) { + func_index = export->index - module->import_function_count; + func_type = module->functions[func_index]->func_type; + if (func_type->param_count == 1 && func_type->result_count == 0 + && func_type->types[0] == malloc_free_io_type) { + bh_assert(module->free_function == (uint32)-1); + module->free_function = export->index; + LOG_VERBOSE("Found free function, name: %s, index: %u", + export->name, export->index); + } + } + } + } + +#if WASM_ENABLE_FAST_INTERP != 0 && WASM_ENABLE_LABELS_AS_VALUES != 0 + handle_table = wasm_interp_get_handle_table(); +#endif + + for (i = 0; i < module->function_count; i++) { + WASMFunction *func = module->functions[i]; + if (!wasm_loader_prepare_bytecode(module, func, i, error_buf, + error_buf_size)) { + return false; + } + + if (i == module->function_count - 1 + && func->code + func->code_size != buf_code_end) { + set_error_buf(error_buf, error_buf_size, + "code section size mismatch"); + return false; + } + } + + if (!module->possible_memory_grow) { +#if WASM_ENABLE_SHRUNK_MEMORY != 0 + if (aux_data_end_global && aux_heap_base_global + && aux_stack_top_global) { + uint64 init_memory_size; + uint64 shrunk_memory_size = align_uint64(aux_heap_base, 8); + + /* Only resize(shrunk) the memory size if num_bytes_per_page is in + * valid range of uint32 */ + if (shrunk_memory_size <= UINT32_MAX) { + if (module->import_memory_count) { + WASMMemoryImport *memory_import = + &module->import_memories[0].u.memory; + init_memory_size = + (uint64)memory_import->mem_type.num_bytes_per_page + * memory_import->mem_type.init_page_count; + if (shrunk_memory_size <= init_memory_size) { + /* Reset memory info to decrease memory usage */ + memory_import->mem_type.num_bytes_per_page = + (uint32)shrunk_memory_size; + memory_import->mem_type.init_page_count = 1; + LOG_VERBOSE("Shrink import memory size to %" PRIu64, + shrunk_memory_size); + } + } + + if (module->memory_count) { + WASMMemory *memory = &module->memories[0]; + init_memory_size = (uint64)memory->num_bytes_per_page + * memory->init_page_count; + if (shrunk_memory_size <= init_memory_size) { + /* Reset memory info to decrease memory usage */ + memory->num_bytes_per_page = (uint32)shrunk_memory_size; + memory->init_page_count = 1; + LOG_VERBOSE("Shrink memory size to %" PRIu64, + shrunk_memory_size); + } + } + } + } +#endif /* WASM_ENABLE_SHRUNK_MEMORY != 0 */ + +#if WASM_ENABLE_MULTI_MODULE == 0 + if (module->import_memory_count) { + WASMMemoryImport *memory_import = + &module->import_memories[0].u.memory; + /* Only resize the memory to one big page if num_bytes_per_page is + * in valid range of uint32 */ + if (memory_import->mem_type.init_page_count < DEFAULT_MAX_PAGES) { + memory_import->mem_type.num_bytes_per_page *= + memory_import->mem_type.init_page_count; + + if (memory_import->mem_type.init_page_count > 0) + memory_import->mem_type.init_page_count = + memory_import->mem_type.max_page_count = 1; + else + memory_import->mem_type.init_page_count = + memory_import->mem_type.max_page_count = 0; + } + } + if (module->memory_count) { + WASMMemory *memory = &module->memories[0]; + /* Only resize(shrunk) the memory size if num_bytes_per_page is in + * valid range of uint32 */ + if (memory->init_page_count < DEFAULT_MAX_PAGES) { + memory->num_bytes_per_page *= memory->init_page_count; + if (memory->init_page_count > 0) + memory->init_page_count = memory->max_page_count = 1; + else + memory->init_page_count = memory->max_page_count = 0; + } + } +#endif + } + +#if WASM_ENABLE_MEMORY64 != 0 + if (!check_memory64_flags_consistency(module, error_buf, error_buf_size, + false)) + return false; +#endif + + calculate_global_data_offset(module); + +#if WASM_ENABLE_FAST_JIT != 0 + if (!init_fast_jit_functions(module, error_buf, error_buf_size)) { + return false; + } +#endif + +#if WASM_ENABLE_JIT != 0 + if (!init_llvm_jit_functions_stage1(module, error_buf, error_buf_size)) { + return false; + } +#if !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0) + if (!init_llvm_jit_functions_stage2(module, error_buf, error_buf_size)) { + return false; + } +#else + /* Run aot_compile_wasm in a backend thread, so as not to block the main + thread fast jit execution, since applying llvm optimizations in + aot_compile_wasm may cost a lot of time. + Create thread with enough native stack to apply llvm optimizations */ + if (os_thread_create(&module->llvm_jit_init_thread, + init_llvm_jit_functions_stage2_callback, + (void *)module, APP_THREAD_STACK_SIZE_DEFAULT * 8) + != 0) { + set_error_buf(error_buf, error_buf_size, + "create orcjit compile thread failed"); + return false; + } +#endif +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 + /* Create threads to compile the jit functions */ + if (!compile_jit_functions(module, error_buf, error_buf_size)) { + return false; + } +#endif + +#if WASM_ENABLE_MEMORY_TRACING != 0 + wasm_runtime_dump_module_mem_consumption((WASMModuleCommon *)module); +#endif + return true; +} + +static WASMModule * +create_module(char *name, char *error_buf, uint32 error_buf_size) +{ + WASMModule *module = + loader_malloc(sizeof(WASMModule), error_buf, error_buf_size); + bh_list_status ret; + + if (!module) { + return NULL; + } + + module->module_type = Wasm_Module_Bytecode; + + /* Set start_function to -1, means no start function */ + module->start_function = (uint32)-1; + + module->name = name; + module->is_binary_freeable = false; + +#if WASM_ENABLE_FAST_INTERP == 0 + module->br_table_cache_list = &module->br_table_cache_list_head; + ret = bh_list_init(module->br_table_cache_list); + bh_assert(ret == BH_LIST_SUCCESS); +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + module->import_module_list = &module->import_module_list_head; + ret = bh_list_init(module->import_module_list); + bh_assert(ret == BH_LIST_SUCCESS); +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 + ret = bh_list_init(&module->fast_opcode_list); + bh_assert(ret == BH_LIST_SUCCESS); +#endif + +#if WASM_ENABLE_GC != 0 + if (!(module->ref_type_set = + wasm_reftype_set_create(GC_REFTYPE_MAP_SIZE_DEFAULT))) { + set_error_buf(error_buf, error_buf_size, "create reftype map failed"); + goto fail1; + } + + if (os_mutex_init(&module->rtt_type_lock)) { + set_error_buf(error_buf, error_buf_size, "init rtt type lock failed"); + goto fail2; + } +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 \ + || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0) + if (os_mutex_init(&module->instance_list_lock) != 0) { + set_error_buf(error_buf, error_buf_size, + "init instance list lock failed"); + goto fail3; + } +#endif + +#if WASM_ENABLE_LIBC_WASI != 0 + wasi_args_set_defaults(&module->wasi_args); +#endif /* WASM_ENABLE_LIBC_WASI != 0 */ + + (void)ret; + return module; + +#if WASM_ENABLE_DEBUG_INTERP != 0 \ + || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT \ + && WASM_ENABLE_LAZY_JIT != 0) +fail3: +#endif +#if WASM_ENABLE_GC != 0 + os_mutex_destroy(&module->rtt_type_lock); +fail2: + bh_hash_map_destroy(module->ref_type_set); +fail1: +#endif + wasm_runtime_free(module); + return NULL; +} + +#if WASM_ENABLE_DEBUG_INTERP != 0 +static bool +record_fast_op(WASMModule *module, uint8 *pos, uint8 orig_op, char *error_buf, + uint32 error_buf_size) +{ + WASMFastOPCodeNode *fast_op = + loader_malloc(sizeof(WASMFastOPCodeNode), error_buf, error_buf_size); + if (fast_op) { + fast_op->offset = pos - module->load_addr; + fast_op->orig_op = orig_op; + bh_list_insert(&module->fast_opcode_list, fast_op); + } + return fast_op ? true : false; +} +#endif + +WASMModule * +wasm_loader_load_from_sections(WASMSection *section_list, char *error_buf, + uint32 error_buf_size) +{ + WASMModule *module = create_module("", error_buf, error_buf_size); + if (!module) + return NULL; + + if (!load_from_sections(module, section_list, false, true, false, error_buf, + error_buf_size)) { + wasm_loader_unload(module); + return NULL; + } + + LOG_VERBOSE("Load module from sections success.\n"); + return module; +} + +static void +destroy_sections(WASMSection *section_list) +{ + WASMSection *section = section_list, *next; + while (section) { + next = section->next; + wasm_runtime_free(section); + section = next; + } +} + +/* clang-format off */ +static uint8 section_ids[] = { + SECTION_TYPE_USER, + SECTION_TYPE_TYPE, + SECTION_TYPE_IMPORT, + SECTION_TYPE_FUNC, + SECTION_TYPE_TABLE, + SECTION_TYPE_MEMORY, +#if WASM_ENABLE_TAGS != 0 + SECTION_TYPE_TAG, +#endif +#if WASM_ENABLE_STRINGREF != 0 + /* must immediately precede the global section, + or where the global section would be */ + SECTION_TYPE_STRINGREF, +#endif + SECTION_TYPE_GLOBAL, + SECTION_TYPE_EXPORT, + SECTION_TYPE_START, + SECTION_TYPE_ELEM, +#if WASM_ENABLE_BULK_MEMORY != 0 + SECTION_TYPE_DATACOUNT, +#endif + SECTION_TYPE_CODE, + SECTION_TYPE_DATA +}; +/* clang-format on */ + +static uint8 +get_section_index(uint8 section_type) +{ + uint8 max_id = sizeof(section_ids) / sizeof(uint8); + + for (uint8 i = 0; i < max_id; i++) { + if (section_type == section_ids[i]) + return i; + } + + return (uint8)-1; +} + +static bool +create_sections(const uint8 *buf, uint32 size, WASMSection **p_section_list, + char *error_buf, uint32 error_buf_size) +{ + WASMSection *section_list_end = NULL, *section; + const uint8 *p = buf, *p_end = buf + size; + uint8 section_type, section_index, last_section_index = (uint8)-1; + uint32 section_size; + + bh_assert(!*p_section_list); + + p += 8; + while (p < p_end) { + CHECK_BUF(p, p_end, 1); + section_type = read_uint8(p); + section_index = get_section_index(section_type); + if (section_index != (uint8)-1) { + if (section_type != SECTION_TYPE_USER) { + /* Custom sections may be inserted at any place, + while other sections must occur at most once + and in prescribed order. */ + if (last_section_index != (uint8)-1 + && (section_index <= last_section_index)) { + set_error_buf(error_buf, error_buf_size, + "unexpected content after last section or " + "junk after last section"); + return false; + } + last_section_index = section_index; + } + read_leb_uint32(p, p_end, section_size); + CHECK_BUF1(p, p_end, section_size); + + if (!(section = loader_malloc(sizeof(WASMSection), error_buf, + error_buf_size))) { + return false; + } + + section->section_type = section_type; + section->section_body = (uint8 *)p; + section->section_body_size = section_size; + + if (!section_list_end) + *p_section_list = section_list_end = section; + else { + section_list_end->next = section; + section_list_end = section; + } + + p += section_size; + } + else { + set_error_buf(error_buf, error_buf_size, "invalid section id"); + return false; + } + } + + return true; +fail: + return false; +} + +static void +exchange32(uint8 *p_data) +{ + uint8 value = *p_data; + *p_data = *(p_data + 3); + *(p_data + 3) = value; + + value = *(p_data + 1); + *(p_data + 1) = *(p_data + 2); + *(p_data + 2) = value; +} + +static union { + int a; + char b; +} __ue = { .a = 1 }; + +#define is_little_endian() (__ue.b == 1) + +static bool +load(const uint8 *buf, uint32 size, WASMModule *module, + bool wasm_binary_freeable, bool no_resolve, char *error_buf, + uint32 error_buf_size) +{ + const uint8 *buf_end = buf + size; + const uint8 *p = buf, *p_end = buf_end; + uint32 magic_number, version; + WASMSection *section_list = NULL; + + CHECK_BUF1(p, p_end, sizeof(uint32)); + magic_number = read_uint32(p); + if (!is_little_endian()) + exchange32((uint8 *)&magic_number); + + if (magic_number != WASM_MAGIC_NUMBER) { + set_error_buf(error_buf, error_buf_size, "magic header not detected"); + return false; + } + + CHECK_BUF1(p, p_end, sizeof(uint32)); + version = read_uint32(p); + if (!is_little_endian()) + exchange32((uint8 *)&version); + + if (version != WASM_CURRENT_VERSION) { + set_error_buf(error_buf, error_buf_size, "unknown binary version"); + return false; + } + + module->package_version = version; + + if (!create_sections(buf, size, §ion_list, error_buf, error_buf_size) + || !load_from_sections(module, section_list, true, wasm_binary_freeable, + no_resolve, error_buf, error_buf_size)) { + destroy_sections(section_list); + return false; + } + + destroy_sections(section_list); + return true; +fail: + return false; +} + +#if WASM_ENABLE_LIBC_WASI != 0 +/** + * refer to + * https://github.com/WebAssembly/WASI/blob/main/design/application-abi.md + */ +static bool +check_wasi_abi_compatibility(const WASMModule *module, +#if WASM_ENABLE_MULTI_MODULE != 0 + bool main_module, +#endif + char *error_buf, uint32 error_buf_size) +{ + /** + * be careful with: + * wasi compatible modules(command/reactor) which don't import any wasi + * APIs. Usually, a command has to import a "prox_exit" at least, but a + * reactor can depend on nothing. At the same time, each has its own entry + * point. + * + * observations: + * - clang always injects `_start` into a command + * - clang always injects `_initialize` into a reactor + * - `iwasm -f` allows to run a function in the reactor + * + * strong assumptions: + * - no one will define either `_start` or `_initialize` on purpose + * - `_start` should always be `void _start(void)` + * - `_initialize` should always be `void _initialize(void)` + * + */ + + /* clang-format off */ + /** + * + * | | import_wasi_api True | | import_wasi_api False | | + * | ----------- | -------------------- | ---------------- | --------------------- | ---------------- | + * | | \_initialize() Y | \_initialize() N | \_initialize() Y | \_initialize() N | + * | \_start() Y | N | COMMANDER | N | COMMANDER | + * | \_start() N | REACTOR | N | REACTOR | OTHERS | + */ + /* clang-format on */ + + WASMExport *initialize = NULL, *memory = NULL, *start = NULL; + uint32 import_function_count = module->import_function_count; + WASMFuncType *func_type; + + /* (func (export "_start") (...) */ + start = wasm_loader_find_export(module, "", "_start", EXPORT_KIND_FUNC, + error_buf, error_buf_size); + if (start) { + if (start->index < import_function_count) { + set_error_buf( + error_buf, error_buf_size, + "the builtin _start function can not be an import function"); + return false; + } + + func_type = + module->functions[start->index - import_function_count]->func_type; + if (func_type->param_count || func_type->result_count) { + set_error_buf(error_buf, error_buf_size, + "the signature of builtin _start function is wrong"); + return false; + } + } + else { + /* (func (export "_initialize") (...) */ + initialize = + wasm_loader_find_export(module, "", "_initialize", EXPORT_KIND_FUNC, + error_buf, error_buf_size); + + if (initialize) { + if (initialize->index < import_function_count) { + set_error_buf(error_buf, error_buf_size, + "the builtin _initialize function can not be an " + "import function"); + return false; + } + + func_type = + module->functions[initialize->index - import_function_count] + ->func_type; + if (func_type->param_count || func_type->result_count) { + set_error_buf( + error_buf, error_buf_size, + "the signature of builtin _initialize function is wrong"); + return false; + } + } + } + + /* filter out non-wasi compatible modules */ + if (!module->import_wasi_api && !start && !initialize) { + return true; + } + + /* should have one at least */ + if (module->import_wasi_api && !start && !initialize) { + LOG_WARNING("warning: a module with WASI apis should be either " + "a command or a reactor"); + } + + /* + * there is at least one of `_start` and `_initialize` in below cases. + * according to the assumption, they should be all wasi compatible + */ + +#if WASM_ENABLE_MULTI_MODULE != 0 + /* filter out commands (with `_start`) cases */ + if (start && !main_module) { + set_error_buf( + error_buf, error_buf_size, + "a command (with _start function) can not be a sub-module"); + return false; + } +#endif + + /* + * it is ok a reactor acts as a main module, + * so skip the check about (with `_initialize`) + */ + + memory = wasm_loader_find_export(module, "", "memory", EXPORT_KIND_MEMORY, + error_buf, error_buf_size); + if (!memory +#if WASM_ENABLE_LIB_WASI_THREADS != 0 + /* + * with wasi-threads, it's still an open question if a memory + * should be exported. + * + * https://github.com/WebAssembly/wasi-threads/issues/22 + * https://github.com/WebAssembly/WASI/issues/502 + * + * Note: this code assumes the number of memories is at most 1. + */ + && module->import_memory_count == 0 +#endif + ) { + set_error_buf(error_buf, error_buf_size, + "a module with WASI apis must export memory by default"); + return false; + } + + return true; +} +#endif + +WASMModule * +wasm_loader_load(uint8 *buf, uint32 size, +#if WASM_ENABLE_MULTI_MODULE != 0 + bool main_module, +#endif + const LoadArgs *args, char *error_buf, uint32 error_buf_size) +{ + WASMModule *module = create_module(args->name, error_buf, error_buf_size); + if (!module) { + return NULL; + } + +#if WASM_ENABLE_DEBUG_INTERP != 0 || WASM_ENABLE_FAST_JIT != 0 \ + || WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_JIT != 0 + module->load_addr = (uint8 *)buf; + module->load_size = size; +#endif + + if (!load(buf, size, module, args->wasm_binary_freeable, args->no_resolve, + error_buf, error_buf_size)) { + goto fail; + } + +#if WASM_ENABLE_LIBC_WASI != 0 + /* Check the WASI application ABI */ + if (!check_wasi_abi_compatibility(module, +#if WASM_ENABLE_MULTI_MODULE != 0 + main_module, +#endif + error_buf, error_buf_size)) { + goto fail; + } +#endif + + LOG_VERBOSE("Load module success.\n"); + return module; + +fail: + wasm_loader_unload(module); + return NULL; +} + +void +wasm_loader_unload(WASMModule *module) +{ + uint32 i; + + if (!module) + return; + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + module->orcjit_stop_compiling = true; + if (module->llvm_jit_init_thread) + os_thread_join(module->llvm_jit_init_thread, NULL); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 + /* Stop Fast/LLVM JIT compilation firstly to avoid accessing + module internal data after they were freed */ + orcjit_stop_compile_threads(module); +#endif + +#if WASM_ENABLE_JIT != 0 + if (module->func_ptrs) + wasm_runtime_free(module->func_ptrs); + if (module->comp_ctx) + aot_destroy_comp_context(module->comp_ctx); + if (module->comp_data) + aot_destroy_comp_data(module->comp_data); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + if (module->tierup_wait_lock_inited) { + os_mutex_destroy(&module->tierup_wait_lock); + os_cond_destroy(&module->tierup_wait_cond); + } +#endif + + if (module->imports) + wasm_runtime_free(module->imports); + + if (module->functions) { + for (i = 0; i < module->function_count; i++) { + if (module->functions[i]) { + if (module->functions[i]->local_offsets) + wasm_runtime_free(module->functions[i]->local_offsets); +#if WASM_ENABLE_FAST_INTERP != 0 + if (module->functions[i]->code_compiled) + wasm_runtime_free(module->functions[i]->code_compiled); + if (module->functions[i]->consts) + wasm_runtime_free(module->functions[i]->consts); +#endif +#if WASM_ENABLE_FAST_JIT != 0 + if (module->functions[i]->fast_jit_jitted_code) { + jit_code_cache_free( + module->functions[i]->fast_jit_jitted_code); + } +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0 + if (module->functions[i]->call_to_fast_jit_from_llvm_jit) { + jit_code_cache_free( + module->functions[i]->call_to_fast_jit_from_llvm_jit); + } +#endif +#endif +#if WASM_ENABLE_GC != 0 + if (module->functions[i]->local_ref_type_maps) { + wasm_runtime_free( + module->functions[i]->local_ref_type_maps); + } +#endif + wasm_runtime_free(module->functions[i]); + } + } + wasm_runtime_free(module->functions); + } + + if (module->tables) { +#if WASM_ENABLE_GC != 0 + for (i = 0; i < module->table_count; i++) { + destroy_init_expr(module, &module->tables[i].init_expr); + } +#endif + wasm_runtime_free(module->tables); + } + + if (module->memories) + wasm_runtime_free(module->memories); + + if (module->globals) { +#if WASM_ENABLE_GC != 0 || WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + for (i = 0; i < module->global_count; i++) { + destroy_init_expr(module, &module->globals[i].init_expr); + } +#endif + wasm_runtime_free(module->globals); + } + +#if WASM_ENABLE_TAGS != 0 + if (module->tags) { + for (i = 0; i < module->tag_count; i++) { + if (module->tags[i]) + wasm_runtime_free(module->tags[i]); + } + wasm_runtime_free(module->tags); + } +#endif + + if (module->exports) + wasm_runtime_free(module->exports); + + if (module->table_segments) { + for (i = 0; i < module->table_seg_count; i++) { + if (module->table_segments[i].init_values) { +#if WASM_ENABLE_GC != 0 + uint32 j; + for (j = 0; j < module->table_segments[i].value_count; j++) { + destroy_init_expr( + module, &module->table_segments[i].init_values[j]); + } +#endif + wasm_runtime_free(module->table_segments[i].init_values); + } +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + destroy_init_expr(module, &module->table_segments[i].base_offset); +#endif + } + wasm_runtime_free(module->table_segments); + } + + if (module->data_segments) { + for (i = 0; i < module->data_seg_count; i++) { + if (module->data_segments[i]) { + if (module->data_segments[i]->is_data_cloned) + wasm_runtime_free(module->data_segments[i]->data); +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + destroy_init_expr(module, + &(module->data_segments[i]->base_offset)); +#endif + wasm_runtime_free(module->data_segments[i]); + } + } + wasm_runtime_free(module->data_segments); + } + + if (module->types) { + for (i = 0; i < module->type_count; i++) { + if (module->types[i]) + destroy_wasm_type(module->types[i]); + } + wasm_runtime_free(module->types); + } + + if (module->const_str_list) { + StringNode *node = module->const_str_list, *node_next; + while (node) { + node_next = node->next; + wasm_runtime_free(node); + node = node_next; + } + } + +#if WASM_ENABLE_STRINGREF != 0 + if (module->string_literal_ptrs) { + wasm_runtime_free((void *)module->string_literal_ptrs); + } + if (module->string_literal_lengths) { + wasm_runtime_free(module->string_literal_lengths); + } +#endif + +#if WASM_ENABLE_FAST_INTERP == 0 + if (module->br_table_cache_list) { + BrTableCache *node = bh_list_first_elem(module->br_table_cache_list); + BrTableCache *node_next; + while (node) { + node_next = bh_list_elem_next(node); + wasm_runtime_free(node); + node = node_next; + } + } +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + /* just release the sub module list */ + if (module->import_module_list) { + WASMRegisteredModule *node = + bh_list_first_elem(module->import_module_list); + while (node) { + WASMRegisteredModule *next = bh_list_elem_next(node); + bh_list_remove(module->import_module_list, node); + /* + * unload(sub_module) will be triggered during runtime_destroy(). + * every module in the global module list will be unloaded one by + * one. so don't worry. + */ + wasm_runtime_free(node); + /* + * the module file reading buffer will be released + * in runtime_destroy() + */ + node = next; + } + } +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 + WASMFastOPCodeNode *fast_opcode = + bh_list_first_elem(&module->fast_opcode_list); + while (fast_opcode) { + WASMFastOPCodeNode *next = bh_list_elem_next(fast_opcode); + wasm_runtime_free(fast_opcode); + fast_opcode = next; + } +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 \ + || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0) + os_mutex_destroy(&module->instance_list_lock); +#endif + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 + wasm_runtime_destroy_custom_sections(module->custom_section_list); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 + if (module->fast_jit_func_ptrs) { + wasm_runtime_free(module->fast_jit_func_ptrs); + } + + for (i = 0; i < WASM_ORC_JIT_BACKEND_THREAD_NUM; i++) { + if (module->fast_jit_thread_locks_inited[i]) { + os_mutex_destroy(&module->fast_jit_thread_locks[i]); + } + } +#endif + +#if WASM_ENABLE_GC != 0 + os_mutex_destroy(&module->rtt_type_lock); + bh_hash_map_destroy(module->ref_type_set); + if (module->rtt_types) { + for (i = 0; i < module->type_count; i++) { + if (module->rtt_types[i]) + wasm_runtime_free(module->rtt_types[i]); + } + wasm_runtime_free(module->rtt_types); + } +#if WASM_ENABLE_STRINGREF != 0 + for (i = 0; i < WASM_TYPE_STRINGVIEWITER - WASM_TYPE_STRINGREF + 1; i++) { + if (module->stringref_rtts[i]) + wasm_runtime_free(module->stringref_rtts[i]); + } +#endif +#endif +#if WASM_ENABLE_BRANCH_HINTS != 0 + for (i = 0; i < module->function_count; i++) { + // be carefull when adding more hints. This only works as long as + // the hint structs have been allocated all at once as an array. + // With only branch-hints at the moment, this is the case. + if (module->function_hints != NULL && module->function_hints[i] != NULL) + wasm_runtime_free(module->function_hints[i]); + } + if (module->function_hints != NULL) + wasm_runtime_free(module->function_hints); +#endif + wasm_runtime_free(module); +} + +bool +wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, + const uint8 *start_addr, const uint8 *code_end_addr, + uint8 label_type, uint8 **p_else_addr, + uint8 **p_end_addr) +{ + const uint8 *p = start_addr, *p_end = code_end_addr; + uint8 *else_addr = NULL; + char error_buf[128]; + uint32 block_nested_depth = 1, count, i, j, t; + uint32 error_buf_size = sizeof(error_buf); + uint8 opcode, u8; + BlockAddr block_stack[16] = { { 0 } }, *block; + + i = ((uintptr_t)start_addr) & (uintptr_t)(BLOCK_ADDR_CACHE_SIZE - 1); + block = block_addr_cache + BLOCK_ADDR_CONFLICT_SIZE * i; + + for (j = 0; j < BLOCK_ADDR_CONFLICT_SIZE; j++) { + if (block[j].start_addr == start_addr) { + /* Cache hit */ + *p_else_addr = block[j].else_addr; + *p_end_addr = block[j].end_addr; + return true; + } + } + + /* Cache unhit */ + block_stack[0].start_addr = start_addr; + + while (p < code_end_addr) { + opcode = *p++; +#if WASM_ENABLE_DEBUG_INTERP != 0 + op_break_retry: +#endif + switch (opcode) { + case WASM_OP_UNREACHABLE: + case WASM_OP_NOP: + break; + +#if WASM_ENABLE_EXCE_HANDLING != 0 + case WASM_OP_TRY: + u8 = read_uint8(p); + if (block_nested_depth + < sizeof(block_stack) / sizeof(BlockAddr)) { + block_stack[block_nested_depth].start_addr = p; + block_stack[block_nested_depth].else_addr = NULL; + } + block_nested_depth++; + break; + case EXT_OP_TRY: + skip_leb_uint32(p, p_end); + if (block_nested_depth + < sizeof(block_stack) / sizeof(BlockAddr)) { + block_stack[block_nested_depth].start_addr = p; + block_stack[block_nested_depth].else_addr = NULL; + } + block_nested_depth++; + break; + case WASM_OP_CATCH: + if (block_nested_depth == 1) { + *p_end_addr = (uint8 *)(p - 1); + /* stop search and return the address of the catch block */ + return true; + } + break; + case WASM_OP_CATCH_ALL: + if (block_nested_depth == 1) { + *p_end_addr = (uint8 *)(p - 1); + /* stop search and return the address of the catch_all block + */ + return true; + } + break; + case WASM_OP_THROW: + /* skip tag_index */ + skip_leb(p); + break; + case WASM_OP_RETHROW: + /* skip depth */ + skip_leb(p); + break; + case WASM_OP_DELEGATE: + if (block_nested_depth == 1) { + *p_end_addr = (uint8 *)(p - 1); + return true; + } + else { + skip_leb(p); + /* the DELEGATE opcode ends the tryblock, */ + block_nested_depth--; + if (block_nested_depth + < sizeof(block_stack) / sizeof(BlockAddr)) + block_stack[block_nested_depth].end_addr = + (uint8 *)(p - 1); + } + break; +#endif /* end of WASM_ENABLE_EXCE_HANDLING != 0 */ + + case WASM_OP_BLOCK: + case WASM_OP_LOOP: + case WASM_OP_IF: + { + /* block result type: 0x40/0x7F/0x7E/0x7D/0x7C */ + u8 = read_uint8(p); + if (is_byte_a_type(u8)) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(u8)) { + /* the possible extra bytes of GC ref type have been + modified to OP_NOP, no need to resolve them again */ + } +#endif + } + else { + p--; + /* block type */ + skip_leb_int32(p, p_end); + } + if (block_nested_depth + < sizeof(block_stack) / sizeof(BlockAddr)) { + block_stack[block_nested_depth].start_addr = p; + block_stack[block_nested_depth].else_addr = NULL; + } + block_nested_depth++; + break; + } + + case EXT_OP_BLOCK: + case EXT_OP_LOOP: + case EXT_OP_IF: + /* block type */ + skip_leb_int32(p, p_end); + if (block_nested_depth + < sizeof(block_stack) / sizeof(BlockAddr)) { + block_stack[block_nested_depth].start_addr = p; + block_stack[block_nested_depth].else_addr = NULL; + } + block_nested_depth++; + break; + + case WASM_OP_ELSE: + if (label_type == LABEL_TYPE_IF && block_nested_depth == 1) + else_addr = (uint8 *)(p - 1); + if (block_nested_depth - 1 + < sizeof(block_stack) / sizeof(BlockAddr)) + block_stack[block_nested_depth - 1].else_addr = + (uint8 *)(p - 1); + break; + + case WASM_OP_END: + if (block_nested_depth == 1) { + if (label_type == LABEL_TYPE_IF) + *p_else_addr = else_addr; + *p_end_addr = (uint8 *)(p - 1); + + block_stack[0].end_addr = (uint8 *)(p - 1); + for (t = 0; t < sizeof(block_stack) / sizeof(BlockAddr); + t++) { + start_addr = block_stack[t].start_addr; + if (start_addr) { + i = ((uintptr_t)start_addr) + & (uintptr_t)(BLOCK_ADDR_CACHE_SIZE - 1); + block = + block_addr_cache + BLOCK_ADDR_CONFLICT_SIZE * i; + for (j = 0; j < BLOCK_ADDR_CONFLICT_SIZE; j++) + if (!block[j].start_addr) + break; + + if (j == BLOCK_ADDR_CONFLICT_SIZE) { + memmove(block + 1, block, + (BLOCK_ADDR_CONFLICT_SIZE - 1) + * sizeof(BlockAddr)); + j = 0; + } + block[j].start_addr = block_stack[t].start_addr; + block[j].else_addr = block_stack[t].else_addr; + block[j].end_addr = block_stack[t].end_addr; + } + else + break; + } + return true; + } + else { + block_nested_depth--; + if (block_nested_depth + < sizeof(block_stack) / sizeof(BlockAddr)) + block_stack[block_nested_depth].end_addr = + (uint8 *)(p - 1); + } + break; + + case WASM_OP_BR: + case WASM_OP_BR_IF: + skip_leb_uint32(p, p_end); /* labelidx */ + break; + + case WASM_OP_BR_TABLE: + read_leb_uint32(p, p_end, count); /* label num */ +#if WASM_ENABLE_FAST_INTERP != 0 + for (i = 0; i <= count; i++) /* labelidxs */ + skip_leb_uint32(p, p_end); +#else + p += count + 1; + while (*p == WASM_OP_NOP) + p++; +#endif + break; + +#if WASM_ENABLE_FAST_INTERP == 0 + case EXT_OP_BR_TABLE_CACHE: + read_leb_uint32(p, p_end, count); /* label num */ + while (*p == WASM_OP_NOP) + p++; + break; +#endif + + case WASM_OP_RETURN: + break; + + case WASM_OP_CALL: +#if WASM_ENABLE_TAIL_CALL != 0 + case WASM_OP_RETURN_CALL: +#endif + skip_leb_uint32(p, p_end); /* funcidx */ + break; + + case WASM_OP_CALL_INDIRECT: +#if WASM_ENABLE_TAIL_CALL != 0 + case WASM_OP_RETURN_CALL_INDIRECT: +#endif + skip_leb_uint32(p, p_end); /* typeidx */ +#if WASM_ENABLE_CALL_INDIRECT_OVERLONG != 0 || WASM_ENABLE_GC != 0 + skip_leb_uint32(p, p_end); /* tableidx */ +#else + u8 = read_uint8(p); /* 0x00 */ +#endif + break; + +#if WASM_ENABLE_GC != 0 + case WASM_OP_CALL_REF: + case WASM_OP_RETURN_CALL_REF: + skip_leb_uint32(p, p_end); /* typeidx */ + break; +#endif + + case WASM_OP_DROP: + case WASM_OP_SELECT: + case WASM_OP_DROP_64: + case WASM_OP_SELECT_64: +#if WASM_ENABLE_SIMDE != 0 + case WASM_OP_SELECT_128: +#endif + break; + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + case WASM_OP_SELECT_T: + { + skip_leb_uint32(p, p_end); /* vec length */ + u8 = read_uint8(p); /* typeidx */ + /* the possible extra bytes of GC ref type have been + modified to OP_NOP, no need to resolve them again */ + break; + } + + case WASM_OP_TABLE_GET: + case WASM_OP_TABLE_SET: + skip_leb_uint32(p, p_end); /* table index */ + break; + case WASM_OP_REF_NULL: + { + u8 = read_uint8(p); /* type */ + if (is_byte_a_type(u8)) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(u8)) { + /* the possible extra bytes of GC ref type have been + modified to OP_NOP, no need to resolve them again */ + } +#endif + } + else { + p--; + skip_leb_uint32(p, p_end); + } + break; + } + case WASM_OP_REF_IS_NULL: + break; + case WASM_OP_REF_FUNC: + skip_leb_uint32(p, p_end); /* func index */ + break; +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_GC != 0 + case WASM_OP_REF_AS_NON_NULL: + case WASM_OP_REF_EQ: + break; + case WASM_OP_BR_ON_NULL: + case WASM_OP_BR_ON_NON_NULL: + skip_leb_uint32(p, p_end); /* label index */ + break; +#endif /* end of WASM_ENABLE_GC != 0 */ + + case WASM_OP_GET_LOCAL: + case WASM_OP_SET_LOCAL: + case WASM_OP_TEE_LOCAL: + case WASM_OP_GET_GLOBAL: + case WASM_OP_SET_GLOBAL: + case WASM_OP_GET_GLOBAL_64: + case WASM_OP_SET_GLOBAL_64: +#if WASM_ENABLE_SIMDE != 0 + case WASM_OP_GET_GLOBAL_V128: + case WASM_OP_SET_GLOBAL_V128: +#endif + case WASM_OP_SET_GLOBAL_AUX_STACK: + skip_leb_uint32(p, p_end); /* local index */ + break; + + case EXT_OP_GET_LOCAL_FAST: + case EXT_OP_SET_LOCAL_FAST: + case EXT_OP_TEE_LOCAL_FAST: + CHECK_BUF(p, p_end, 1); + p++; + break; + + case WASM_OP_I32_LOAD: + case WASM_OP_I64_LOAD: + case WASM_OP_F32_LOAD: + case WASM_OP_F64_LOAD: + case WASM_OP_I32_LOAD8_S: + case WASM_OP_I32_LOAD8_U: + case WASM_OP_I32_LOAD16_S: + case WASM_OP_I32_LOAD16_U: + case WASM_OP_I64_LOAD8_S: + case WASM_OP_I64_LOAD8_U: + case WASM_OP_I64_LOAD16_S: + case WASM_OP_I64_LOAD16_U: + case WASM_OP_I64_LOAD32_S: + case WASM_OP_I64_LOAD32_U: + case WASM_OP_I32_STORE: + case WASM_OP_I64_STORE: + case WASM_OP_F32_STORE: + case WASM_OP_F64_STORE: + case WASM_OP_I32_STORE8: + case WASM_OP_I32_STORE16: + case WASM_OP_I64_STORE8: + case WASM_OP_I64_STORE16: + case WASM_OP_I64_STORE32: + skip_leb_align(p, p_end); /* align */ + skip_leb_mem_offset(p, p_end); /* offset */ + break; + + case WASM_OP_MEMORY_SIZE: + case WASM_OP_MEMORY_GROW: + skip_leb_memidx(p, p_end); /* memidx */ + break; + + case WASM_OP_I32_CONST: + skip_leb_int32(p, p_end); + break; + case WASM_OP_I64_CONST: + skip_leb_int64(p, p_end); + break; + case WASM_OP_F32_CONST: + p += sizeof(float32); + break; + case WASM_OP_F64_CONST: + p += sizeof(float64); + break; + + case WASM_OP_I32_EQZ: + case WASM_OP_I32_EQ: + case WASM_OP_I32_NE: + case WASM_OP_I32_LT_S: + case WASM_OP_I32_LT_U: + case WASM_OP_I32_GT_S: + case WASM_OP_I32_GT_U: + case WASM_OP_I32_LE_S: + case WASM_OP_I32_LE_U: + case WASM_OP_I32_GE_S: + case WASM_OP_I32_GE_U: + case WASM_OP_I64_EQZ: + case WASM_OP_I64_EQ: + case WASM_OP_I64_NE: + case WASM_OP_I64_LT_S: + case WASM_OP_I64_LT_U: + case WASM_OP_I64_GT_S: + case WASM_OP_I64_GT_U: + case WASM_OP_I64_LE_S: + case WASM_OP_I64_LE_U: + case WASM_OP_I64_GE_S: + case WASM_OP_I64_GE_U: + case WASM_OP_F32_EQ: + case WASM_OP_F32_NE: + case WASM_OP_F32_LT: + case WASM_OP_F32_GT: + case WASM_OP_F32_LE: + case WASM_OP_F32_GE: + case WASM_OP_F64_EQ: + case WASM_OP_F64_NE: + case WASM_OP_F64_LT: + case WASM_OP_F64_GT: + case WASM_OP_F64_LE: + case WASM_OP_F64_GE: + case WASM_OP_I32_CLZ: + case WASM_OP_I32_CTZ: + case WASM_OP_I32_POPCNT: + case WASM_OP_I32_ADD: + case WASM_OP_I32_SUB: + case WASM_OP_I32_MUL: + case WASM_OP_I32_DIV_S: + case WASM_OP_I32_DIV_U: + case WASM_OP_I32_REM_S: + case WASM_OP_I32_REM_U: + case WASM_OP_I32_AND: + case WASM_OP_I32_OR: + case WASM_OP_I32_XOR: + case WASM_OP_I32_SHL: + case WASM_OP_I32_SHR_S: + case WASM_OP_I32_SHR_U: + case WASM_OP_I32_ROTL: + case WASM_OP_I32_ROTR: + case WASM_OP_I64_CLZ: + case WASM_OP_I64_CTZ: + case WASM_OP_I64_POPCNT: + case WASM_OP_I64_ADD: + case WASM_OP_I64_SUB: + case WASM_OP_I64_MUL: + case WASM_OP_I64_DIV_S: + case WASM_OP_I64_DIV_U: + case WASM_OP_I64_REM_S: + case WASM_OP_I64_REM_U: + case WASM_OP_I64_AND: + case WASM_OP_I64_OR: + case WASM_OP_I64_XOR: + case WASM_OP_I64_SHL: + case WASM_OP_I64_SHR_S: + case WASM_OP_I64_SHR_U: + case WASM_OP_I64_ROTL: + case WASM_OP_I64_ROTR: + case WASM_OP_F32_ABS: + case WASM_OP_F32_NEG: + case WASM_OP_F32_CEIL: + case WASM_OP_F32_FLOOR: + case WASM_OP_F32_TRUNC: + case WASM_OP_F32_NEAREST: + case WASM_OP_F32_SQRT: + case WASM_OP_F32_ADD: + case WASM_OP_F32_SUB: + case WASM_OP_F32_MUL: + case WASM_OP_F32_DIV: + case WASM_OP_F32_MIN: + case WASM_OP_F32_MAX: + case WASM_OP_F32_COPYSIGN: + case WASM_OP_F64_ABS: + case WASM_OP_F64_NEG: + case WASM_OP_F64_CEIL: + case WASM_OP_F64_FLOOR: + case WASM_OP_F64_TRUNC: + case WASM_OP_F64_NEAREST: + case WASM_OP_F64_SQRT: + case WASM_OP_F64_ADD: + case WASM_OP_F64_SUB: + case WASM_OP_F64_MUL: + case WASM_OP_F64_DIV: + case WASM_OP_F64_MIN: + case WASM_OP_F64_MAX: + case WASM_OP_F64_COPYSIGN: + case WASM_OP_I32_WRAP_I64: + case WASM_OP_I32_TRUNC_S_F32: + case WASM_OP_I32_TRUNC_U_F32: + case WASM_OP_I32_TRUNC_S_F64: + case WASM_OP_I32_TRUNC_U_F64: + case WASM_OP_I64_EXTEND_S_I32: + case WASM_OP_I64_EXTEND_U_I32: + case WASM_OP_I64_TRUNC_S_F32: + case WASM_OP_I64_TRUNC_U_F32: + case WASM_OP_I64_TRUNC_S_F64: + case WASM_OP_I64_TRUNC_U_F64: + case WASM_OP_F32_CONVERT_S_I32: + case WASM_OP_F32_CONVERT_U_I32: + case WASM_OP_F32_CONVERT_S_I64: + case WASM_OP_F32_CONVERT_U_I64: + case WASM_OP_F32_DEMOTE_F64: + case WASM_OP_F64_CONVERT_S_I32: + case WASM_OP_F64_CONVERT_U_I32: + case WASM_OP_F64_CONVERT_S_I64: + case WASM_OP_F64_CONVERT_U_I64: + case WASM_OP_F64_PROMOTE_F32: + case WASM_OP_I32_REINTERPRET_F32: + case WASM_OP_I64_REINTERPRET_F64: + case WASM_OP_F32_REINTERPRET_I32: + case WASM_OP_F64_REINTERPRET_I64: + case WASM_OP_I32_EXTEND8_S: + case WASM_OP_I32_EXTEND16_S: + case WASM_OP_I64_EXTEND8_S: + case WASM_OP_I64_EXTEND16_S: + case WASM_OP_I64_EXTEND32_S: + break; + +#if WASM_ENABLE_GC != 0 + case WASM_OP_GC_PREFIX: + { + uint32 opcode1; + + read_leb_uint32(p, p_end, opcode1); + /* opcode1 was checked in wasm_loader_prepare_bytecode and + is no larger than UINT8_MAX */ + opcode = (uint8)opcode1; + + switch (opcode) { + case WASM_OP_STRUCT_NEW: + case WASM_OP_STRUCT_NEW_DEFAULT: + skip_leb_uint32(p, p_end); /* typeidx */ + break; + case WASM_OP_STRUCT_GET: + case WASM_OP_STRUCT_GET_S: + case WASM_OP_STRUCT_GET_U: + case WASM_OP_STRUCT_SET: + skip_leb_uint32(p, p_end); /* typeidx */ + skip_leb_uint32(p, p_end); /* fieldidx */ + break; + + case WASM_OP_ARRAY_NEW: + case WASM_OP_ARRAY_NEW_DEFAULT: + case WASM_OP_ARRAY_GET: + case WASM_OP_ARRAY_GET_S: + case WASM_OP_ARRAY_GET_U: + case WASM_OP_ARRAY_SET: + case WASM_OP_ARRAY_FILL: + skip_leb_uint32(p, p_end); /* typeidx */ + break; + case WASM_OP_ARRAY_COPY: + skip_leb_uint32(p, p_end); /* typeidx1 */ + skip_leb_uint32(p, p_end); /* typeidx2 */ + break; + case WASM_OP_ARRAY_LEN: + break; + case WASM_OP_ARRAY_NEW_FIXED: + case WASM_OP_ARRAY_NEW_DATA: + case WASM_OP_ARRAY_NEW_ELEM: + skip_leb_uint32(p, p_end); /* typeidx */ + skip_leb_uint32(p, p_end); /* N/dataidx/elemidx */ + break; + + case WASM_OP_REF_I31: + case WASM_OP_I31_GET_S: + case WASM_OP_I31_GET_U: + break; + + case WASM_OP_REF_TEST: + case WASM_OP_REF_CAST: + case WASM_OP_REF_TEST_NULLABLE: + case WASM_OP_REF_CAST_NULLABLE: + skip_leb_int32(p, p_end); /* heaptype */ + break; + case WASM_OP_BR_ON_CAST: + case WASM_OP_BR_ON_CAST_FAIL: + p += sizeof(uint8); /* castflag */ + skip_leb_uint32(p, p_end); /* labelidx */ + skip_leb_int32(p, p_end); /* heaptype */ + skip_leb_int32(p, p_end); /* heaptype2 */ + break; + + case WASM_OP_ANY_CONVERT_EXTERN: + case WASM_OP_EXTERN_CONVERT_ANY: + break; + +#if WASM_ENABLE_STRINGREF != 0 + case WASM_OP_STRING_NEW_UTF8: + case WASM_OP_STRING_NEW_WTF16: + case WASM_OP_STRING_NEW_LOSSY_UTF8: + case WASM_OP_STRING_NEW_WTF8: + skip_leb_uint32(p, p_end); /* memory index 0x00 */ + break; + case WASM_OP_STRING_CONST: + skip_leb_int32(p, p_end); /* contents */ + break; + case WASM_OP_STRING_MEASURE_UTF8: + case WASM_OP_STRING_MEASURE_WTF8: + case WASM_OP_STRING_MEASURE_WTF16: + break; + case WASM_OP_STRING_ENCODE_UTF8: + case WASM_OP_STRING_ENCODE_WTF16: + case WASM_OP_STRING_ENCODE_LOSSY_UTF8: + case WASM_OP_STRING_ENCODE_WTF8: + skip_leb_uint32(p, p_end); /* memory index 0x00 */ + break; + case WASM_OP_STRING_CONCAT: + case WASM_OP_STRING_EQ: + case WASM_OP_STRING_IS_USV_SEQUENCE: + case WASM_OP_STRING_AS_WTF8: + case WASM_OP_STRINGVIEW_WTF8_ADVANCE: + break; + case WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8: + case WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8: + case WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8: + skip_leb_uint32(p, p_end); /* memory index 0x00 */ + break; + case WASM_OP_STRINGVIEW_WTF8_SLICE: + case WASM_OP_STRING_AS_WTF16: + case WASM_OP_STRINGVIEW_WTF16_LENGTH: + case WASM_OP_STRINGVIEW_WTF16_GET_CODEUNIT: + break; + case WASM_OP_STRINGVIEW_WTF16_ENCODE: + skip_leb_uint32(p, p_end); /* memory index 0x00 */ + break; + case WASM_OP_STRINGVIEW_WTF16_SLICE: + case WASM_OP_STRING_AS_ITER: + case WASM_OP_STRINGVIEW_ITER_NEXT: + case WASM_OP_STRINGVIEW_ITER_ADVANCE: + case WASM_OP_STRINGVIEW_ITER_REWIND: + case WASM_OP_STRINGVIEW_ITER_SLICE: + case WASM_OP_STRING_NEW_UTF8_ARRAY: + case WASM_OP_STRING_NEW_WTF16_ARRAY: + case WASM_OP_STRING_NEW_LOSSY_UTF8_ARRAY: + case WASM_OP_STRING_NEW_WTF8_ARRAY: + case WASM_OP_STRING_ENCODE_UTF8_ARRAY: + case WASM_OP_STRING_ENCODE_WTF16_ARRAY: + case WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY: + case WASM_OP_STRING_ENCODE_WTF8_ARRAY: + break; +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ + default: + return false; + } + break; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + + case WASM_OP_MISC_PREFIX: + { + uint32 opcode1; + + read_leb_uint32(p, p_end, opcode1); + /* opcode1 was checked in wasm_loader_prepare_bytecode and + is no larger than UINT8_MAX */ + opcode = (uint8)opcode1; + + switch (opcode) { + case WASM_OP_I32_TRUNC_SAT_S_F32: + case WASM_OP_I32_TRUNC_SAT_U_F32: + case WASM_OP_I32_TRUNC_SAT_S_F64: + case WASM_OP_I32_TRUNC_SAT_U_F64: + case WASM_OP_I64_TRUNC_SAT_S_F32: + case WASM_OP_I64_TRUNC_SAT_U_F32: + case WASM_OP_I64_TRUNC_SAT_S_F64: + case WASM_OP_I64_TRUNC_SAT_U_F64: + break; +#if WASM_ENABLE_BULK_MEMORY != 0 + case WASM_OP_MEMORY_INIT: + skip_leb_uint32(p, p_end); + skip_leb_memidx(p, p_end); + break; + case WASM_OP_DATA_DROP: + skip_leb_uint32(p, p_end); + break; +#endif /* WASM_ENABLE_BULK_MEMORY */ +#if WASM_ENABLE_BULK_MEMORY_OPT != 0 + case WASM_OP_MEMORY_COPY: + skip_leb_memidx(p, p_end); + skip_leb_memidx(p, p_end); + break; + case WASM_OP_MEMORY_FILL: + skip_leb_memidx(p, p_end); + break; +#endif /* WASM_ENABLE_BULK_MEMORY_OPT */ +#if WASM_ENABLE_REF_TYPES != 0 + case WASM_OP_TABLE_INIT: + case WASM_OP_TABLE_COPY: + /* tableidx */ + skip_leb_uint32(p, p_end); + /* elemidx */ + skip_leb_uint32(p, p_end); + break; + case WASM_OP_ELEM_DROP: + /* elemidx */ + skip_leb_uint32(p, p_end); + break; + case WASM_OP_TABLE_SIZE: + case WASM_OP_TABLE_GROW: + case WASM_OP_TABLE_FILL: + skip_leb_uint32(p, p_end); /* table idx */ + break; +#endif /* WASM_ENABLE_REF_TYPES */ + default: + return false; + } + break; + } + +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) \ + || (WASM_ENABLE_FAST_INTERP != 0) + case WASM_OP_SIMD_PREFIX: + { + uint32 opcode1; + + read_leb_uint32(p, p_end, opcode1); + /* opcode1 was checked in wasm_loader_prepare_bytecode and + is no larger than UINT8_MAX */ + opcode = (uint8)opcode1; + + /* follow the order of enum WASMSimdEXTOpcode in wasm_opcode.h + */ + switch (opcode) { + case SIMD_v128_load: + case SIMD_v128_load8x8_s: + case SIMD_v128_load8x8_u: + case SIMD_v128_load16x4_s: + case SIMD_v128_load16x4_u: + case SIMD_v128_load32x2_s: + case SIMD_v128_load32x2_u: + case SIMD_v128_load8_splat: + case SIMD_v128_load16_splat: + case SIMD_v128_load32_splat: + case SIMD_v128_load64_splat: + case SIMD_v128_store: + /* memarg align */ + skip_leb_uint32(p, p_end); + /* memarg offset */ + skip_leb_mem_offset(p, p_end); + break; + + case SIMD_v128_const: + case SIMD_v8x16_shuffle: + /* immByte[16] immLaneId[16] */ + CHECK_BUF1(p, p_end, 16); + p += 16; + break; + + case SIMD_i8x16_extract_lane_s: + case SIMD_i8x16_extract_lane_u: + case SIMD_i8x16_replace_lane: + case SIMD_i16x8_extract_lane_s: + case SIMD_i16x8_extract_lane_u: + case SIMD_i16x8_replace_lane: + case SIMD_i32x4_extract_lane: + case SIMD_i32x4_replace_lane: + case SIMD_i64x2_extract_lane: + case SIMD_i64x2_replace_lane: + case SIMD_f32x4_extract_lane: + case SIMD_f32x4_replace_lane: + case SIMD_f64x2_extract_lane: + case SIMD_f64x2_replace_lane: + /* ImmLaneId */ + CHECK_BUF(p, p_end, 1); + p++; + break; + + case SIMD_v128_load8_lane: + case SIMD_v128_load16_lane: + case SIMD_v128_load32_lane: + case SIMD_v128_load64_lane: + case SIMD_v128_store8_lane: + case SIMD_v128_store16_lane: + case SIMD_v128_store32_lane: + case SIMD_v128_store64_lane: + /* memarg align */ + skip_leb_uint32(p, p_end); + /* memarg offset */ + skip_leb_mem_offset(p, p_end); + /* ImmLaneId */ + CHECK_BUF(p, p_end, 1); + p++; + break; + + case SIMD_v128_load32_zero: + case SIMD_v128_load64_zero: + /* memarg align */ + skip_leb_uint32(p, p_end); + /* memarg offset */ + skip_leb_mem_offset(p, p_end); + break; + + default: + /* + * since latest SIMD specific used almost every value + * from 0x00 to 0xff, the default branch will present + * all opcodes without imm + * https://github.com/WebAssembly/simd/blob/main/proposals/simd/NewOpcodes.md + */ + break; + } + break; + } +#endif /* end of (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) || \ + (WASM_ENABLE_FAST_INTERP != 0) */ +#endif /* end of WASM_ENABLE_SIMD */ + +#if WASM_ENABLE_SHARED_MEMORY != 0 + case WASM_OP_ATOMIC_PREFIX: + { + uint32 opcode1; + + /* atomic_op (u32_leb) + memarg (2 u32_leb) */ + read_leb_uint32(p, p_end, opcode1); + /* opcode1 was checked in wasm_loader_prepare_bytecode and + is no larger than UINT8_MAX */ + opcode = (uint8)opcode1; + + if (opcode != WASM_OP_ATOMIC_FENCE) { + skip_leb_uint32(p, p_end); /* align */ + skip_leb_mem_offset(p, p_end); /* offset */ + } + else { + /* atomic.fence doesn't have memarg */ + p++; + } + break; + } +#endif +#if WASM_ENABLE_DEBUG_INTERP != 0 + case DEBUG_OP_BREAK: + { + WASMDebugInstance *debug_instance = + wasm_exec_env_get_instance(exec_env); + char original_opcode[1]; + uint64 size = 1; + WASMModuleInstance *module_inst = + (WASMModuleInstance *)exec_env->module_inst; + uint64 offset = (p - 1) >= module_inst->module->load_addr + ? (p - 1) - module_inst->module->load_addr + : ~0; + if (debug_instance) { + if (wasm_debug_instance_get_obj_mem(debug_instance, offset, + original_opcode, &size) + && size == 1) { + LOG_VERBOSE("WASM loader find OP_BREAK , recover it " + "with %02x: ", + original_opcode[0]); + opcode = original_opcode[0]; + goto op_break_retry; + } + } + break; + } +#endif + + default: + return false; + } + } + + (void)u8; + (void)exec_env; + return false; +fail: + return false; +} + +#if WASM_ENABLE_FAST_INTERP != 0 + +#if WASM_DEBUG_PREPROCESSOR != 0 +#define LOG_OP(...) os_printf(__VA_ARGS__) +#else +#define LOG_OP(...) (void)0 +#endif + +#define PATCH_ELSE 0 +#define PATCH_END 1 +typedef struct BranchBlockPatch { + struct BranchBlockPatch *next; + uint8 patch_type; + uint8 *code_compiled; +} BranchBlockPatch; +#endif + +typedef struct BranchBlock { + uint8 label_type; + BlockType block_type; + uint8 *start_addr; + uint8 *else_addr; + uint8 *end_addr; + uint32 stack_cell_num; +#if WASM_ENABLE_GC != 0 + uint32 reftype_map_num; + /* Indicate which local is used inside current block, used to validate + * local.get with non-nullable ref types */ + uint8 *local_use_mask; + uint32 local_use_mask_size; +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + uint16 dynamic_offset; + uint8 *code_compiled; + BranchBlockPatch *patch_list; + /* This is used to save params frame_offset of of if block */ + int16 *param_frame_offsets; + /* This is used to recover the dynamic offset for else branch, + * and also to remember the start offset of dynamic space which + * stores the block arguments for loop block, so we can use it + * to copy the stack operands to the loop block's arguments in + * wasm_loader_emit_br_info for opcode br. */ + uint16 start_dynamic_offset; +#endif + + /* Indicate the operand stack is in polymorphic state. + * If the opcode is one of unreachable/br/br_table/return, stack is marked + * to polymorphic state until the block's 'end' opcode is processed. + * If stack is in polymorphic state and stack is empty, instruction can + * pop any type of value directly without decreasing stack top pointer + * and stack cell num. */ + bool is_stack_polymorphic; +} BranchBlock; + +typedef struct WASMLoaderContext { + /* frame ref stack */ + uint8 *frame_ref; + uint8 *frame_ref_bottom; + uint8 *frame_ref_boundary; + uint32 frame_ref_size; + uint32 stack_cell_num; + uint32 max_stack_cell_num; + +#if WASM_ENABLE_GC != 0 + /* frame reftype map stack */ + WASMRefTypeMap *frame_reftype_map; + WASMRefTypeMap *frame_reftype_map_bottom; + WASMRefTypeMap *frame_reftype_map_boundary; + uint32 frame_reftype_map_size; + uint32 reftype_map_num; + uint32 max_reftype_map_num; + /* Current module */ + WASMModule *module; + /* Current module's ref_type_set */ + HashMap *ref_type_set; + /* Always point to local variable ref_type of + wasm_loader_prepare_bytecode */ + WASMRefType *ref_type_tmp; +#endif + + /* frame csp stack */ + BranchBlock *frame_csp; + BranchBlock *frame_csp_bottom; + BranchBlock *frame_csp_boundary; + uint32 frame_csp_size; + uint32 csp_num; + uint32 max_csp_num; + +#if WASM_ENABLE_FAST_INTERP != 0 + /* frame offset stack */ + int16 *frame_offset; + int16 *frame_offset_bottom; + int16 *frame_offset_boundary; + uint32 frame_offset_size; + int16 dynamic_offset; + int16 start_dynamic_offset; + int16 max_dynamic_offset; + + /* preserved local offset */ + int16 preserved_local_offset; + + /* const buffer for i64 and f64 consts, note that the raw bytes + * of i64 and f64 are the same, so we read an i64 value from an + * f64 const with its raw bytes, something like `*(int64 *)&f64 */ + int64 *i64_consts; + uint32 i64_const_max_num; + uint32 i64_const_num; + /* const buffer for i32 and f32 consts */ + int32 *i32_consts; + uint32 i32_const_max_num; + uint32 i32_const_num; + /* const buffer for V128 */ + V128 *v128_consts; + uint32 v128_const_max_num; + uint32 v128_const_num; + + /* processed code */ + uint8 *p_code_compiled; + uint8 *p_code_compiled_end; + uint32 code_compiled_size; + /* If the last opcode will be dropped, the peak memory usage will be larger + * than the final code_compiled_size, we record the peak size to ensure + * there will not be invalid memory access during second traverse */ + uint32 code_compiled_peak_size; +#endif +} WASMLoaderContext; + +#define CHECK_CSP_PUSH() \ + do { \ + if (ctx->frame_csp >= ctx->frame_csp_boundary) { \ + MEM_REALLOC( \ + ctx->frame_csp_bottom, ctx->frame_csp_size, \ + (uint32)(ctx->frame_csp_size + 8 * sizeof(BranchBlock))); \ + ctx->frame_csp_size += (uint32)(8 * sizeof(BranchBlock)); \ + ctx->frame_csp_boundary = \ + ctx->frame_csp_bottom \ + + ctx->frame_csp_size / sizeof(BranchBlock); \ + ctx->frame_csp = ctx->frame_csp_bottom + ctx->csp_num; \ + } \ + } while (0) + +#define CHECK_CSP_POP() \ + do { \ + if (ctx->csp_num < 1) { \ + set_error_buf(error_buf, error_buf_size, \ + "type mismatch: " \ + "expect data but block stack was empty"); \ + goto fail; \ + } \ + } while (0) + +#if WASM_ENABLE_FAST_INTERP != 0 +static bool +check_offset_push(WASMLoaderContext *ctx, char *error_buf, + uint32 error_buf_size) +{ + uint32 cell_num = (uint32)(ctx->frame_offset - ctx->frame_offset_bottom); + if (ctx->frame_offset >= ctx->frame_offset_boundary) { + MEM_REALLOC(ctx->frame_offset_bottom, ctx->frame_offset_size, + ctx->frame_offset_size + 16); + ctx->frame_offset_size += 16; + ctx->frame_offset_boundary = + ctx->frame_offset_bottom + ctx->frame_offset_size / sizeof(int16); + ctx->frame_offset = ctx->frame_offset_bottom + cell_num; + } + return true; +fail: + return false; +} + +static bool +check_offset_pop(WASMLoaderContext *ctx, uint32 cells) +{ + if (ctx->frame_offset - cells < ctx->frame_offset_bottom) + return false; + return true; +} + +static void +free_label_patch_list(BranchBlock *frame_csp) +{ + BranchBlockPatch *label_patch = frame_csp->patch_list; + BranchBlockPatch *next; + while (label_patch != NULL) { + next = label_patch->next; + wasm_runtime_free(label_patch); + label_patch = next; + } + frame_csp->patch_list = NULL; +} + +static void +free_all_label_patch_lists(BranchBlock *frame_csp, uint32 csp_num) +{ + BranchBlock *tmp_csp = frame_csp; + uint32 i; + + for (i = 0; i < csp_num; i++) { + free_label_patch_list(tmp_csp); + tmp_csp++; + } +} + +static void +free_all_label_param_frame_offsets(BranchBlock *frame_csp, uint32 csp_num) +{ + BranchBlock *tmp_csp = frame_csp; + uint32 i; + + for (i = 0; i < csp_num; i++) { + if (tmp_csp->param_frame_offsets) + wasm_runtime_free(tmp_csp->param_frame_offsets); + tmp_csp++; + } +} +#endif /* end of WASM_ENABLE_FAST_INTERP */ + +#if WASM_ENABLE_GC != 0 +static bool +wasm_loader_init_local_use_masks(WASMLoaderContext *ctx, uint32 local_count, + char *error_buf, uint32 error_buf_size) +{ + BranchBlock *current_csp = ctx->frame_csp - 1; + uint32 local_mask_size; + + if (local_count == 0) { + current_csp->local_use_mask_size = 0; + return true; + } + + /* if current_csp->local_use_mask is not NULL, then it is re-init masks for + * else branch, we don't need to allocate memory again */ + if (!current_csp->local_use_mask) { + local_mask_size = (local_count + 7) / sizeof(uint8); + if (!(current_csp->local_use_mask = + loader_malloc(local_mask_size, error_buf, error_buf_size))) { + return false; + } + current_csp->local_use_mask_size = local_mask_size; + } + else { + local_mask_size = current_csp->local_use_mask_size; + bh_assert(current_csp->label_type == LABEL_TYPE_IF); + } + + if (current_csp->label_type != LABEL_TYPE_FUNCTION) { + /* For non-function blocks, inherit the use status from parent block */ + BranchBlock *parent_csp = current_csp - 1; + + bh_assert(parent_csp >= ctx->frame_csp_bottom); + bh_assert(parent_csp->local_use_mask); + + bh_memcpy_s(current_csp->local_use_mask, local_mask_size, + parent_csp->local_use_mask, local_mask_size); + } + + return true; +} + +static void +wasm_loader_destroy_curr_local_use_masks(WASMLoaderContext *ctx) +{ + BranchBlock *current_csp = ctx->frame_csp - 1; + + bh_assert(current_csp->local_use_mask + || current_csp->local_use_mask_size == 0); + + if (current_csp->local_use_mask) { + wasm_runtime_free(current_csp->local_use_mask); + } + + current_csp->local_use_mask = NULL; + current_csp->local_use_mask_size = 0; +} + +static void +wasm_loader_clean_all_local_use_masks(WASMLoaderContext *ctx) +{ + BranchBlock *tmp_csp = ctx->frame_csp_bottom; + uint32 i; + + for (i = 0; i < ctx->csp_num; i++) { + if (tmp_csp->local_use_mask) { + wasm_runtime_free(tmp_csp->local_use_mask); + tmp_csp->local_use_mask = NULL; + tmp_csp->local_use_mask_size = 0; + } + tmp_csp++; + } +} + +static void +wasm_loader_mask_local(WASMLoaderContext *ctx, uint32 index) +{ + BranchBlock *current_csp = ctx->frame_csp - 1; + uint32 byte_offset = index / sizeof(uint8); + uint32 bit_offset = index % sizeof(uint8); + + bh_assert(byte_offset < current_csp->local_use_mask_size); + bh_assert(current_csp->local_use_mask); + + current_csp->local_use_mask[byte_offset] |= (1 << bit_offset); +} + +static bool +wasm_loader_get_local_status(WASMLoaderContext *ctx, uint32 index) +{ + BranchBlock *current_csp = ctx->frame_csp - 1; + uint32 byte_offset = index / sizeof(uint8); + uint32 bit_offset = index % sizeof(uint8); + + bh_assert(byte_offset < current_csp->local_use_mask_size); + bh_assert(current_csp->local_use_mask); + + return (current_csp->local_use_mask[byte_offset] & (1 << bit_offset)) + ? true + : false; +} +#endif /* end of WASM_ENABLE_GC != 0 */ + +static void +wasm_loader_ctx_destroy(WASMLoaderContext *ctx) +{ + if (ctx) { + if (ctx->frame_ref_bottom) + wasm_runtime_free(ctx->frame_ref_bottom); +#if WASM_ENABLE_GC != 0 + if (ctx->frame_reftype_map_bottom) + wasm_runtime_free(ctx->frame_reftype_map_bottom); +#endif + if (ctx->frame_csp_bottom) { +#if WASM_ENABLE_FAST_INTERP != 0 + free_all_label_patch_lists(ctx->frame_csp_bottom, ctx->csp_num); + free_all_label_param_frame_offsets(ctx->frame_csp_bottom, + ctx->csp_num); +#endif +#if WASM_ENABLE_GC != 0 + wasm_loader_clean_all_local_use_masks(ctx); +#endif + wasm_runtime_free(ctx->frame_csp_bottom); + } +#if WASM_ENABLE_FAST_INTERP != 0 + if (ctx->frame_offset_bottom) + wasm_runtime_free(ctx->frame_offset_bottom); + if (ctx->i64_consts) + wasm_runtime_free(ctx->i64_consts); + if (ctx->i32_consts) + wasm_runtime_free(ctx->i32_consts); + if (ctx->v128_consts) + wasm_runtime_free(ctx->v128_consts); +#endif + wasm_runtime_free(ctx); + } +} + +static WASMLoaderContext * +wasm_loader_ctx_init(WASMFunction *func, char *error_buf, uint32 error_buf_size) +{ + WASMLoaderContext *loader_ctx = + loader_malloc(sizeof(WASMLoaderContext), error_buf, error_buf_size); + if (!loader_ctx) + return NULL; + + loader_ctx->frame_ref_size = 32; + if (!(loader_ctx->frame_ref_bottom = loader_ctx->frame_ref = loader_malloc( + loader_ctx->frame_ref_size, error_buf, error_buf_size))) + goto fail; + loader_ctx->frame_ref_boundary = loader_ctx->frame_ref_bottom + 32; + +#if WASM_ENABLE_GC != 0 + loader_ctx->frame_reftype_map_size = sizeof(WASMRefTypeMap) * 16; + if (!(loader_ctx->frame_reftype_map_bottom = loader_ctx->frame_reftype_map = + loader_malloc(loader_ctx->frame_reftype_map_size, error_buf, + error_buf_size))) + goto fail; + loader_ctx->frame_reftype_map_boundary = + loader_ctx->frame_reftype_map_bottom + 16; +#endif + + loader_ctx->frame_csp_size = sizeof(BranchBlock) * 8; + if (!(loader_ctx->frame_csp_bottom = loader_ctx->frame_csp = loader_malloc( + loader_ctx->frame_csp_size, error_buf, error_buf_size))) + goto fail; + loader_ctx->frame_csp_boundary = loader_ctx->frame_csp_bottom + 8; + +#if WASM_ENABLE_EXCE_HANDLING != 0 + func->exception_handler_count = 0; +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + loader_ctx->frame_offset_size = sizeof(int16) * 32; + if (!(loader_ctx->frame_offset_bottom = loader_ctx->frame_offset = + loader_malloc(loader_ctx->frame_offset_size, error_buf, + error_buf_size))) + goto fail; + loader_ctx->frame_offset_boundary = loader_ctx->frame_offset_bottom + 32; + + loader_ctx->i64_const_max_num = 8; + if (!(loader_ctx->i64_consts = + loader_malloc(sizeof(int64) * loader_ctx->i64_const_max_num, + error_buf, error_buf_size))) + goto fail; + loader_ctx->i32_const_max_num = 8; + if (!(loader_ctx->i32_consts = + loader_malloc(sizeof(int32) * loader_ctx->i32_const_max_num, + error_buf, error_buf_size))) + goto fail; + loader_ctx->v128_const_max_num = 8; + if (!(loader_ctx->v128_consts = + loader_malloc(sizeof(V128) * loader_ctx->v128_const_max_num, + error_buf, error_buf_size))) + goto fail; + + if (func->param_cell_num >= (int32)INT16_MAX - func->local_cell_num) { + set_error_buf(error_buf, error_buf_size, + "fast interpreter offset overflow"); + goto fail; + } + + loader_ctx->start_dynamic_offset = loader_ctx->dynamic_offset = + loader_ctx->max_dynamic_offset = + func->param_cell_num + func->local_cell_num; +#endif + return loader_ctx; + +fail: + wasm_loader_ctx_destroy(loader_ctx); + return NULL; +} + +static bool +check_stack_push(WASMLoaderContext *ctx, uint8 type, char *error_buf, + uint32 error_buf_size) +{ + uint32 cell_num_needed = wasm_value_type_cell_num(type); + + if (ctx->frame_ref + cell_num_needed > ctx->frame_ref_boundary) { + /* Increase the frame ref stack */ + MEM_REALLOC(ctx->frame_ref_bottom, ctx->frame_ref_size, + ctx->frame_ref_size + 16); + ctx->frame_ref_size += 16; + ctx->frame_ref_boundary = ctx->frame_ref_bottom + ctx->frame_ref_size; + ctx->frame_ref = ctx->frame_ref_bottom + ctx->stack_cell_num; + } + +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(type) + && ctx->frame_reftype_map >= ctx->frame_reftype_map_boundary) { + /* Increase the frame reftype map stack */ + bh_assert( + (uint32)((ctx->frame_reftype_map - ctx->frame_reftype_map_bottom) + * sizeof(WASMRefTypeMap)) + == ctx->frame_reftype_map_size); + MEM_REALLOC(ctx->frame_reftype_map_bottom, ctx->frame_reftype_map_size, + ctx->frame_reftype_map_size + + (uint32)sizeof(WASMRefTypeMap) * 8); + ctx->frame_reftype_map = + ctx->frame_reftype_map_bottom + + ctx->frame_reftype_map_size / ((uint32)sizeof(WASMRefTypeMap)); + ctx->frame_reftype_map_size += (uint32)sizeof(WASMRefTypeMap) * 8; + ctx->frame_reftype_map_boundary = + ctx->frame_reftype_map_bottom + + ctx->frame_reftype_map_size / ((uint32)sizeof(WASMRefTypeMap)); + } +#endif + return true; +fail: + return false; +} + +static bool +wasm_loader_push_frame_ref(WASMLoaderContext *ctx, uint8 type, char *error_buf, + uint32 error_buf_size) +{ + uint32 type_cell_num = wasm_value_type_cell_num(type); + uint32 i; + + if (!check_stack_push(ctx, type, error_buf, error_buf_size)) + return false; + +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(type)) { + WASMRefType *ref_type; + if (!(ref_type = + reftype_set_insert(ctx->ref_type_set, ctx->ref_type_tmp, + error_buf, error_buf_size))) { + return false; + } + + if (ctx->frame_reftype_map >= ctx->frame_reftype_map_boundary) { + /* Increase the frame reftype map stack */ + bh_assert((uint32)((ctx->frame_reftype_map + - ctx->frame_reftype_map_bottom) + * sizeof(WASMRefTypeMap)) + == ctx->frame_reftype_map_size); + MEM_REALLOC(ctx->frame_reftype_map_bottom, + ctx->frame_reftype_map_size, + ctx->frame_reftype_map_size + + (uint32)sizeof(WASMRefTypeMap) * 8); + ctx->frame_reftype_map = ctx->frame_reftype_map_bottom + + ctx->frame_reftype_map_size + / ((uint32)sizeof(WASMRefTypeMap)); + ctx->frame_reftype_map_size += (uint32)sizeof(WASMRefTypeMap) * 8; + ctx->frame_reftype_map_boundary = + ctx->frame_reftype_map_bottom + + ctx->frame_reftype_map_size + / ((uint32)sizeof(WASMRefTypeMap)); + } + + ctx->frame_reftype_map->index = ctx->stack_cell_num; + ctx->frame_reftype_map->ref_type = ref_type; + ctx->frame_reftype_map++; + ctx->reftype_map_num++; + if (ctx->reftype_map_num > ctx->max_reftype_map_num) + ctx->max_reftype_map_num = ctx->reftype_map_num; + } +#endif + + for (i = 0; i < type_cell_num; i++) + *ctx->frame_ref++ = type; + ctx->stack_cell_num += type_cell_num; + + if (ctx->stack_cell_num > ctx->max_stack_cell_num) { + ctx->max_stack_cell_num = ctx->stack_cell_num; + if (ctx->max_stack_cell_num > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "operand stack depth limit exceeded"); + return false; + } + } + return true; +#if WASM_ENABLE_GC != 0 +fail: + return false; +#endif +} + +static bool +check_stack_top_values(WASMLoaderContext *ctx, uint8 *frame_ref, + int32 stack_cell_num, +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap *frame_reftype_map, int32 reftype_map_num, +#endif + uint8 type, +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type, +#endif + char *error_buf, uint32 error_buf_size) +{ + int32 type_cell_num = (int32)wasm_value_type_cell_num(type), i; +#if WASM_ENABLE_GC != 0 + WASMRefType *frame_reftype = NULL; +#endif + + if (stack_cell_num < type_cell_num) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: expect data but stack was empty"); + return false; + } + +#if WASM_ENABLE_GC == 0 + for (i = 0; i < type_cell_num; i++) { + if (*(frame_ref - 1 - i) != type) { + set_error_buf_v(error_buf, error_buf_size, "%s%s%s", + "type mismatch: expect ", type2str(type), + " but got other"); + return false; + } + } +#else + if (wasm_is_type_multi_byte_type(*(frame_ref - 1))) { + bh_assert(reftype_map_num > 0); + frame_reftype = (frame_reftype_map - 1)->ref_type; + } + if (!wasm_reftype_is_subtype_of(*(frame_ref - 1), frame_reftype, type, + ref_type, ctx->module->types, + ctx->module->type_count)) { + set_error_buf_v(error_buf, error_buf_size, "%s%s%s", + "type mismatch: expect ", type2str(type), + " but got other"); + return false; + } + for (i = 0; i < type_cell_num - 1; i++) { + if (*(frame_ref - 2 - i) != *(frame_ref - 1)) { + set_error_buf_v(error_buf, error_buf_size, "%s%s%s", + "type mismatch: expect ", type2str(type), + " but got other"); + return false; + } + } +#endif + + return true; +} + +static bool +check_stack_pop(WASMLoaderContext *ctx, uint8 type, char *error_buf, + uint32 error_buf_size) +{ + int32 block_stack_cell_num = + (int32)(ctx->stack_cell_num - (ctx->frame_csp - 1)->stack_cell_num); +#if WASM_ENABLE_GC != 0 + int32 reftype_map_num = + (int32)(ctx->reftype_map_num - (ctx->frame_csp - 1)->reftype_map_num); +#endif + + if (block_stack_cell_num > 0) { + if (*(ctx->frame_ref - 1) == VALUE_TYPE_ANY) + /* the stack top is a value of any type, return success */ + return true; + } + +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_reftype(type) && block_stack_cell_num > 0) { + uint8 stack_top_type = *(ctx->frame_ref - 1); + WASMRefType *stack_top_ref_type = NULL; + + if (wasm_is_type_multi_byte_type(stack_top_type)) { + bh_assert(reftype_map_num > 0); + stack_top_ref_type = (*(ctx->frame_reftype_map - 1)).ref_type; + } + + if (wasm_reftype_is_subtype_of(stack_top_type, stack_top_ref_type, type, + ctx->ref_type_tmp, ctx->module->types, + ctx->module->type_count)) { + if (wasm_is_type_multi_byte_type(stack_top_type)) { + uint32 ref_type_struct_size = + wasm_reftype_struct_size(stack_top_ref_type); + bh_memcpy_s(ctx->ref_type_tmp, (uint32)sizeof(WASMRefType), + stack_top_ref_type, ref_type_struct_size); + } + return true; + } + } +#endif + + if (!check_stack_top_values(ctx, ctx->frame_ref, block_stack_cell_num, +#if WASM_ENABLE_GC != 0 + ctx->frame_reftype_map, reftype_map_num, +#endif + type, +#if WASM_ENABLE_GC != 0 + ctx->ref_type_tmp, +#endif + error_buf, error_buf_size)) { + return false; + } + + return true; +} + +static bool +wasm_loader_pop_frame_ref(WASMLoaderContext *ctx, uint8 type, char *error_buf, + uint32 error_buf_size) +{ + BranchBlock *cur_block = ctx->frame_csp - 1; + int32 available_stack_cell = + (int32)(ctx->stack_cell_num - cur_block->stack_cell_num); + uint32 cell_num_to_pop = wasm_value_type_cell_num(type); + + /* Directly return success if current block is in stack + polymorphic state while stack is empty. */ + if (available_stack_cell <= 0 && cur_block->is_stack_polymorphic) + return true; + + if (type == VALUE_TYPE_VOID) + return true; + + if (!check_stack_pop(ctx, type, error_buf, error_buf_size)) + return false; + + bh_assert(available_stack_cell > 0); + if (*(ctx->frame_ref - 1) == VALUE_TYPE_ANY) { + type = VALUE_TYPE_ANY; + cell_num_to_pop = 1; + } + + ctx->frame_ref -= cell_num_to_pop; + ctx->stack_cell_num -= cell_num_to_pop; +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(*ctx->frame_ref)) { + ctx->frame_reftype_map--; + ctx->reftype_map_num--; + } +#endif + + return true; +} + +#if WASM_ENABLE_GC != 0 +/* Get the stack top element of current block */ +static bool +wasm_loader_get_frame_ref_top(WASMLoaderContext *ctx, uint8 *p_type, + WASMRefType **p_ref_type, char *error_buf, + uint32 error_buf_size) +{ + BranchBlock *cur_block = ctx->frame_csp - 1; + int32 available_stack_cell = + (int32)(ctx->stack_cell_num - cur_block->stack_cell_num); + + if (available_stack_cell <= 0) { + /* Directly return success if current block is in stack + polymorphic state while stack is empty. */ + if (cur_block->is_stack_polymorphic) { + *p_type = VALUE_TYPE_ANY; + return true; + } + else { + set_error_buf( + error_buf, error_buf_size, + "type mismatch: expect data but block stack was empty"); + return false; + } + } + + *p_type = *(ctx->frame_ref - 1); + if (wasm_is_type_multi_byte_type(*p_type)) { + int32 available_reftype_map = + (int32)(ctx->reftype_map_num + - (ctx->frame_csp - 1)->reftype_map_num); + bh_assert(available_reftype_map > 0); + (void)available_reftype_map; + *p_ref_type = (ctx->frame_reftype_map - 1)->ref_type; + } + + return true; +} + +#if WASM_ENABLE_FAST_INTERP != 0 +static bool +wasm_loader_pop_frame_ref_offset(WASMLoaderContext *ctx, uint8 type, + char *error_buf, uint32 error_buf_size); +#endif + +/* Check whether the stack top elem is a heap object, and if yes, + pop and return it */ +static bool +wasm_loader_pop_heap_obj(WASMLoaderContext *ctx, uint8 *p_type, + WASMRefType *ref_ht_ret, char *error_buf, + uint32 error_buf_size) +{ + uint8 type = 0; + WASMRefType *ref_type = NULL; + + /* Get stack top element */ + if (!wasm_loader_get_frame_ref_top(ctx, &type, &ref_type, error_buf, + error_buf_size)) { + return false; + } + + if (type != VALUE_TYPE_ANY /* block isn't in stack polymorphic state */ + /* stack top isn't a ref type */ + && !wasm_is_type_reftype(type)) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: expect heap object but got others"); + return false; + } + + /* POP stack top */ + if (wasm_is_type_multi_byte_type(type)) { + bh_assert(ref_type); + bh_memcpy_s(ctx->ref_type_tmp, sizeof(WASMRefType), ref_type, + wasm_reftype_struct_size(ref_type)); + } + +#if WASM_ENABLE_FAST_INTERP != 0 + if (!wasm_loader_pop_frame_ref_offset(ctx, type, error_buf, + error_buf_size)) { + return false; + } +#else + if (!wasm_loader_pop_frame_ref(ctx, type, error_buf, error_buf_size)) { + return false; + } +#endif + + if (p_type) + *p_type = type; + if (wasm_is_type_multi_byte_type(type) && ref_ht_ret) { + bh_memcpy_s(ref_ht_ret, sizeof(WASMRefType), ref_type, + wasm_reftype_struct_size(ref_type)); + } + return true; +} + +/* Check whether the stack top elem is subtype of (ref null ht), + and if yes, pop it and return the converted (ref ht) */ +static bool +wasm_loader_pop_nullable_ht(WASMLoaderContext *ctx, uint8 *p_type, + WASMRefType *ref_ht_ret, char *error_buf, + uint32 error_buf_size) +{ + uint8 type = 0; + WASMRefType ref_type = { 0 }; + + if (!wasm_loader_pop_heap_obj(ctx, &type, &ref_type, error_buf, + error_buf_size)) { + return false; + } + + /* Convert to related (ref ht) and return */ + if (type >= REF_TYPE_ARRAYREF && type <= REF_TYPE_NULLFUNCREF) { + /* Return (ref array/struct/i31/eq/any/extern/func/none/noextern/nofunc) + */ + wasm_set_refheaptype_common(&ref_ht_ret->ref_ht_common, false, + HEAP_TYPE_ARRAY + + (type - REF_TYPE_ARRAYREF)); + type = ref_ht_ret->ref_type; + } + else if (wasm_is_reftype_htref_nullable(type) + || wasm_is_reftype_htref_non_nullable(type)) { + bh_memcpy_s(ref_ht_ret, (uint32)sizeof(WASMRefType), &ref_type, + wasm_reftype_struct_size(&ref_type)); + /* Convert to (ref ht) */ + ref_ht_ret->ref_ht_common.ref_type = REF_TYPE_HT_NON_NULLABLE; + ref_ht_ret->ref_ht_common.nullable = false; + type = ref_ht_ret->ref_type; + } + *p_type = type; + + return true; +} + +/* Check whether the stack top elem is (ref null $t) or (ref $t), + and if yes, pop it and return the type_idx */ +static bool +wasm_loader_pop_nullable_typeidx(WASMLoaderContext *ctx, uint8 *p_type, + uint32 *p_type_idx, char *error_buf, + uint32 error_buf_size) +{ + uint8 type = 0; + int32 type_idx = -1; + WASMRefType *ref_type = NULL; + + /* Get stack top element */ + if (!wasm_loader_get_frame_ref_top(ctx, &type, &ref_type, error_buf, + error_buf_size)) { + return false; + } + + if (type != VALUE_TYPE_ANY) { + /* stack top isn't (ref null $t) */ + if (!((wasm_is_reftype_htref_nullable(type) + || wasm_is_reftype_htref_non_nullable(type)) + && wasm_is_refheaptype_typeidx(&ref_type->ref_ht_common))) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: expect (ref null $t) but got others"); + return false; + } + type_idx = ref_type->ref_ht_typeidx.type_idx; + + bh_memcpy_s(ctx->ref_type_tmp, sizeof(WASMRefType), ref_type, + wasm_reftype_struct_size(ref_type)); + } + + /* POP stack top */ +#if WASM_ENABLE_FAST_INTERP != 0 + if (!wasm_loader_pop_frame_ref_offset(ctx, type, error_buf, + error_buf_size)) { + return false; + } +#else + if (!wasm_loader_pop_frame_ref(ctx, type, error_buf, error_buf_size)) { + return false; + } +#endif + + /* Convert to type_idx and return */ + *p_type = type; + if (type != VALUE_TYPE_ANY) + *p_type_idx = (uint32)type_idx; + return true; +} +#endif /* WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_FAST_INTERP == 0 +static bool +wasm_loader_push_pop_frame_ref(WASMLoaderContext *ctx, uint8 pop_cnt, + uint8 type_push, uint8 type_pop, char *error_buf, + uint32 error_buf_size) +{ + for (int i = 0; i < pop_cnt; i++) { + if (!wasm_loader_pop_frame_ref(ctx, type_pop, error_buf, + error_buf_size)) + return false; + } + if (!wasm_loader_push_frame_ref(ctx, type_push, error_buf, error_buf_size)) + return false; + return true; +} +#endif + +static bool +wasm_loader_push_frame_csp(WASMLoaderContext *ctx, uint8 label_type, + BlockType block_type, uint8 *start_addr, + char *error_buf, uint32 error_buf_size) +{ + CHECK_CSP_PUSH(); + memset(ctx->frame_csp, 0, sizeof(BranchBlock)); + ctx->frame_csp->label_type = label_type; + ctx->frame_csp->block_type = block_type; + ctx->frame_csp->start_addr = start_addr; + ctx->frame_csp->stack_cell_num = ctx->stack_cell_num; +#if WASM_ENABLE_GC != 0 + ctx->frame_csp->reftype_map_num = ctx->reftype_map_num; +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + ctx->frame_csp->dynamic_offset = ctx->dynamic_offset; + ctx->frame_csp->patch_list = NULL; +#endif + ctx->frame_csp++; + ctx->csp_num++; + if (ctx->csp_num > ctx->max_csp_num) { + ctx->max_csp_num = ctx->csp_num; + if (ctx->max_csp_num > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "label stack depth limit exceeded"); + return false; + } + } + return true; +fail: + return false; +} + +static bool +wasm_loader_pop_frame_csp(WASMLoaderContext *ctx, char *error_buf, + uint32 error_buf_size) +{ + CHECK_CSP_POP(); +#if WASM_ENABLE_FAST_INTERP != 0 + if ((ctx->frame_csp - 1)->param_frame_offsets) + wasm_runtime_free((ctx->frame_csp - 1)->param_frame_offsets); +#endif + ctx->frame_csp--; + ctx->csp_num--; + + return true; +fail: + return false; +} + +#if WASM_ENABLE_FAST_INTERP != 0 + +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 +#define emit_label(opcode) \ + do { \ + wasm_loader_emit_ptr(loader_ctx, handle_table[opcode]); \ + LOG_OP("\nemit_op [%02x]\t", opcode); \ + } while (0) +#define skip_label() \ + do { \ + wasm_loader_emit_backspace(loader_ctx, sizeof(void *)); \ + LOG_OP("\ndelete last op\n"); \ + } while (0) +#else /* else of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#if UINTPTR_MAX == UINT64_MAX +#define emit_label(opcode) \ + do { \ + int32 offset = \ + (int32)((uint8 *)handle_table[opcode] - (uint8 *)handle_table[0]); \ + /* emit int32 relative offset in 64-bit target */ \ + wasm_loader_emit_uint32(loader_ctx, offset); \ + LOG_OP("\nemit_op [%02x]\t", opcode); \ + } while (0) +#else +#define emit_label(opcode) \ + do { \ + uint32 label_addr = (uint32)(uintptr_t)handle_table[opcode]; \ + /* emit uint32 label address in 32-bit target */ \ + wasm_loader_emit_uint32(loader_ctx, label_addr); \ + LOG_OP("\nemit_op [%02x]\t", opcode); \ + } while (0) +#endif +#define skip_label() \ + do { \ + wasm_loader_emit_backspace(loader_ctx, sizeof(int32)); \ + LOG_OP("\ndelete last op\n"); \ + } while (0) +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#else /* else of WASM_ENABLE_LABELS_AS_VALUES */ +#define emit_label(opcode) \ + do { \ + wasm_loader_emit_uint8(loader_ctx, opcode); \ + LOG_OP("\nemit_op [%02x]\t", opcode); \ + } while (0) +#define skip_label() \ + do { \ + wasm_loader_emit_backspace(loader_ctx, sizeof(uint8)); \ + LOG_OP("\ndelete last op\n"); \ + } while (0) +#endif /* end of WASM_ENABLE_LABELS_AS_VALUES */ + +#define emit_empty_label_addr_and_frame_ip(type) \ + do { \ + if (!add_label_patch_to_list(loader_ctx->frame_csp - 1, type, \ + loader_ctx->p_code_compiled, error_buf, \ + error_buf_size)) \ + goto fail; \ + /* label address, to be patched */ \ + wasm_loader_emit_ptr(loader_ctx, NULL); \ + } while (0) + +#define emit_br_info(frame_csp, is_br) \ + do { \ + if (!wasm_loader_emit_br_info(loader_ctx, frame_csp, is_br, error_buf, \ + error_buf_size)) \ + goto fail; \ + } while (0) + +#define LAST_OP_OUTPUT_I32() \ + (last_op >= WASM_OP_I32_EQZ && last_op <= WASM_OP_I32_ROTR) \ + || (last_op == WASM_OP_I32_LOAD || last_op == WASM_OP_F32_LOAD) \ + || (last_op >= WASM_OP_I32_LOAD8_S && last_op <= WASM_OP_I32_LOAD16_U) \ + || (last_op >= WASM_OP_F32_ABS && last_op <= WASM_OP_F32_COPYSIGN) \ + || (last_op >= WASM_OP_I32_WRAP_I64 \ + && last_op <= WASM_OP_I32_TRUNC_U_F64) \ + || (last_op >= WASM_OP_F32_CONVERT_S_I32 \ + && last_op <= WASM_OP_F32_DEMOTE_F64) \ + || (last_op == WASM_OP_I32_REINTERPRET_F32) \ + || (last_op == WASM_OP_F32_REINTERPRET_I32) \ + || (last_op == EXT_OP_COPY_STACK_TOP) + +#define LAST_OP_OUTPUT_I64() \ + (last_op >= WASM_OP_I64_CLZ && last_op <= WASM_OP_I64_ROTR) \ + || (last_op >= WASM_OP_F64_ABS && last_op <= WASM_OP_F64_COPYSIGN) \ + || (last_op == WASM_OP_I64_LOAD || last_op == WASM_OP_F64_LOAD) \ + || (last_op >= WASM_OP_I64_LOAD8_S && last_op <= WASM_OP_I64_LOAD32_U) \ + || (last_op >= WASM_OP_I64_EXTEND_S_I32 \ + && last_op <= WASM_OP_I64_TRUNC_U_F64) \ + || (last_op >= WASM_OP_F64_CONVERT_S_I32 \ + && last_op <= WASM_OP_F64_PROMOTE_F32) \ + || (last_op == WASM_OP_I64_REINTERPRET_F64) \ + || (last_op == WASM_OP_F64_REINTERPRET_I64) \ + || (last_op == EXT_OP_COPY_STACK_TOP_I64) + +#define GET_CONST_OFFSET(type, val) \ + do { \ + if (!(wasm_loader_get_const_offset(loader_ctx, type, &val, \ + &operand_offset, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define GET_CONST_F32_OFFSET(type, fval) \ + do { \ + if (!(wasm_loader_get_const_offset(loader_ctx, type, &fval, \ + &operand_offset, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define GET_CONST_F64_OFFSET(type, fval) \ + do { \ + if (!(wasm_loader_get_const_offset(loader_ctx, type, &fval, \ + &operand_offset, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define emit_operand(ctx, offset) \ + do { \ + wasm_loader_emit_int16(ctx, offset); \ + LOG_OP("%d\t", offset); \ + } while (0) + +#define emit_byte(ctx, byte) \ + do { \ + wasm_loader_emit_uint8(ctx, byte); \ + LOG_OP("%d\t", byte); \ + } while (0) + +#define emit_uint32(ctx, value) \ + do { \ + wasm_loader_emit_uint32(ctx, value); \ + LOG_OP("%d\t", value); \ + } while (0) + +#define emit_uint64(ctx, value) \ + do { \ + wasm_loader_emit_const(ctx, &value, false); \ + LOG_OP("%lld\t", value); \ + } while (0) + +#define emit_float32(ctx, value) \ + do { \ + wasm_loader_emit_const(ctx, &value, true); \ + LOG_OP("%f\t", value); \ + } while (0) + +#define emit_float64(ctx, value) \ + do { \ + wasm_loader_emit_const(ctx, &value, false); \ + LOG_OP("%f\t", value); \ + } while (0) + +static bool +wasm_loader_ctx_reinit(WASMLoaderContext *ctx) +{ + if (!(ctx->p_code_compiled = + loader_malloc(ctx->code_compiled_peak_size, NULL, 0))) + return false; + ctx->p_code_compiled_end = + ctx->p_code_compiled + ctx->code_compiled_peak_size; + + /* clean up frame ref */ + memset(ctx->frame_ref_bottom, 0, ctx->frame_ref_size); + ctx->frame_ref = ctx->frame_ref_bottom; + ctx->stack_cell_num = 0; + +#if WASM_ENABLE_GC != 0 + /* clean up reftype map */ + memset(ctx->frame_reftype_map_bottom, 0, ctx->frame_reftype_map_size); + ctx->frame_reftype_map = ctx->frame_reftype_map_bottom; + ctx->reftype_map_num = 0; +#endif + + /* clean up frame csp */ + memset(ctx->frame_csp_bottom, 0, ctx->frame_csp_size); + ctx->frame_csp = ctx->frame_csp_bottom; + ctx->csp_num = 0; + ctx->max_csp_num = 0; + + /* clean up frame offset */ + memset(ctx->frame_offset_bottom, 0, ctx->frame_offset_size); + ctx->frame_offset = ctx->frame_offset_bottom; + ctx->dynamic_offset = ctx->start_dynamic_offset; + + /* init preserved local offsets */ + ctx->preserved_local_offset = ctx->max_dynamic_offset; + + /* const buf is reserved */ + return true; +} + +static void +increase_compiled_code_space(WASMLoaderContext *ctx, int32 size) +{ + ctx->code_compiled_size += size; + if (ctx->code_compiled_size >= ctx->code_compiled_peak_size) { + ctx->code_compiled_peak_size = ctx->code_compiled_size; + } +} + +static void +wasm_loader_emit_const(WASMLoaderContext *ctx, void *value, bool is_32_bit) +{ + uint32 size = is_32_bit ? sizeof(uint32) : sizeof(uint64); + + if (ctx->p_code_compiled) { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); +#endif + bh_memcpy_s(ctx->p_code_compiled, + (uint32)(ctx->p_code_compiled_end - ctx->p_code_compiled), + value, size); + ctx->p_code_compiled += size; + } + else { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert((ctx->code_compiled_size & 1) == 0); +#endif + increase_compiled_code_space(ctx, size); + } +} + +static void +wasm_loader_emit_uint32(WASMLoaderContext *ctx, uint32 value) +{ + if (ctx->p_code_compiled) { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); +#endif + STORE_U32(ctx->p_code_compiled, value); + ctx->p_code_compiled += sizeof(uint32); + } + else { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert((ctx->code_compiled_size & 1) == 0); +#endif + increase_compiled_code_space(ctx, sizeof(uint32)); + } +} + +static void +wasm_loader_emit_int16(WASMLoaderContext *ctx, int16 value) +{ + if (ctx->p_code_compiled) { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); +#endif + STORE_U16(ctx->p_code_compiled, (uint16)value); + ctx->p_code_compiled += sizeof(int16); + } + else { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert((ctx->code_compiled_size & 1) == 0); +#endif + increase_compiled_code_space(ctx, sizeof(uint16)); + } +} + +static void +wasm_loader_emit_uint8(WASMLoaderContext *ctx, uint8 value) +{ + if (ctx->p_code_compiled) { + *(ctx->p_code_compiled) = value; + ctx->p_code_compiled += sizeof(uint8); +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + ctx->p_code_compiled++; + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); +#endif + } + else { + increase_compiled_code_space(ctx, sizeof(uint8)); +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + increase_compiled_code_space(ctx, sizeof(uint8)); + bh_assert((ctx->code_compiled_size & 1) == 0); +#endif + } +} + +static void +wasm_loader_emit_ptr(WASMLoaderContext *ctx, void *value) +{ + if (ctx->p_code_compiled) { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); +#endif + STORE_PTR(ctx->p_code_compiled, value); + ctx->p_code_compiled += sizeof(void *); + } + else { +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + bh_assert((ctx->code_compiled_size & 1) == 0); +#endif + increase_compiled_code_space(ctx, sizeof(void *)); + } +} + +static void +wasm_loader_emit_backspace(WASMLoaderContext *ctx, uint32 size) +{ + if (ctx->p_code_compiled) { + ctx->p_code_compiled -= size; +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + if (size == sizeof(uint8)) { + ctx->p_code_compiled--; + bh_assert(((uintptr_t)ctx->p_code_compiled & 1) == 0); + } +#endif + } + else { + ctx->code_compiled_size -= size; +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 + if (size == sizeof(uint8)) { + ctx->code_compiled_size--; + bh_assert((ctx->code_compiled_size & 1) == 0); + } +#endif + } +} + +static bool +preserve_referenced_local(WASMLoaderContext *loader_ctx, uint8 opcode, + uint32 local_index, uint32 local_type, + bool *preserved, char *error_buf, + uint32 error_buf_size) +{ + + uint32 i = 0; + int16 preserved_offset = (int16)local_index; + + *preserved = false; + while (i < loader_ctx->stack_cell_num) { + uint8 cur_type = loader_ctx->frame_ref_bottom[i]; + + /* move previous local into dynamic space before a set/tee_local opcode + */ + if (loader_ctx->frame_offset_bottom[i] == (int16)local_index) { + if (!(*preserved)) { + *preserved = true; + skip_label(); + preserved_offset = loader_ctx->preserved_local_offset; + if (loader_ctx->p_code_compiled) { + bh_assert(preserved_offset != (int16)local_index); + } + if (is_32bit_type(local_type)) { + /* Only increase preserve offset in the second traversal */ + if (loader_ctx->p_code_compiled) + loader_ctx->preserved_local_offset++; + emit_label(EXT_OP_COPY_STACK_TOP); + } +#if WASM_ENABLE_SIMDE != 0 + else if (local_type == VALUE_TYPE_V128) { + if (loader_ctx->p_code_compiled) + loader_ctx->preserved_local_offset += 4; + emit_label(EXT_OP_COPY_STACK_TOP_V128); + } +#endif + else { + if (loader_ctx->p_code_compiled) + loader_ctx->preserved_local_offset += 2; + emit_label(EXT_OP_COPY_STACK_TOP_I64); + } + + /* overflow */ + if (preserved_offset > loader_ctx->preserved_local_offset) { + set_error_buf_v(error_buf, error_buf_size, + "too much local cells 0x%x", + loader_ctx->preserved_local_offset); + return false; + } + + emit_operand(loader_ctx, local_index); + emit_operand(loader_ctx, preserved_offset); + emit_label(opcode); + } + loader_ctx->frame_offset_bottom[i] = preserved_offset; + } + + if (cur_type == VALUE_TYPE_V128) { + i += 4; + } + else if (is_32bit_type(cur_type)) { + i++; + } + else { + i += 2; + } + } + + (void)error_buf; + (void)error_buf_size; + return true; +} + +static bool +preserve_local_for_block(WASMLoaderContext *loader_ctx, uint8 opcode, + char *error_buf, uint32 error_buf_size) +{ + uint32 i = 0; + bool preserve_local; + + /* preserve locals before blocks to ensure that "tee/set_local" inside + blocks will not influence the value of these locals */ + while (i < loader_ctx->stack_cell_num) { + int16 cur_offset = loader_ctx->frame_offset_bottom[i]; + uint8 cur_type = loader_ctx->frame_ref_bottom[i]; + + if ((cur_offset < loader_ctx->start_dynamic_offset) + && (cur_offset >= 0)) { + if (!(preserve_referenced_local(loader_ctx, opcode, cur_offset, + cur_type, &preserve_local, + error_buf, error_buf_size))) + return false; + } + + if (cur_type == VALUE_TYPE_V128) { + i += 4; + } + else if (is_32bit_type(cur_type)) { + i++; + } + else { + i += 2; + } + } + + return true; +} + +static bool +add_label_patch_to_list(BranchBlock *frame_csp, uint8 patch_type, + uint8 *p_code_compiled, char *error_buf, + uint32 error_buf_size) +{ + BranchBlockPatch *patch = + loader_malloc(sizeof(BranchBlockPatch), error_buf, error_buf_size); + if (!patch) { + return false; + } + patch->patch_type = patch_type; + patch->code_compiled = p_code_compiled; + if (!frame_csp->patch_list) { + frame_csp->patch_list = patch; + patch->next = NULL; + } + else { + patch->next = frame_csp->patch_list; + frame_csp->patch_list = patch; + } + return true; +} + +static void +apply_label_patch(WASMLoaderContext *ctx, uint8 depth, uint8 patch_type) +{ + BranchBlock *frame_csp = ctx->frame_csp - depth; + BranchBlockPatch *node = frame_csp->patch_list; + BranchBlockPatch *node_prev = NULL, *node_next; + + if (!ctx->p_code_compiled) + return; + + while (node) { + node_next = node->next; + if (node->patch_type == patch_type) { + STORE_PTR(node->code_compiled, ctx->p_code_compiled); + if (node_prev == NULL) { + frame_csp->patch_list = node_next; + } + else { + node_prev->next = node_next; + } + wasm_runtime_free(node); + } + else { + node_prev = node; + } + node = node_next; + } +} + +static bool +wasm_loader_emit_br_info(WASMLoaderContext *ctx, BranchBlock *frame_csp, + bool is_br, char *error_buf, uint32 error_buf_size) +{ + /* br info layout: + * a) arity of target block + * b) total cell num of arity values + * c) each arity value's cell num + * d) each arity value's src frame offset + * e) each arity values's dst dynamic offset + * f) branch target address + * + * Note: b-e are omitted when arity is 0 so that + * interpreter can recover the br info quickly. + */ + BlockType *block_type = &frame_csp->block_type; + uint8 *types = NULL, cell; +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap *reftype_maps; + uint32 reftype_map_count; +#endif + uint32 arity = 0; + int32 i; + int16 *frame_offset = ctx->frame_offset; + uint16 dynamic_offset; + + /* Note: loop's arity is different from if and block. loop's arity is + * its parameter count while if and block arity is result count. + */ +#if WASM_ENABLE_GC == 0 + if (frame_csp->label_type == LABEL_TYPE_LOOP) + arity = block_type_get_param_types(block_type, &types); + else + arity = block_type_get_result_types(block_type, &types); +#else + if (frame_csp->label_type == LABEL_TYPE_LOOP) + arity = block_type_get_param_types(block_type, &types, &reftype_maps, + &reftype_map_count); + else + arity = block_type_get_result_types(block_type, &types, &reftype_maps, + &reftype_map_count); +#endif + + /* Part a */ + emit_uint32(ctx, arity); + + if (arity) { + /* Part b */ + emit_uint32(ctx, wasm_get_cell_num(types, arity)); + /* Part c */ + for (i = (int32)arity - 1; i >= 0; i--) { + cell = (uint8)wasm_value_type_cell_num(types[i]); + emit_byte(ctx, cell); + } + /* Part d */ + for (i = (int32)arity - 1; i >= 0; i--) { + cell = (uint8)wasm_value_type_cell_num(types[i]); + frame_offset -= cell; + emit_operand(ctx, *(int16 *)(frame_offset)); + } + /* Part e */ + if (frame_csp->label_type == LABEL_TYPE_LOOP) + /* Use start_dynamic_offset which was set in + copy_params_to_dynamic_space */ + dynamic_offset = frame_csp->start_dynamic_offset + + wasm_get_cell_num(types, arity); + else + dynamic_offset = + frame_csp->dynamic_offset + wasm_get_cell_num(types, arity); + if (is_br) + ctx->dynamic_offset = dynamic_offset; + for (i = (int32)arity - 1; i >= 0; i--) { + cell = (uint8)wasm_value_type_cell_num(types[i]); + dynamic_offset -= cell; + emit_operand(ctx, dynamic_offset); + } + } + + /* Part f */ + if (frame_csp->label_type == LABEL_TYPE_LOOP) { + wasm_loader_emit_ptr(ctx, frame_csp->code_compiled); + } + else { + if (!add_label_patch_to_list(frame_csp, PATCH_END, ctx->p_code_compiled, + error_buf, error_buf_size)) + return false; + /* label address, to be patched */ + wasm_loader_emit_ptr(ctx, NULL); + } + + return true; +} + +static bool +wasm_loader_push_frame_offset(WASMLoaderContext *ctx, uint8 type, + bool disable_emit, int16 operand_offset, + char *error_buf, uint32 error_buf_size) +{ + uint32 cell_num_to_push, i; + + if (type == VALUE_TYPE_VOID) + return true; + + /* only check memory overflow in first traverse */ + if (ctx->p_code_compiled == NULL) { + if (!check_offset_push(ctx, error_buf, error_buf_size)) + return false; + } + + if (disable_emit) + *(ctx->frame_offset)++ = operand_offset; + else { + emit_operand(ctx, ctx->dynamic_offset); + *(ctx->frame_offset)++ = ctx->dynamic_offset; + ctx->dynamic_offset++; + if (ctx->dynamic_offset > ctx->max_dynamic_offset) { + ctx->max_dynamic_offset = ctx->dynamic_offset; + if (ctx->max_dynamic_offset >= INT16_MAX) { + goto fail; + } + } + } + + if (is_32bit_type(type)) + return true; + + cell_num_to_push = wasm_value_type_cell_num(type) - 1; + for (i = 0; i < cell_num_to_push; i++) { + if (ctx->p_code_compiled == NULL) { + if (!check_offset_push(ctx, error_buf, error_buf_size)) + return false; + } + + ctx->frame_offset++; + if (!disable_emit) { + ctx->dynamic_offset++; + if (ctx->dynamic_offset > ctx->max_dynamic_offset) { + ctx->max_dynamic_offset = ctx->dynamic_offset; + if (ctx->max_dynamic_offset >= INT16_MAX) + goto fail; + } + } + } + + return true; + +fail: + set_error_buf(error_buf, error_buf_size, + "fast interpreter offset overflow"); + return false; +} + +/* This function should be in front of wasm_loader_pop_frame_ref + as they both use ctx->stack_cell_num, and ctx->stack_cell_num + will be modified by wasm_loader_pop_frame_ref */ +static bool +wasm_loader_pop_frame_offset(WASMLoaderContext *ctx, uint8 type, + char *error_buf, uint32 error_buf_size) +{ + /* if ctx->frame_csp equals ctx->frame_csp_bottom, + then current block is the function block */ + uint32 depth = ctx->frame_csp > ctx->frame_csp_bottom ? 1 : 0; + BranchBlock *cur_block = ctx->frame_csp - depth; + int32 available_stack_cell = + (int32)(ctx->stack_cell_num - cur_block->stack_cell_num); + uint32 cell_num_to_pop; + + /* Directly return success if current block is in stack + polymorphic state while stack is empty. */ + if (available_stack_cell <= 0 && cur_block->is_stack_polymorphic) + return true; + + if (type == VALUE_TYPE_VOID) + return true; + + /* Change type to ANY when the stack top is ANY, so as to avoid + popping unneeded offsets, e.g. if type is I64/F64, we may pop + two offsets */ + if (available_stack_cell > 0 && *(ctx->frame_ref - 1) == VALUE_TYPE_ANY) + type = VALUE_TYPE_ANY; + + cell_num_to_pop = wasm_value_type_cell_num(type); + + /* Check the offset stack bottom to ensure the frame offset + stack will not go underflow. But we don't thrown error + and return true here, because the error msg should be + given in wasm_loader_pop_frame_ref */ + if (!check_offset_pop(ctx, cell_num_to_pop)) + return true; + + ctx->frame_offset -= cell_num_to_pop; + if ((*(ctx->frame_offset) > ctx->start_dynamic_offset) + && (*(ctx->frame_offset) < ctx->max_dynamic_offset)) + ctx->dynamic_offset -= cell_num_to_pop; + + emit_operand(ctx, *(ctx->frame_offset)); + + (void)error_buf; + (void)error_buf_size; + return true; +} + +static bool +wasm_loader_push_frame_ref_offset(WASMLoaderContext *ctx, uint8 type, + bool disable_emit, int16 operand_offset, + char *error_buf, uint32 error_buf_size) +{ + if (!(wasm_loader_push_frame_offset(ctx, type, disable_emit, operand_offset, + error_buf, error_buf_size))) + return false; + if (!(wasm_loader_push_frame_ref(ctx, type, error_buf, error_buf_size))) + return false; + + return true; +} + +static bool +wasm_loader_pop_frame_ref_offset(WASMLoaderContext *ctx, uint8 type, + char *error_buf, uint32 error_buf_size) +{ + /* put wasm_loader_pop_frame_offset in front of wasm_loader_pop_frame_ref */ + if (!wasm_loader_pop_frame_offset(ctx, type, error_buf, error_buf_size)) + return false; + if (!wasm_loader_pop_frame_ref(ctx, type, error_buf, error_buf_size)) + return false; + + return true; +} + +static bool +wasm_loader_push_pop_frame_ref_offset(WASMLoaderContext *ctx, uint8 pop_cnt, + uint8 type_push, uint8 type_pop, + bool disable_emit, int16 operand_offset, + char *error_buf, uint32 error_buf_size) +{ + uint8 i; + + for (i = 0; i < pop_cnt; i++) { + if (!wasm_loader_pop_frame_offset(ctx, type_pop, error_buf, + error_buf_size)) + return false; + + if (!wasm_loader_pop_frame_ref(ctx, type_pop, error_buf, + error_buf_size)) + return false; + } + + if (!wasm_loader_push_frame_offset(ctx, type_push, disable_emit, + operand_offset, error_buf, + error_buf_size)) + return false; + + if (!wasm_loader_push_frame_ref(ctx, type_push, error_buf, error_buf_size)) + return false; + + return true; +} + +static int +cmp_i64_const(const void *p_i64_const1, const void *p_i64_const2) +{ + int64 i64_const1 = *(int64 *)p_i64_const1; + int64 i64_const2 = *(int64 *)p_i64_const2; + + return (i64_const1 < i64_const2) ? -1 : (i64_const1 > i64_const2) ? 1 : 0; +} + +static int +cmp_i32_const(const void *p_i32_const1, const void *p_i32_const2) +{ + int32 i32_const1 = *(int32 *)p_i32_const1; + int32 i32_const2 = *(int32 *)p_i32_const2; + + return (i32_const1 < i32_const2) ? -1 : (i32_const1 > i32_const2) ? 1 : 0; +} + +static int +cmp_v128_const(const void *p_v128_const1, const void *p_v128_const2) +{ + V128 v128_const1 = *(V128 *)p_v128_const1; + V128 v128_const2 = *(V128 *)p_v128_const2; + + return memcmp(&v128_const1, &v128_const2, sizeof(V128)); +} + +static bool +wasm_loader_get_const_offset(WASMLoaderContext *ctx, uint8 type, void *value, + int16 *offset, char *error_buf, + uint32 error_buf_size) +{ + if (!ctx->p_code_compiled) { + /* Treat i64 and f64 as the same by reading i64 value from + the raw bytes */ + if (type == VALUE_TYPE_I64 || type == VALUE_TYPE_F64) { + /* No slot left, emit const instead */ + if (ctx->i64_const_num * 2 + ctx->i32_const_num > INT16_MAX - 2) { + *offset = 0; + return true; + } + + /* Traverse the list if the const num is small */ + if (ctx->i64_const_num < 10) { + for (uint32 i = 0; i < ctx->i64_const_num; i++) { + if (ctx->i64_consts[i] == *(int64 *)value) { + *offset = -1; + return true; + } + } + } + + if (ctx->i64_const_num >= ctx->i64_const_max_num) { + MEM_REALLOC(ctx->i64_consts, + sizeof(int64) * ctx->i64_const_max_num, + sizeof(int64) * (ctx->i64_const_max_num * 2)); + ctx->i64_const_max_num *= 2; + } + ctx->i64_consts[ctx->i64_const_num++] = *(int64 *)value; + } + else if (type == VALUE_TYPE_V128) { + /* No slot left, emit const instead */ + if (ctx->v128_const_num * 4 > INT16_MAX - 2) { + *offset = 0; + return true; + } + + /* Traverse the list if the const num is small */ + if (ctx->v128_const_num < 10) { + for (uint32 i = 0; i < ctx->v128_const_num; i++) { + if (memcmp(&ctx->v128_consts[i], value, sizeof(V128)) + == 0) { + *offset = -1; + return true; + } + } + } + + if (ctx->v128_const_num >= ctx->v128_const_max_num) { + MEM_REALLOC(ctx->v128_consts, + sizeof(V128) * ctx->v128_const_max_num, + sizeof(V128) * (ctx->v128_const_max_num * 2)); + ctx->v128_const_max_num *= 2; + } + ctx->v128_consts[ctx->v128_const_num++] = *(V128 *)value; + } + else { + /* Treat i32 and f32 as the same by reading i32 value from + the raw bytes */ + bh_assert(type == VALUE_TYPE_I32 || type == VALUE_TYPE_F32); + + /* No slot left, emit const instead */ + if (ctx->i64_const_num * 2 + ctx->i32_const_num > INT16_MAX - 1) { + *offset = 0; + return true; + } + + /* Traverse the list if the const num is small */ + if (ctx->i32_const_num < 10) { + for (uint32 i = 0; i < ctx->i32_const_num; i++) { + if (ctx->i32_consts[i] == *(int32 *)value) { + *offset = -1; + return true; + } + } + } + + if (ctx->i32_const_num >= ctx->i32_const_max_num) { + MEM_REALLOC(ctx->i32_consts, + sizeof(int32) * ctx->i32_const_max_num, + sizeof(int32) * (ctx->i32_const_max_num * 2)); + ctx->i32_const_max_num *= 2; + } + ctx->i32_consts[ctx->i32_const_num++] = *(int32 *)value; + } + + *offset = -1; + return true; + } + else { + if (type == VALUE_TYPE_I64 || type == VALUE_TYPE_F64) { + int64 key = *(int64 *)value, *i64_const; + i64_const = bsearch(&key, ctx->i64_consts, ctx->i64_const_num, + sizeof(int64), cmp_i64_const); + if (!i64_const) { /* not found, emit const instead */ + *offset = 0; + return true; + } + + /* constant index is encoded as negative value */ + *offset = -(int32)(ctx->i64_const_num * 2 + ctx->i32_const_num) + + (int32)(i64_const - ctx->i64_consts) * 2; + } + else if (type == VALUE_TYPE_V128) { + V128 key = *(V128 *)value, *v128_const; + v128_const = bsearch(&key, ctx->v128_consts, ctx->v128_const_num, + sizeof(V128), cmp_v128_const); + if (!v128_const) { /* not found, emit const instead */ + *offset = 0; + return true; + } + + /* constant index is encoded as negative value */ + *offset = -(int32)(ctx->v128_const_num) + + (int32)(v128_const - ctx->v128_consts); + } + + else { + int32 key = *(int32 *)value, *i32_const; + i32_const = bsearch(&key, ctx->i32_consts, ctx->i32_const_num, + sizeof(int32), cmp_i32_const); + if (!i32_const) { /* not found, emit const instead */ + *offset = 0; + return true; + } + + /* constant index is encoded as negative value */ + *offset = -(int32)(ctx->i32_const_num) + + (int32)(i32_const - ctx->i32_consts); + } + + return true; + } +fail: + return false; +} + +/* + PUSH(POP)_XXX = push(pop) frame_ref + push(pop) frame_offset + -- Mostly used for the binary / compare operation + PUSH(POP)_OFFSET_TYPE only push(pop) the frame_offset stack + -- Mostly used in block / control instructions + + The POP will always emit the offset on the top of the frame_offset stack + PUSH can be used in two ways: + 1. directly PUSH: + PUSH_XXX(); + will allocate a dynamic space and emit + 2. silent PUSH: + operand_offset = xxx; disable_emit = true; + PUSH_XXX(); + only push the frame_offset stack, no emit +*/ + +#define TEMPLATE_PUSH(Type) \ + do { \ + if (!wasm_loader_push_frame_ref_offset(loader_ctx, VALUE_TYPE_##Type, \ + disable_emit, operand_offset, \ + error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +#define TEMPLATE_PUSH_REF(Type) \ + do { \ + if (!wasm_loader_push_frame_ref_offset(loader_ctx, Type, disable_emit, \ + operand_offset, error_buf, \ + error_buf_size)) \ + goto fail; \ + } while (0) + +#define TEMPLATE_POP(Type) \ + do { \ + if (!wasm_loader_pop_frame_ref_offset(loader_ctx, VALUE_TYPE_##Type, \ + error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +#define TEMPLATE_POP_REF(Type) \ + do { \ + if (!wasm_loader_pop_frame_ref_offset(loader_ctx, Type, error_buf, \ + error_buf_size)) \ + goto fail; \ + } while (0) + +#define PUSH_OFFSET_TYPE(type) \ + do { \ + if (!(wasm_loader_push_frame_offset(loader_ctx, type, disable_emit, \ + operand_offset, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define POP_OFFSET_TYPE(type) \ + do { \ + if (!(wasm_loader_pop_frame_offset(loader_ctx, type, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define POP_AND_PUSH(type_pop, type_push) \ + do { \ + if (!(wasm_loader_push_pop_frame_ref_offset( \ + loader_ctx, 1, type_push, type_pop, disable_emit, \ + operand_offset, error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +/* type of POPs should be the same */ +#define POP2_AND_PUSH(type_pop, type_push) \ + do { \ + if (!(wasm_loader_push_pop_frame_ref_offset( \ + loader_ctx, 2, type_push, type_pop, disable_emit, \ + operand_offset, error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +#else /* WASM_ENABLE_FAST_INTERP */ + +#define TEMPLATE_PUSH(Type) \ + do { \ + if (!(wasm_loader_push_frame_ref(loader_ctx, VALUE_TYPE_##Type, \ + error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +#define TEMPLATE_PUSH_REF(Type) \ + do { \ + if (!(wasm_loader_push_frame_ref(loader_ctx, Type, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define TEMPLATE_POP(Type) \ + do { \ + if (!(wasm_loader_pop_frame_ref(loader_ctx, VALUE_TYPE_##Type, \ + error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +#define TEMPLATE_POP_REF(Type) \ + do { \ + if (!(wasm_loader_pop_frame_ref(loader_ctx, Type, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define POP_AND_PUSH(type_pop, type_push) \ + do { \ + if (!(wasm_loader_push_pop_frame_ref(loader_ctx, 1, type_push, \ + type_pop, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +/* type of POPs should be the same */ +#define POP2_AND_PUSH(type_pop, type_push) \ + do { \ + if (!(wasm_loader_push_pop_frame_ref(loader_ctx, 2, type_push, \ + type_pop, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) +#endif /* WASM_ENABLE_FAST_INTERP */ + +#define PUSH_I32() TEMPLATE_PUSH(I32) +#define PUSH_F32() TEMPLATE_PUSH(F32) +#define PUSH_I64() TEMPLATE_PUSH(I64) +#define PUSH_F64() TEMPLATE_PUSH(F64) +#define PUSH_V128() TEMPLATE_PUSH(V128) +#define PUSH_FUNCREF() TEMPLATE_PUSH(FUNCREF) +#define PUSH_EXTERNREF() TEMPLATE_PUSH(EXTERNREF) +#define PUSH_REF(Type) TEMPLATE_PUSH_REF(Type) +#define POP_REF(Type) TEMPLATE_POP_REF(Type) +#define PUSH_MEM_OFFSET() TEMPLATE_PUSH_REF(mem_offset_type) +#define PUSH_PAGE_COUNT() PUSH_MEM_OFFSET() +#define PUSH_TBL_ELEM_IDX() TEMPLATE_PUSH_REF(table_elem_idx_type) + +#define POP_I32() TEMPLATE_POP(I32) +#define POP_F32() TEMPLATE_POP(F32) +#define POP_I64() TEMPLATE_POP(I64) +#define POP_F64() TEMPLATE_POP(F64) +#define POP_V128() TEMPLATE_POP(V128) +#define POP_FUNCREF() TEMPLATE_POP(FUNCREF) +#define POP_EXTERNREF() TEMPLATE_POP(EXTERNREF) +#define POP_STRINGREF() TEMPLATE_POP(STRINGREF) +#define POP_MEM_OFFSET() TEMPLATE_POP_REF(mem_offset_type) +#define POP_TBL_ELEM_IDX() TEMPLATE_POP_REF(table_elem_idx_type) + +#if WASM_ENABLE_FAST_INTERP != 0 + +static bool +reserve_block_ret(WASMLoaderContext *loader_ctx, uint8 opcode, + bool disable_emit, char *error_buf, uint32 error_buf_size) +{ + int16 operand_offset = 0; + BranchBlock *block = (opcode == WASM_OP_ELSE) ? loader_ctx->frame_csp - 1 + : loader_ctx->frame_csp; + BlockType *block_type = &block->block_type; + uint8 *return_types = NULL; +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap *reftype_maps = NULL; + uint32 reftype_map_count; +#endif + uint32 return_count = 0, value_count = 0, total_cel_num = 0; + int32 i = 0; + int16 dynamic_offset, dynamic_offset_org, *frame_offset = NULL, + *frame_offset_org = NULL; + +#if WASM_ENABLE_GC == 0 + return_count = block_type_get_result_types(block_type, &return_types); +#else + return_count = block_type_get_result_types( + block_type, &return_types, &reftype_maps, &reftype_map_count); +#endif + + /* If there is only one return value, use EXT_OP_COPY_STACK_TOP/_I64/V128 + * instead of EXT_OP_COPY_STACK_VALUES for interpreter performance. */ + if (return_count == 1) { + uint8 cell = (uint8)wasm_value_type_cell_num(return_types[0]); + if (block->dynamic_offset != *(loader_ctx->frame_offset - cell)) { + /* insert op_copy before else opcode */ + if (opcode == WASM_OP_ELSE) + skip_label(); +#if WASM_ENABLE_SIMDE != 0 + if (cell == 4) { + emit_label(EXT_OP_COPY_STACK_TOP_V128); + } +#endif + if (cell <= 2) { + emit_label(cell == 1 ? EXT_OP_COPY_STACK_TOP + : EXT_OP_COPY_STACK_TOP_I64); + } + emit_operand(loader_ctx, *(loader_ctx->frame_offset - cell)); + emit_operand(loader_ctx, block->dynamic_offset); + + if (opcode == WASM_OP_ELSE) { + *(loader_ctx->frame_offset - cell) = block->dynamic_offset; + } + else { + loader_ctx->frame_offset -= cell; + loader_ctx->dynamic_offset = block->dynamic_offset; + PUSH_OFFSET_TYPE(return_types[0]); + wasm_loader_emit_backspace(loader_ctx, sizeof(int16)); + } + if (opcode == WASM_OP_ELSE) + emit_label(opcode); + } + return true; + } + + /* Copy stack top values to block's results which are in dynamic space. + * The instruction format: + * Part a: values count + * Part b: all values total cell num + * Part c: each value's cell_num, src offset and dst offset + * Part d: each value's src offset and dst offset + * Part e: each value's dst offset + */ + frame_offset = frame_offset_org = loader_ctx->frame_offset; + dynamic_offset = dynamic_offset_org = + block->dynamic_offset + wasm_get_cell_num(return_types, return_count); + + /* First traversal to get the count of values needed to be copied. */ + for (i = (int32)return_count - 1; i >= 0; i--) { + uint8 cells = (uint8)wasm_value_type_cell_num(return_types[i]); + + if (frame_offset - cells < loader_ctx->frame_offset_bottom) { + set_error_buf(error_buf, error_buf_size, "frame offset underflow"); + goto fail; + } + + if (cells == 4) { + bool needs_copy = false; + int16 v128_dynamic = dynamic_offset - cells; + + for (int j = 0; j < 4; j++) { + if (*(frame_offset - j - 1) != (v128_dynamic + j)) { + needs_copy = true; + break; + } + } + + if (needs_copy) { + value_count++; + total_cel_num += cells; + } + + frame_offset -= cells; + dynamic_offset = v128_dynamic; + } + else { + frame_offset -= cells; + dynamic_offset -= cells; + if (dynamic_offset != *frame_offset) { + value_count++; + total_cel_num += cells; + } + } + } + + if (value_count) { + uint32 j = 0; + uint8 *emit_data = NULL, *cells = NULL; + int16 *src_offsets = NULL; + uint16 *dst_offsets = NULL; + uint64 size = + (uint64)value_count + * (sizeof(*cells) + sizeof(*src_offsets) + sizeof(*dst_offsets)); + + /* Allocate memory for the emit data */ + if (!(emit_data = loader_malloc(size, error_buf, error_buf_size))) + return false; + + cells = emit_data; + src_offsets = (int16 *)(cells + value_count); + dst_offsets = (uint16 *)(src_offsets + value_count); + + /* insert op_copy before else opcode */ + if (opcode == WASM_OP_ELSE) + skip_label(); + emit_label(EXT_OP_COPY_STACK_VALUES); + /* Part a) */ + emit_uint32(loader_ctx, value_count); + /* Part b) */ + emit_uint32(loader_ctx, total_cel_num); + + /* Second traversal to get each value's cell num, src offset and dst + * offset. */ + frame_offset = frame_offset_org; + dynamic_offset = dynamic_offset_org; + for (i = (int32)return_count - 1, j = 0; i >= 0; i--) { + uint8 cell = (uint8)wasm_value_type_cell_num(return_types[i]); + + if (cell == 4) { + bool needs_copy = false; + int16 v128_dynamic = dynamic_offset - cell; + + for (int k = 0; k < 4; k++) { + if (*(frame_offset - k - 1) != (v128_dynamic + k)) { + needs_copy = true; + break; + } + } + + if (needs_copy) { + cells[j] = cell; + src_offsets[j] = *(frame_offset - cell); + dst_offsets[j] = v128_dynamic; + j++; + } + + frame_offset -= cell; + dynamic_offset = v128_dynamic; + } + else { + frame_offset -= cell; + dynamic_offset -= cell; + if (dynamic_offset != *frame_offset) { + cells[j] = cell; + /* src offset */ + src_offsets[j] = *frame_offset; + /* dst offset */ + dst_offsets[j] = dynamic_offset; + j++; + } + } + + if (opcode == WASM_OP_ELSE) { + if (cell == 4) { + for (int k = 0; k < cell; k++) { + *(frame_offset + k) = dynamic_offset + k; + } + } + else { + *frame_offset = dynamic_offset; + } + } + else { + loader_ctx->frame_offset = frame_offset; + loader_ctx->dynamic_offset = dynamic_offset; + if (!(wasm_loader_push_frame_offset( + loader_ctx, return_types[i], disable_emit, + operand_offset, error_buf, error_buf_size))) { + wasm_runtime_free(emit_data); + goto fail; + } + wasm_loader_emit_backspace(loader_ctx, sizeof(int16)); + loader_ctx->frame_offset = frame_offset_org; + loader_ctx->dynamic_offset = dynamic_offset_org; + } + } + + bh_assert(j == value_count); + + /* Emit the cells, src_offsets and dst_offsets */ + for (j = 0; j < value_count; j++) + emit_byte(loader_ctx, cells[j]); + for (j = 0; j < value_count; j++) + emit_operand(loader_ctx, src_offsets[j]); + for (j = 0; j < value_count; j++) + emit_operand(loader_ctx, dst_offsets[j]); + + if (opcode == WASM_OP_ELSE) + emit_label(opcode); + + wasm_runtime_free(emit_data); + } + + return true; + +fail: + return false; +} +#endif /* WASM_ENABLE_FAST_INTERP */ + +#define PUSH_TYPE(type) \ + do { \ + if (!(wasm_loader_push_frame_ref(loader_ctx, type, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#define POP_TYPE(type) \ + do { \ + if (!(wasm_loader_pop_frame_ref(loader_ctx, type, error_buf, \ + error_buf_size))) \ + goto fail; \ + } while (0) + +#if WASM_ENABLE_GC == 0 +#define PUSH_CSP(label_type, block_type, _start_addr) \ + do { \ + if (!wasm_loader_push_frame_csp(loader_ctx, label_type, block_type, \ + _start_addr, error_buf, \ + error_buf_size)) \ + goto fail; \ + } while (0) + +#define POP_CSP() \ + do { \ + if (!wasm_loader_pop_frame_csp(loader_ctx, error_buf, error_buf_size)) \ + goto fail; \ + } while (0) +#else +#define PUSH_CSP(label_type, block_type, _start_addr) \ + do { \ + if (!wasm_loader_push_frame_csp(loader_ctx, label_type, block_type, \ + _start_addr, error_buf, \ + error_buf_size)) \ + goto fail; \ + if (!wasm_loader_init_local_use_masks(loader_ctx, local_count, \ + error_buf, error_buf_size)) { \ + goto fail; \ + } \ + } while (0) + +#define POP_CSP() \ + do { \ + wasm_loader_destroy_curr_local_use_masks(loader_ctx); \ + if (!wasm_loader_pop_frame_csp(loader_ctx, error_buf, error_buf_size)) \ + goto fail; \ + } while (0) +#endif /* end of WASM_ENABLE_GC == 0 */ + +#if WASM_ENABLE_GC == 0 +#define GET_LOCAL_REFTYPE() (void)0 +#else +#define GET_LOCAL_REFTYPE() \ + do { \ + if (wasm_is_type_multi_byte_type(local_type)) { \ + WASMRefType *_ref_type; \ + if (local_idx < param_count) \ + _ref_type = wasm_reftype_map_find( \ + param_reftype_maps, param_reftype_map_count, local_idx); \ + else \ + _ref_type = wasm_reftype_map_find(local_reftype_maps, \ + local_reftype_map_count, \ + local_idx - param_count); \ + bh_assert(_ref_type); \ + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), _ref_type, \ + wasm_reftype_struct_size(_ref_type)); \ + } \ + } while (0) +#endif /* end of WASM_ENABLE_GC == 0 */ + +#define GET_LOCAL_INDEX_TYPE_AND_OFFSET() \ + do { \ + read_leb_uint32(p, p_end, local_idx); \ + if (local_idx >= param_count + local_count) { \ + set_error_buf(error_buf, error_buf_size, "unknown local"); \ + goto fail; \ + } \ + local_type = local_idx < param_count \ + ? param_types[local_idx] \ + : local_types[local_idx - param_count]; \ + local_offset = local_offsets[local_idx]; \ + GET_LOCAL_REFTYPE(); \ + } while (0) + +static bool +check_memory(WASMModule *module, char *error_buf, uint32 error_buf_size) +{ + if (module->memory_count == 0 && module->import_memory_count == 0) { + set_error_buf(error_buf, error_buf_size, "unknown memory"); + return false; + } + return true; +} + +#define CHECK_MEMORY() \ + do { \ + if (!check_memory(module, error_buf, error_buf_size)) \ + goto fail; \ + } while (0) + +static bool +check_memory_access_align(uint8 opcode, uint32 align, char *error_buf, + uint32 error_buf_size) +{ + uint8 mem_access_aligns[] = { + 2, 3, 2, 3, 0, 0, 1, 1, 0, 0, 1, 1, 2, 2, /* loads */ + 2, 3, 2, 3, 0, 1, 0, 1, 2 /* stores */ + }; + bh_assert(opcode >= WASM_OP_I32_LOAD && opcode <= WASM_OP_I64_STORE32); + if (align > mem_access_aligns[opcode - WASM_OP_I32_LOAD]) { + set_error_buf(error_buf, error_buf_size, + "invalid memop flags: alignment must not be larger " + "than natural"); + return false; + } + return true; +} + +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) \ + || (WASM_ENABLE_FAST_INTERP != 0) +static bool +check_simd_memory_access_align(uint8 opcode, uint32 align, char *error_buf, + uint32 error_buf_size) +{ + uint8 mem_access_aligns[] = { + 4, /* load */ + 3, 3, 3, 3, 3, 3, /* load and extend */ + 0, 1, 2, 3, /* load and splat */ + 4, /* store */ + }; + + uint8 mem_access_aligns_load_lane[] = { + 0, 1, 2, 3, /* load lane */ + 0, 1, 2, 3, /* store lane */ + 2, 3 /* store zero */ + }; + + if (!((opcode <= SIMD_v128_store) + || (SIMD_v128_load8_lane <= opcode + && opcode <= SIMD_v128_load64_zero))) { + set_error_buf(error_buf, error_buf_size, + "the opcode doesn't include memarg"); + return false; + } + + if ((opcode <= SIMD_v128_store + && align > mem_access_aligns[opcode - SIMD_v128_load]) + || (SIMD_v128_load8_lane <= opcode && opcode <= SIMD_v128_load64_zero + && align > mem_access_aligns_load_lane[opcode + - SIMD_v128_load8_lane])) { + set_error_buf(error_buf, error_buf_size, + "invalid memop flags: alignment must not be larger " + "than natural"); + return false; + } + + return true; +} + +static bool +check_simd_access_lane(uint8 opcode, uint8 lane, char *error_buf, + uint32 error_buf_size) +{ + switch (opcode) { + case SIMD_i8x16_extract_lane_s: + case SIMD_i8x16_extract_lane_u: + case SIMD_i8x16_replace_lane: + if (lane >= 16) { + goto fail; + } + break; + case SIMD_i16x8_extract_lane_s: + case SIMD_i16x8_extract_lane_u: + case SIMD_i16x8_replace_lane: + if (lane >= 8) { + goto fail; + } + break; + case SIMD_i32x4_extract_lane: + case SIMD_i32x4_replace_lane: + case SIMD_f32x4_extract_lane: + case SIMD_f32x4_replace_lane: + if (lane >= 4) { + goto fail; + } + break; + case SIMD_i64x2_extract_lane: + case SIMD_i64x2_replace_lane: + case SIMD_f64x2_extract_lane: + case SIMD_f64x2_replace_lane: + if (lane >= 2) { + goto fail; + } + break; + + case SIMD_v128_load8_lane: + case SIMD_v128_load16_lane: + case SIMD_v128_load32_lane: + case SIMD_v128_load64_lane: + case SIMD_v128_store8_lane: + case SIMD_v128_store16_lane: + case SIMD_v128_store32_lane: + case SIMD_v128_store64_lane: + case SIMD_v128_load32_zero: + case SIMD_v128_load64_zero: + { + uint8 max_lanes[] = { 16, 8, 4, 2, 16, 8, 4, 2, 4, 2 }; + if (lane >= max_lanes[opcode - SIMD_v128_load8_lane]) { + goto fail; + } + break; + } + default: + goto fail; + } + + return true; +fail: + set_error_buf(error_buf, error_buf_size, "invalid lane index"); + return false; +} + +static bool +check_simd_shuffle_mask(V128 mask, char *error_buf, uint32 error_buf_size) +{ + uint8 i; + for (i = 0; i != 16; ++i) { + if (mask.i8x16[i] < 0 || mask.i8x16[i] >= 32) { + set_error_buf(error_buf, error_buf_size, "invalid lane index"); + return false; + } + } + return true; +} +#endif /* end of (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) */ +#endif /* end of WASM_ENABLE_SIMD */ + +#if WASM_ENABLE_SHARED_MEMORY != 0 +static bool +check_memory_align_equal(uint8 opcode, uint32 align, char *error_buf, + uint32 error_buf_size) +{ + uint8 wait_notify_aligns[] = { 2, 2, 3 }; + uint8 mem_access_aligns[] = { + 2, 3, 0, 1, 0, 1, 2, + }; + uint8 expect; + + bh_assert((opcode <= WASM_OP_ATOMIC_WAIT64) + || (opcode >= WASM_OP_ATOMIC_I32_LOAD + && opcode <= WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U)); + if (opcode <= WASM_OP_ATOMIC_WAIT64) { + expect = wait_notify_aligns[opcode - WASM_OP_ATOMIC_NOTIFY]; + } + else { + /* 7 opcodes in every group */ + expect = mem_access_aligns[(opcode - WASM_OP_ATOMIC_I32_LOAD) % 7]; + } + if (align != expect) { + set_error_buf(error_buf, error_buf_size, + "alignment isn't equal to natural"); + return false; + } + return true; +} +#endif /* end of WASM_ENABLE_SHARED_MEMORY */ + +static bool +wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, uint8 opcode, + char *error_buf, uint32 error_buf_size) +{ + BranchBlock *target_block, *cur_block; + BlockType *target_block_type; + uint8 type, *types = NULL, *frame_ref; + uint32 arity = 0; + int32 i, available_stack_cell; + uint16 cell_num; +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap *frame_reftype_map; + WASMRefTypeMap *reftype_maps = NULL, *reftype_map = NULL; + WASMRefType *ref_type; + uint32 reftype_map_count = 0; + int32 available_reftype_map; + bool is_type_multi_byte; +#endif + + uint8 *frame_ref_old = loader_ctx->frame_ref; + uint8 *frame_ref_after_popped = NULL; + uint8 frame_ref_tmp[4] = { 0 }; + uint8 *frame_ref_buf = frame_ref_tmp; + uint32 stack_cell_num_old = loader_ctx->stack_cell_num; +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap *frame_reftype_map_old = loader_ctx->frame_reftype_map; + WASMRefTypeMap *frame_reftype_map_after_popped = NULL; + WASMRefTypeMap frame_reftype_map_tmp[4] = { 0 }; + WASMRefTypeMap *frame_reftype_map_buf = frame_reftype_map_tmp; + uint32 reftype_map_num_old = loader_ctx->reftype_map_num; +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + int16 *frame_offset_old = loader_ctx->frame_offset; + int16 *frame_offset_after_popped = NULL; + int16 frame_offset_tmp[4] = { 0 }; + int16 *frame_offset_buf = frame_offset_tmp; + uint16 dynamic_offset_old = (loader_ctx->frame_csp - 1)->dynamic_offset; +#endif + bool ret = false; + + bh_assert(loader_ctx->csp_num > 0); + if (loader_ctx->csp_num - 1 < depth) { + set_error_buf(error_buf, error_buf_size, + "unknown label, " + "unexpected end of section or function"); + return false; + } + + cur_block = loader_ctx->frame_csp - 1; + target_block = loader_ctx->frame_csp - (depth + 1); + target_block_type = &target_block->block_type; + frame_ref = loader_ctx->frame_ref; +#if WASM_ENABLE_GC != 0 + frame_reftype_map = loader_ctx->frame_reftype_map; +#endif + + /* Note: loop's arity is different from if and block. loop's arity is + * its parameter count while if and block arity is result count. + */ +#if WASM_ENABLE_GC == 0 + if (target_block->label_type == LABEL_TYPE_LOOP) + arity = block_type_get_param_types(target_block_type, &types); + else + arity = block_type_get_result_types(target_block_type, &types); +#else + if (target_block->label_type == LABEL_TYPE_LOOP) + arity = block_type_get_param_types(target_block_type, &types, + &reftype_maps, &reftype_map_count); + else + arity = block_type_get_result_types(target_block_type, &types, + &reftype_maps, &reftype_map_count); +#endif + + /* If the stack is in polymorphic state, just clear the stack + * and then re-push the values to make the stack top values + * match block type. */ + if (cur_block->is_stack_polymorphic) { +#if WASM_ENABLE_GC != 0 + int32 j = (int32)reftype_map_count - 1; +#endif + for (i = (int32)arity - 1; i >= 0; i--) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(types[i])) { + bh_assert(reftype_maps[j].index == i); + bh_memcpy_s(loader_ctx->ref_type_tmp, sizeof(WASMRefType), + reftype_maps[j].ref_type, + wasm_reftype_struct_size(reftype_maps[j].ref_type)); + j--; + } +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(types[i]); +#endif + POP_TYPE(types[i]); + } + + /* Backup stack data since it may be changed in the below + push operations, and the stack data may be used when + checking other target blocks of opcode br_table */ + if (opcode == WASM_OP_BR_TABLE) { + uint64 total_size; + + frame_ref_after_popped = loader_ctx->frame_ref; + total_size = (uint64)sizeof(uint8) + * (frame_ref_old - frame_ref_after_popped); + if (total_size > sizeof(frame_ref_tmp) + && !(frame_ref_buf = loader_malloc(total_size, error_buf, + error_buf_size))) { + goto fail; + } + bh_memcpy_s(frame_ref_buf, (uint32)total_size, + frame_ref_after_popped, (uint32)total_size); + +#if WASM_ENABLE_GC != 0 + frame_reftype_map_after_popped = loader_ctx->frame_reftype_map; + total_size = + (uint64)sizeof(WASMRefTypeMap) + * (frame_reftype_map_old - frame_reftype_map_after_popped); + if (total_size > sizeof(frame_reftype_map_tmp) + && !(frame_reftype_map_buf = loader_malloc( + total_size, error_buf, error_buf_size))) { + goto fail; + } + bh_memcpy_s(frame_reftype_map_buf, (uint32)total_size, + frame_reftype_map_after_popped, (uint32)total_size); +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + frame_offset_after_popped = loader_ctx->frame_offset; + total_size = (uint64)sizeof(int16) + * (frame_offset_old - frame_offset_after_popped); + if (total_size > sizeof(frame_offset_tmp) + && !(frame_offset_buf = loader_malloc(total_size, error_buf, + error_buf_size))) { + goto fail; + } + bh_memcpy_s(frame_offset_buf, (uint32)total_size, + frame_offset_after_popped, (uint32)total_size); +#endif + } + +#if WASM_ENABLE_GC != 0 + j = 0; +#endif + for (i = 0; i < (int32)arity; i++) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(types[i])) { + bh_assert(reftype_maps[j].index == i); + bh_memcpy_s(loader_ctx->ref_type_tmp, sizeof(WASMRefType), + reftype_maps[j].ref_type, + wasm_reftype_struct_size(reftype_maps[j].ref_type)); + j++; + } +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + bool disable_emit = true; + int16 operand_offset = 0; + PUSH_OFFSET_TYPE(types[i]); +#endif + PUSH_TYPE(types[i]); + } + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_br_info(target_block, opcode == WASM_OP_BR); +#endif + + /* Restore the stack data, note that frame_ref_bottom, + frame_reftype_map_bottom, frame_offset_bottom may be + re-allocated in the above push operations */ + if (opcode == WASM_OP_BR_TABLE) { + uint32 total_size; + + /* The stack operand num should not be smaller than before + after pop and push operations */ + bh_assert(loader_ctx->stack_cell_num >= stack_cell_num_old); + loader_ctx->stack_cell_num = stack_cell_num_old; + loader_ctx->frame_ref = + loader_ctx->frame_ref_bottom + stack_cell_num_old; + total_size = (uint32)(sizeof(uint8) + * (frame_ref_old - frame_ref_after_popped)); + bh_memcpy_s((uint8 *)loader_ctx->frame_ref - total_size, total_size, + frame_ref_buf, total_size); + +#if WASM_ENABLE_GC != 0 + /* The stack operand num should not be smaller than before + after pop and push operations */ + bh_assert(loader_ctx->reftype_map_num >= reftype_map_num_old); + loader_ctx->reftype_map_num = reftype_map_num_old; + loader_ctx->frame_reftype_map = + loader_ctx->frame_reftype_map_bottom + reftype_map_num_old; + total_size = (uint32)(sizeof(WASMRefTypeMap) + * (frame_reftype_map_old + - frame_reftype_map_after_popped)); + bh_memcpy_s((uint8 *)loader_ctx->frame_reftype_map - total_size, + total_size, frame_reftype_map_buf, total_size); +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + loader_ctx->frame_offset = + loader_ctx->frame_offset_bottom + stack_cell_num_old; + total_size = + (uint32)(sizeof(int16) + * (frame_offset_old - frame_offset_after_popped)); + bh_memcpy_s((uint8 *)loader_ctx->frame_offset - total_size, + total_size, frame_offset_buf, total_size); + (loader_ctx->frame_csp - 1)->dynamic_offset = dynamic_offset_old; +#endif + } + + ret = true; + goto cleanup_and_return; + } + + available_stack_cell = + (int32)(loader_ctx->stack_cell_num - cur_block->stack_cell_num); +#if WASM_ENABLE_GC != 0 + available_reftype_map = + (int32)(loader_ctx->reftype_map_num + - (loader_ctx->frame_csp - 1)->reftype_map_num); + reftype_map = reftype_maps ? reftype_maps + reftype_map_count - 1 : NULL; +#endif + + /* Check stack top values match target block type */ + for (i = (int32)arity - 1; i >= 0; i--) { + type = types[i]; +#if WASM_ENABLE_GC != 0 + ref_type = NULL; + is_type_multi_byte = wasm_is_type_multi_byte_type(type); + if (is_type_multi_byte) { + bh_assert(reftype_map); + ref_type = reftype_map->ref_type; + } +#endif + + if (available_stack_cell <= 0 && cur_block->is_stack_polymorphic) + break; + + if (!check_stack_top_values(loader_ctx, frame_ref, available_stack_cell, +#if WASM_ENABLE_GC != 0 + frame_reftype_map, available_reftype_map, +#endif + type, +#if WASM_ENABLE_GC != 0 + ref_type, +#endif + error_buf, error_buf_size)) { + goto fail; + } + cell_num = wasm_value_type_cell_num(types[i]); + frame_ref -= cell_num; + available_stack_cell -= cell_num; +#if WASM_ENABLE_GC != 0 + if (is_type_multi_byte) { + frame_reftype_map--; + available_reftype_map--; + reftype_map--; + } +#endif + } + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_br_info(target_block, opcode == WASM_OP_BR); +#endif + + ret = true; + +cleanup_and_return: +fail: + if (frame_ref_buf && frame_ref_buf != frame_ref_tmp) + wasm_runtime_free(frame_ref_buf); +#if WASM_ENABLE_GC != 0 + if (frame_reftype_map_buf && frame_reftype_map_buf != frame_reftype_map_tmp) + wasm_runtime_free(frame_reftype_map_buf); +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + if (frame_offset_buf && frame_offset_buf != frame_offset_tmp) + wasm_runtime_free(frame_offset_buf); +#endif + + return ret; +} + +static BranchBlock * +check_branch_block(WASMLoaderContext *loader_ctx, uint8 **p_buf, uint8 *buf_end, + uint8 opcode, char *error_buf, uint32 error_buf_size) +{ + uint8 *p = *p_buf, *p_end = buf_end; + BranchBlock *frame_csp_tmp; + uint32 depth; + + read_leb_uint32(p, p_end, depth); + if (!wasm_loader_check_br(loader_ctx, depth, opcode, error_buf, + error_buf_size)) { + goto fail; + } + + frame_csp_tmp = loader_ctx->frame_csp - depth - 1; + + *p_buf = p; + return frame_csp_tmp; +fail: + return NULL; +} + +#if WASM_ENABLE_EXCE_HANDLING != 0 +static BranchBlock * +check_branch_block_for_delegate(WASMLoaderContext *loader_ctx, uint8 **p_buf, + uint8 *buf_end, char *error_buf, + uint32 error_buf_size) +{ + uint8 *p = *p_buf, *p_end = buf_end; + BranchBlock *frame_csp_tmp; + uint32 depth; + + read_leb_uint32(p, p_end, depth); + /* + * Note: "delegate 0" means the surrounding block, not the + * try-delegate block itself. + * + * Note: the caller hasn't popped the try-delegate frame yet. + */ + bh_assert(loader_ctx->csp_num > 0); + if (loader_ctx->csp_num - 1 <= depth) { +#if WASM_ENABLE_SPEC_TEST == 0 + set_error_buf(error_buf, error_buf_size, "unknown delegate label"); +#else + set_error_buf(error_buf, error_buf_size, "unknown label"); +#endif + goto fail; + } + frame_csp_tmp = loader_ctx->frame_csp - depth - 2; +#if WASM_ENABLE_FAST_INTERP != 0 + emit_br_info(frame_csp_tmp, false); +#endif + + *p_buf = p; + return frame_csp_tmp; +fail: + return NULL; +} +#endif /* end of WASM_ENABLE_EXCE_HANDLING != 0 */ + +static bool +check_block_stack(WASMLoaderContext *loader_ctx, BranchBlock *block, + char *error_buf, uint32 error_buf_size) +{ + BlockType *block_type = &block->block_type; + uint8 *return_types = NULL; + uint32 return_count = 0; + int32 available_stack_cell, return_cell_num, i; + uint8 *frame_ref = NULL; +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap *frame_reftype_map; + WASMRefTypeMap *return_reftype_maps = NULL, *return_reftype_map; + WASMRefType *ref_type; + uint32 param_count, return_reftype_map_count = 0; + int32 available_reftype_map = + (int32)(loader_ctx->reftype_map_num - block->reftype_map_num); +#endif + + available_stack_cell = + (int32)(loader_ctx->stack_cell_num - block->stack_cell_num); + +#if WASM_ENABLE_GC == 0 + return_count = block_type_get_result_types(block_type, &return_types); +#else + return_count = block_type_get_result_types(block_type, &return_types, + &return_reftype_maps, + &return_reftype_map_count); + param_count = + block_type->is_value_type ? 0 : block_type->u.type->param_count; + (void)param_count; +#endif + return_cell_num = + return_count > 0 ? wasm_get_cell_num(return_types, return_count) : 0; + + /* If the stack is in polymorphic state, just clear the stack + * and then re-push the values to make the stack top values + * match block type. */ + if (block->is_stack_polymorphic) { +#if WASM_ENABLE_GC != 0 + int32 j = (int32)return_reftype_map_count - 1; +#endif + for (i = (int32)return_count - 1; i >= 0; i--) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(return_types[i])) { + bh_assert(return_reftype_maps[j].index == i + param_count); + bh_memcpy_s( + loader_ctx->ref_type_tmp, sizeof(WASMRefType), + return_reftype_maps[j].ref_type, + wasm_reftype_struct_size(return_reftype_maps[j].ref_type)); + j--; + } +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(return_types[i]); +#endif + POP_TYPE(return_types[i]); + } + + /* Check stack is empty */ + if (loader_ctx->stack_cell_num != block->stack_cell_num) { + set_error_buf( + error_buf, error_buf_size, + "type mismatch: stack size does not match block type"); + goto fail; + } + +#if WASM_ENABLE_GC != 0 + j = 0; +#endif + for (i = 0; i < (int32)return_count; i++) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(return_types[i])) { + bh_assert(return_reftype_maps[j].index == i + param_count); + bh_memcpy_s( + loader_ctx->ref_type_tmp, sizeof(WASMRefType), + return_reftype_maps[j].ref_type, + wasm_reftype_struct_size(return_reftype_maps[j].ref_type)); + j++; + } +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + bool disable_emit = true; + int16 operand_offset = 0; + PUSH_OFFSET_TYPE(return_types[i]); +#endif + PUSH_TYPE(return_types[i]); + } + return true; + } + + if (available_stack_cell != return_cell_num) { +#if WASM_ENABLE_EXCE_HANDLING != 0 + /* testspec: this error message format is expected by try_catch.wast */ + snprintf( + error_buf, error_buf_size, "type mismatch: %s requires [%s]%s[%s]", + block->label_type == LABEL_TYPE_TRY + || (block->label_type == LABEL_TYPE_CATCH + && return_cell_num > 0) + ? "instruction" + : "block", + return_cell_num > 0 ? type2str(return_types[0]) : "", + " but stack has ", + available_stack_cell > 0 ? type2str(*(loader_ctx->frame_ref - 1)) + : ""); + goto fail; +#else + set_error_buf(error_buf, error_buf_size, + "type mismatch: stack size does not match block type"); + goto fail; +#endif + } + + /* Check stack values match return types */ + frame_ref = loader_ctx->frame_ref; +#if WASM_ENABLE_GC != 0 + frame_reftype_map = loader_ctx->frame_reftype_map; + return_reftype_map = + return_reftype_map_count + ? return_reftype_maps + return_reftype_map_count - 1 + : NULL; +#endif + for (i = (int32)return_count - 1; i >= 0; i--) { + uint8 type = return_types[i]; +#if WASM_ENABLE_GC != 0 + bool is_type_multi_byte = wasm_is_type_multi_byte_type(type); + ref_type = NULL; + if (is_type_multi_byte) { + bh_assert(return_reftype_map); + ref_type = return_reftype_map->ref_type; + } +#endif + if (!check_stack_top_values(loader_ctx, frame_ref, available_stack_cell, +#if WASM_ENABLE_GC != 0 + frame_reftype_map, available_reftype_map, +#endif + type, +#if WASM_ENABLE_GC != 0 + ref_type, +#endif + error_buf, error_buf_size)) + return false; + frame_ref -= wasm_value_type_cell_num(return_types[i]); + available_stack_cell -= wasm_value_type_cell_num(return_types[i]); +#if WASM_ENABLE_GC != 0 + if (is_type_multi_byte) { + frame_reftype_map--; + available_reftype_map--; + return_reftype_map--; + } +#endif + } + + return true; + +fail: + return false; +} + +#if WASM_ENABLE_FAST_INTERP != 0 +/* Copy parameters to dynamic space. + * 1) POP original parameter out; + * 2) Push and copy original values to dynamic space. + * The copy instruction format: + * Part a: param count + * Part b: all param total cell num + * Part c: each param's cell_num, src offset and dst offset + * Part d: each param's src offset + * Part e: each param's dst offset + */ +static bool +copy_params_to_dynamic_space(WASMLoaderContext *loader_ctx, char *error_buf, + uint32 error_buf_size) +{ + bool ret = false; + int16 *frame_offset = NULL; + uint8 *cells = NULL, cell; + int16 *src_offsets = NULL; + uint8 *emit_data = NULL; + uint32 i; + BranchBlock *block = loader_ctx->frame_csp - 1; + BlockType *block_type = &block->block_type; + WASMFuncType *wasm_type = block_type->u.type; + uint32 param_count = block_type->u.type->param_count; + int16 condition_offset = 0; + bool disable_emit = false; + bool is_if_block = (block->label_type == LABEL_TYPE_IF ? true : false); + int16 operand_offset = 0; + + uint64 size = (uint64)param_count * (sizeof(*cells) + sizeof(*src_offsets)); + bh_assert(size > 0); + + /* For if block, we also need copy the condition operand offset. */ + if (is_if_block) + size += sizeof(*cells) + sizeof(*src_offsets); + + /* Allocate memory for the emit data */ + if (!(emit_data = loader_malloc(size, error_buf, error_buf_size))) + return false; + + cells = emit_data; + src_offsets = (int16 *)(cells + param_count); + + if (is_if_block) + condition_offset = *loader_ctx->frame_offset; + + /* POP original parameter out */ + for (i = 0; i < param_count; i++) { + POP_OFFSET_TYPE(wasm_type->types[param_count - i - 1]); + wasm_loader_emit_backspace(loader_ctx, sizeof(int16)); + } + frame_offset = loader_ctx->frame_offset; + + /* Get each param's cell num and src offset */ + for (i = 0; i < param_count; i++) { + cell = (uint8)wasm_value_type_cell_num(wasm_type->types[i]); + cells[i] = cell; + src_offsets[i] = *frame_offset; + frame_offset += cell; + } + /* emit copy instruction */ + emit_label(EXT_OP_COPY_STACK_VALUES); + /* Part a) */ + emit_uint32(loader_ctx, is_if_block ? param_count + 1 : param_count); + /* Part b) */ + emit_uint32(loader_ctx, is_if_block ? wasm_type->param_cell_num + 1 + : wasm_type->param_cell_num); + /* Part c) */ + for (i = 0; i < param_count; i++) + emit_byte(loader_ctx, cells[i]); + if (is_if_block) + emit_byte(loader_ctx, 1); + + /* Part d) */ + for (i = 0; i < param_count; i++) + emit_operand(loader_ctx, src_offsets[i]); + if (is_if_block) + emit_operand(loader_ctx, condition_offset); + + /* Since the start offset to save the block's params and + * the start offset to save the block's results may be + * different, we remember the dynamic offset for loop block + * so that we can use it to copy the stack operands to the + * loop block's params in wasm_loader_emit_br_info. */ + if (block->label_type == LABEL_TYPE_LOOP) + block->start_dynamic_offset = loader_ctx->dynamic_offset; + + /* Part e) */ + /* Push to dynamic space. The push will emit the dst offset. */ + for (i = 0; i < param_count; i++) + PUSH_OFFSET_TYPE(wasm_type->types[i]); + if (is_if_block) + PUSH_OFFSET_TYPE(VALUE_TYPE_I32); + + ret = true; + +fail: + /* Free the emit data */ + wasm_runtime_free(emit_data); + + return ret; +} +#endif + +#if WASM_ENABLE_GC == 0 +#define RESET_REFTYPE_MAP_STACK() (void)0 +#else +#define RESET_REFTYPE_MAP_STACK() \ + do { \ + loader_ctx->reftype_map_num = \ + (loader_ctx->frame_csp - 1)->reftype_map_num; \ + loader_ctx->frame_reftype_map = loader_ctx->frame_reftype_map_bottom \ + + loader_ctx->reftype_map_num; \ + } while (0) +#endif + +/* reset the stack to the state of before entering the last block */ +#if WASM_ENABLE_FAST_INTERP != 0 +#define RESET_STACK() \ + do { \ + loader_ctx->stack_cell_num = \ + (loader_ctx->frame_csp - 1)->stack_cell_num; \ + loader_ctx->frame_ref = \ + loader_ctx->frame_ref_bottom + loader_ctx->stack_cell_num; \ + loader_ctx->frame_offset = \ + loader_ctx->frame_offset_bottom + loader_ctx->stack_cell_num; \ + RESET_REFTYPE_MAP_STACK(); \ + } while (0) +#else +#define RESET_STACK() \ + do { \ + loader_ctx->stack_cell_num = \ + (loader_ctx->frame_csp - 1)->stack_cell_num; \ + loader_ctx->frame_ref = \ + loader_ctx->frame_ref_bottom + loader_ctx->stack_cell_num; \ + RESET_REFTYPE_MAP_STACK(); \ + } while (0) +#endif + +/* set current block's stack polymorphic state */ +#define SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(flag) \ + do { \ + BranchBlock *_cur_block = loader_ctx->frame_csp - 1; \ + _cur_block->is_stack_polymorphic = flag; \ + } while (0) + +#define BLOCK_HAS_PARAM(block_type) \ + (!block_type.is_value_type && block_type.u.type->param_count > 0) + +#define PRESERVE_LOCAL_FOR_BLOCK() \ + do { \ + if (!(preserve_local_for_block(loader_ctx, opcode, error_buf, \ + error_buf_size))) { \ + goto fail; \ + } \ + } while (0) + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 +static bool +get_table_elem_type(const WASMModule *module, uint32 table_idx, + uint8 *p_elem_type, void **p_ref_type, char *error_buf, + uint32 error_buf_size) +{ + if (!check_table_index(module, table_idx, error_buf, error_buf_size)) { + return false; + } + + if (table_idx < module->import_table_count) { + if (p_elem_type) + *p_elem_type = + module->import_tables[table_idx].u.table.table_type.elem_type; +#if WASM_ENABLE_GC != 0 + if (p_ref_type) + *((WASMRefType **)p_ref_type) = + module->import_tables[table_idx] + .u.table.table_type.elem_ref_type; +#endif + } + else { + if (p_elem_type) + *p_elem_type = + module->tables[table_idx - module->import_table_count] + .table_type.elem_type; +#if WASM_ENABLE_GC != 0 + if (p_ref_type) + *((WASMRefType **)p_ref_type) = + module->tables[table_idx - module->import_table_count] + .table_type.elem_ref_type; +#endif + } + return true; +} + +static bool +get_table_seg_elem_type(const WASMModule *module, uint32 table_seg_idx, + uint8 *p_elem_type, void **p_elem_ref_type, + char *error_buf, uint32 error_buf_size) +{ + if (table_seg_idx >= module->table_seg_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown elem segment %u", + table_seg_idx); + return false; + } + + if (p_elem_type) { + *p_elem_type = module->table_segments[table_seg_idx].elem_type; + } +#if WASM_ENABLE_GC != 0 + if (p_elem_ref_type) + *((WASMRefType **)p_elem_ref_type) = + module->table_segments[table_seg_idx].elem_ref_type; +#endif + return true; +} +#endif + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 +const uint8 * +wasm_loader_get_custom_section(WASMModule *module, const char *name, + uint32 *len) +{ + WASMCustomSection *section = module->custom_section_list; + + while (section) { + if ((section->name_len == strlen(name)) + && (memcmp(section->name_addr, name, section->name_len) == 0)) { + if (len) { + *len = section->content_len; + } + return section->content_addr; + } + + section = section->next; + } + + return NULL; +} +#endif + +#if 0 +#define HANDLE_OPCODE(opcode) #opcode +DEFINE_GOTO_TABLE(const char *, op_mnemonics); +#undef HANDLE_OPCODE +#endif + +#if WASM_ENABLE_FAST_INTERP == 0 + +#define pb_read_leb_uint32 read_leb_uint32 +#define pb_read_leb_int32 read_leb_int32 +#define pb_read_leb_int64 read_leb_int64 +#define pb_read_leb_memarg read_leb_memarg +#define pb_read_leb_mem_offset read_leb_mem_offset + +#else + +/* Read leb without malformed format check */ +static uint64 +read_leb_quick(uint8 **p_buf, uint32 maxbits, bool sign) +{ + uint8 *buf = *p_buf; + uint64 result = 0, byte = 0; + uint32 shift = 0; + + do { + byte = *buf++; + result |= ((byte & 0x7f) << shift); + shift += 7; + } while (byte & 0x80); + + if (sign && (shift < maxbits) && (byte & 0x40)) { + /* Sign extend */ + result |= (~((uint64)0)) << shift; + } + + *p_buf = buf; + return result; +} + +#define pb_read_leb_uint32(p, p_end, res) \ + do { \ + if (!loader_ctx->p_code_compiled) \ + /* Enable format check in the first scan */ \ + read_leb_uint32(p, p_end, res); \ + else \ + /* Disable format check in the second scan */ \ + res = (uint32)read_leb_quick(&p, 32, false); \ + } while (0) + +#define pb_read_leb_int32(p, p_end, res) \ + do { \ + if (!loader_ctx->p_code_compiled) \ + /* Enable format check in the first scan */ \ + read_leb_int32(p, p_end, res); \ + else \ + /* Disable format check in the second scan */ \ + res = (int32)read_leb_quick(&p, 32, true); \ + } while (0) + +#define pb_read_leb_int64(p, p_end, res) \ + do { \ + if (!loader_ctx->p_code_compiled) \ + /* Enable format check in the first scan */ \ + read_leb_int64(p, p_end, res); \ + else \ + /* Disable format check in the second scan */ \ + res = (int64)read_leb_quick(&p, 64, true); \ + } while (0) + +#if WASM_ENABLE_MULTI_MEMORY != 0 +#define pb_read_leb_memarg read_leb_memarg +#else +#define pb_read_leb_memarg pb_read_leb_uint32 +#endif + +#if WASM_ENABLE_MEMORY64 != 0 +#define pb_read_leb_mem_offset read_leb_mem_offset +#else +#define pb_read_leb_mem_offset pb_read_leb_uint32 +#endif + +#endif /* end of WASM_ENABLE_FAST_INTERP != 0 */ + +static bool +wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, + uint32 cur_func_idx, char *error_buf, + uint32 error_buf_size) +{ + uint8 *p = func->code, *p_end = func->code + func->code_size, *p_org; + uint32 param_count, local_count, global_count; + uint8 *param_types, *local_types, local_type, global_type, mem_offset_type, + table_elem_idx_type; + BlockType func_block_type; + uint16 *local_offsets, local_offset; + uint32 type_idx, func_idx, local_idx, global_idx, table_idx; + uint32 table_seg_idx, data_seg_idx, count, align, i; + mem_offset_t mem_offset; + int32 i32_const = 0; + int64 i64_const; + uint8 opcode; + bool return_value = false; + WASMLoaderContext *loader_ctx; + BranchBlock *frame_csp_tmp; +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap *param_reftype_maps, *local_reftype_maps; + uint32 param_reftype_map_count, local_reftype_map_count; + int32 heap_type; + WASMRefType wasm_ref_type = { 0 }; + bool need_ref_type_map; +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + int16 operand_offset = 0; + uint8 last_op = 0; + bool disable_emit, preserve_local = false, if_condition_available = true; + float32 f32_const; + float64 f64_const; + /* + * It means that the fast interpreter detected an exception while preparing, + * typically near the block opcode, but it did not immediately trigger + * the exception. The loader should be capable of identifying it near + * the end opcode and then raising the exception. + */ + bool pending_exception = false; + + LOG_OP("\nProcessing func | [%d] params | [%d] locals | [%d] return\n", + func->param_cell_num, func->local_cell_num, func->ret_cell_num); +#endif +#if WASM_ENABLE_MEMORY64 != 0 + bool is_memory64 = has_module_memory64(module); + mem_offset_type = is_memory64 ? VALUE_TYPE_I64 : VALUE_TYPE_I32; +#else + mem_offset_type = VALUE_TYPE_I32; + table_elem_idx_type = VALUE_TYPE_I32; +#endif + uint32 memidx; + + global_count = module->import_global_count + module->global_count; + + param_count = func->func_type->param_count; + param_types = func->func_type->types; + + func_block_type.is_value_type = false; + func_block_type.u.type = func->func_type; + + local_count = func->local_count; + local_types = func->local_types; + local_offsets = func->local_offsets; + +#if WASM_ENABLE_GC != 0 + param_reftype_maps = func->func_type->ref_type_maps; + param_reftype_map_count = func->func_type->ref_type_map_count; + local_reftype_maps = func->local_ref_type_maps; + local_reftype_map_count = func->local_ref_type_map_count; +#endif + + if (!(loader_ctx = wasm_loader_ctx_init(func, error_buf, error_buf_size))) { + goto fail; + } +#if WASM_ENABLE_GC != 0 + loader_ctx->module = module; + loader_ctx->ref_type_set = module->ref_type_set; + loader_ctx->ref_type_tmp = &wasm_ref_type; +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + /* For the first traverse, the initial value of preserved_local_offset has + * not been determined, we use the INT16_MAX to represent that a slot has + * been copied to preserve space. For second traverse, this field will be + * set to the appropriate value in wasm_loader_ctx_reinit. + * This is for Issue #1230, + * https://github.com/bytecodealliance/wasm-micro-runtime/issues/1230, the + * drop opcodes need to know which slots are preserved, so those slots will + * not be treated as dynamically allocated slots */ + loader_ctx->preserved_local_offset = INT16_MAX; + +re_scan: + if (loader_ctx->code_compiled_size > 0) { + if (!wasm_loader_ctx_reinit(loader_ctx)) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + goto fail; + } + p = func->code; + func->code_compiled = loader_ctx->p_code_compiled; + func->code_compiled_size = loader_ctx->code_compiled_size; + + if (loader_ctx->i64_const_num > 0) { + int64 *i64_consts_old = loader_ctx->i64_consts; + + /* Sort the i64 consts */ + qsort(i64_consts_old, loader_ctx->i64_const_num, sizeof(int64), + cmp_i64_const); + + /* Remove the duplicated i64 consts */ + uint32 k = 1; + for (i = 1; i < loader_ctx->i64_const_num; i++) { + if (i64_consts_old[i] != i64_consts_old[i - 1]) { + i64_consts_old[k++] = i64_consts_old[i]; + } + } + + if (k < loader_ctx->i64_const_num) { + int64 *i64_consts_new; + /* Try to reallocate memory with a smaller size */ + if ((i64_consts_new = + wasm_runtime_malloc((uint32)sizeof(int64) * k))) { + bh_memcpy_s(i64_consts_new, (uint32)sizeof(int64) * k, + i64_consts_old, (uint32)sizeof(int64) * k); + /* Free the old memory */ + wasm_runtime_free(i64_consts_old); + loader_ctx->i64_consts = i64_consts_new; + loader_ctx->i64_const_max_num = k; + } + loader_ctx->i64_const_num = k; + } + } + + if (loader_ctx->v128_const_num > 0) { + V128 *v128_consts_old = loader_ctx->v128_consts; + + /* Sort the v128 consts */ + qsort(v128_consts_old, loader_ctx->v128_const_num, sizeof(V128), + cmp_v128_const); + + /* Remove the duplicated v128 consts */ + uint32 k = 1; + for (i = 1; i < loader_ctx->v128_const_num; i++) { + if (!(memcmp(&v128_consts_old[i], &v128_consts_old[i - 1], + sizeof(V128)) + == 0)) { + v128_consts_old[k++] = v128_consts_old[i]; + } + } + + if (k < loader_ctx->v128_const_num) { + V128 *v128_consts_new; + /* Try to reallocate memory with a smaller size */ + if ((v128_consts_new = + wasm_runtime_malloc((uint32)sizeof(V128) * k))) { + bh_memcpy_s(v128_consts_new, (uint32)sizeof(V128) * k, + v128_consts_old, (uint32)sizeof(V128) * k); + /* Free the old memory */ + wasm_runtime_free(v128_consts_old); + loader_ctx->v128_consts = v128_consts_new; + loader_ctx->v128_const_max_num = k; + } + loader_ctx->v128_const_num = k; + } + } + + if (loader_ctx->i32_const_num > 0) { + int32 *i32_consts_old = loader_ctx->i32_consts; + + /* Sort the i32 consts */ + qsort(i32_consts_old, loader_ctx->i32_const_num, sizeof(int32), + cmp_i32_const); + + /* Remove the duplicated i32 consts */ + uint32 k = 1; + for (i = 1; i < loader_ctx->i32_const_num; i++) { + if (i32_consts_old[i] != i32_consts_old[i - 1]) { + i32_consts_old[k++] = i32_consts_old[i]; + } + } + + if (k < loader_ctx->i32_const_num) { + int32 *i32_consts_new; + /* Try to reallocate memory with a smaller size */ + if ((i32_consts_new = + wasm_runtime_malloc((uint32)sizeof(int32) * k))) { + bh_memcpy_s(i32_consts_new, (uint32)sizeof(int32) * k, + i32_consts_old, (uint32)sizeof(int32) * k); + /* Free the old memory */ + wasm_runtime_free(i32_consts_old); + loader_ctx->i32_consts = i32_consts_new; + loader_ctx->i32_const_max_num = k; + } + loader_ctx->i32_const_num = k; + } + } + } +#endif + + PUSH_CSP(LABEL_TYPE_FUNCTION, func_block_type, p); + + while (p < p_end) { + opcode = *p++; +#if WASM_ENABLE_FAST_INTERP != 0 + p_org = p; + disable_emit = false; + emit_label(opcode); +#endif + switch (opcode) { + case WASM_OP_UNREACHABLE: + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + break; + + case WASM_OP_NOP: +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); +#endif + break; + + case WASM_OP_IF: + { +#if WASM_ENABLE_FAST_INTERP != 0 + BranchBlock *parent_block = loader_ctx->frame_csp - 1; + int32 available_stack_cell = + (int32)(loader_ctx->stack_cell_num + - parent_block->stack_cell_num); + + if (available_stack_cell <= 0 + && parent_block->is_stack_polymorphic) + if_condition_available = false; + else + if_condition_available = true; + + PRESERVE_LOCAL_FOR_BLOCK(); +#endif +#if WASM_ENABLE_GC == 0 + POP_I32(); +#endif + goto handle_op_block_and_loop; + } + case WASM_OP_BLOCK: + case WASM_OP_LOOP: +#if WASM_ENABLE_EXCE_HANDLING != 0 + case WASM_OP_TRY: + if (opcode == WASM_OP_TRY) { + /* + * keep track of exception handlers to account for + * memory allocation + */ + func->exception_handler_count++; + + /* + * try is a block + * do nothing special, but execution continues to + * to handle_op_block_and_loop, + * and that be pushes the csp + */ + } + +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + PRESERVE_LOCAL_FOR_BLOCK(); +#endif + handle_op_block_and_loop: + { + uint8 value_type; + BlockType block_type; +#if WASM_ENABLE_FAST_INTERP != 0 + uint32 available_params = 0; +#endif + + CHECK_BUF(p, p_end, 1); + value_type = read_uint8(p); + if (is_byte_a_type(value_type)) { + /* If the first byte is one of these special values: + * 0x40/0x7F/0x7E/0x7D/0x7C, take it as the type of + * the single return value. */ + block_type.is_value_type = true; + block_type.u.value_type.type = value_type; +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (value_type == VALUE_TYPE_V128) + module->is_simd_used = true; + else if (value_type == VALUE_TYPE_FUNCREF + || value_type == VALUE_TYPE_EXTERNREF) + module->is_ref_types_used = true; +#endif +#if WASM_ENABLE_GC != 0 + if (value_type != VALUE_TYPE_VOID) { + p_org = p; + p--; + if (!resolve_value_type((const uint8 **)&p, p_end, + module, module->type_count, + &need_ref_type_map, + &wasm_ref_type, false, + error_buf, error_buf_size)) { + goto fail; + } + if (need_ref_type_map) { + block_type.u.value_type.ref_type_map.index = 0; + if (!(block_type.u.value_type.ref_type_map + .ref_type = reftype_set_insert( + module->ref_type_set, &wasm_ref_type, + error_buf, error_buf_size))) { + goto fail; + } + } + /* Set again as the type might be changed, e.g. + (ref null any) to anyref */ + block_type.u.value_type.type = wasm_ref_type.ref_type; +#if WASM_ENABLE_FAST_INTERP == 0 + while (p_org < p) { +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!record_fast_op(module, p_org, *p_org, + error_buf, error_buf_size)) { + goto fail; + } +#endif + /* Ignore extra bytes for interpreter */ + *p_org++ = WASM_OP_NOP; + } +#endif + } +#endif /* end of WASM_ENABLE_GC != 0 */ + } + else { + int32 type_index; + + /* Resolve the leb128 encoded type index as block type */ + p--; + p_org = p - 1; + pb_read_leb_int32(p, p_end, type_index); + + if (!check_function_type(module, type_index, error_buf, + error_buf_size)) { + goto fail; + } + + block_type.is_value_type = false; + block_type.u.type = + (WASMFuncType *)module->types[type_index]; +#if WASM_ENABLE_FAST_INTERP == 0 + /* If block use type index as block type, change the opcode + * to new extended opcode so that interpreter can resolve + * the block quickly. + */ +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!record_fast_op(module, p_org, *p_org, error_buf, + error_buf_size)) { + goto fail; + } +#endif + *p_org = EXT_OP_BLOCK + (opcode - WASM_OP_BLOCK); +#endif + } + +#if WASM_ENABLE_GC != 0 + if (opcode == WASM_OP_IF) { + POP_I32(); + } +#endif + + /* Pop block parameters from stack */ + if (BLOCK_HAS_PARAM(block_type)) { + WASMFuncType *wasm_type = block_type.u.type; + + BranchBlock *cur_block = loader_ctx->frame_csp - 1; +#if WASM_ENABLE_FAST_INTERP != 0 + uint32 cell_num; + available_params = block_type.u.type->param_count; +#endif + for (i = 0; i < block_type.u.type->param_count; i++) { + + int32 available_stack_cell = + (int32)(loader_ctx->stack_cell_num + - cur_block->stack_cell_num); + if (available_stack_cell <= 0 + && cur_block->is_stack_polymorphic) { +#if WASM_ENABLE_FAST_INTERP != 0 + available_params = i; +#endif + break; + } + + POP_TYPE( + wasm_type->types[wasm_type->param_count - i - 1]); +#if WASM_ENABLE_FAST_INTERP != 0 + /* decrease the frame_offset pointer accordingly to keep + * consistent with frame_ref stack */ + cell_num = wasm_value_type_cell_num( + wasm_type->types[wasm_type->param_count - i - 1]); + loader_ctx->frame_offset -= cell_num; + + if (loader_ctx->frame_offset + < loader_ctx->frame_offset_bottom) { + LOG_DEBUG( + "frame_offset underflow, roll back and " + "let following stack checker report it\n"); + loader_ctx->frame_offset += cell_num; + pending_exception = true; + break; + } +#endif + } + } + PUSH_CSP(LABEL_TYPE_BLOCK + (opcode - WASM_OP_BLOCK), + block_type, p); + + /* Pass parameters to block */ + if (BLOCK_HAS_PARAM(block_type)) { + WASMFuncType *func_type = block_type.u.type; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type; + uint32 j = 0; +#endif + for (i = 0; i < func_type->param_count; i++) { +#if WASM_ENABLE_FAST_INTERP != 0 + uint32 cell_num = + wasm_value_type_cell_num(func_type->types[i]); + if (i >= available_params) { + /* make sure enough space */ + if (loader_ctx->p_code_compiled == NULL) { + loader_ctx->frame_offset += cell_num; + if (!check_offset_push(loader_ctx, error_buf, + error_buf_size)) + goto fail; + /* for following dummy value assignment */ + loader_ctx->frame_offset -= cell_num; + } + + /* If there isn't enough data on stack, push a dummy + * offset to keep the stack consistent with + * frame_ref. + * Since the stack is already in polymorphic state, + * the opcode will not be executed, so the dummy + * offset won't cause any error */ + for (uint32 n = 0; n < cell_num; n++) { + *loader_ctx->frame_offset++ = 0; + } + } + else { + loader_ctx->frame_offset += cell_num; + } +#endif +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(func_type->types[i])) { + bh_assert(func_type->ref_type_maps[j].index == i); + ref_type = func_type->ref_type_maps[j].ref_type; + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + j++; + } +#endif + PUSH_TYPE(func_type->types[i]); + } + } + +#if WASM_ENABLE_FAST_INTERP != 0 + if (opcode == WASM_OP_BLOCK || opcode == WASM_OP_LOOP) { + skip_label(); + + if (BLOCK_HAS_PARAM(block_type)) { + /* Make sure params are in dynamic space */ + if (!copy_params_to_dynamic_space(loader_ctx, error_buf, + error_buf_size)) + goto fail; + } + + if (opcode == WASM_OP_LOOP) { + (loader_ctx->frame_csp - 1)->code_compiled = + loader_ctx->p_code_compiled; + } + } +#if WASM_ENABLE_EXCE_HANDLING != 0 + else if (opcode == WASM_OP_TRY) { + skip_label(); + } +#endif + else if (opcode == WASM_OP_IF) { + BranchBlock *block = loader_ctx->frame_csp - 1; + /* If block has parameters, we should make sure they are in + * dynamic space. Otherwise, when else branch is missing, + * the later opcode may consume incorrect operand offset. + * Spec case: + * (func (export "params-id") (param i32) (result i32) + * (i32.const 1) + * (i32.const 2) + * (if (param i32 i32) (result i32 i32) (local.get 0) + * (then)) (i32.add) + * ) + * + * So we should emit a copy instruction before the if. + * + * And we also need to save the parameter offsets and + * recover them before entering else branch. + * + */ + if (BLOCK_HAS_PARAM(block_type)) { + uint64 size; + + /* In polymorphic state, there may be no if condition on + * the stack, so the offset may not emitted */ + if (if_condition_available) { + /* skip the if condition operand offset */ + wasm_loader_emit_backspace(loader_ctx, + sizeof(int16)); + } + /* skip the if label */ + skip_label(); + /* Emit a copy instruction */ + if (!copy_params_to_dynamic_space(loader_ctx, error_buf, + error_buf_size)) + goto fail; + + /* Emit the if instruction */ + emit_label(opcode); + /* Emit the new condition operand offset */ + POP_OFFSET_TYPE(VALUE_TYPE_I32); + + /* Save top param_count values of frame_offset stack, so + * that we can recover it before executing else branch + */ + size = sizeof(int16) + * (uint64)block_type.u.type->param_cell_num; + if (!(block->param_frame_offsets = loader_malloc( + size, error_buf, error_buf_size))) + goto fail; + bh_memcpy_s(block->param_frame_offsets, (uint32)size, + loader_ctx->frame_offset + - size / sizeof(int16), + (uint32)size); + } + + block->start_dynamic_offset = loader_ctx->dynamic_offset; + + emit_empty_label_addr_and_frame_ip(PATCH_ELSE); + emit_empty_label_addr_and_frame_ip(PATCH_END); + } +#endif + break; + } +#if WASM_ENABLE_EXCE_HANDLING != 0 + case WASM_OP_THROW: + { + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + + uint8 label_type = cur_block->label_type; + uint32 tag_index = 0; + pb_read_leb_int32(p, p_end, tag_index); + + /* check validity of tag_index against module->tag_count */ + /* check tag index is within the tag index space */ + if (tag_index >= module->import_tag_count + module->tag_count) { + snprintf(error_buf, error_buf_size, "unknown tag %d", + tag_index); + goto fail; + } + + /* the tag_type is stored in either the WASMTag (section tags) + * or WASMTagImport (import tag) */ + WASMFuncType *tag_type = NULL; + if (tag_index < module->import_tag_count) { + tag_type = module->import_tags[tag_index].u.tag.tag_type; + } + else { + tag_type = + module->tags[tag_index - module->import_tag_count] + ->tag_type; + } + + if (tag_type->result_count != 0) { + set_error_buf(error_buf, error_buf_size, + "tag type signature does not return void"); + goto fail; + } + + int32 available_stack_cell = + (int32)(loader_ctx->stack_cell_num + - cur_block->stack_cell_num); + int32 tti; + + /* Check stack values match return types by comparing tag param + * types with stack cells */ + uint8 *frame_ref = loader_ctx->frame_ref; +#if WASM_ENABLE_GC != 0 + WASMRefTypeMap *frame_reftype_map = + loader_ctx->frame_reftype_map; + uint32 frame_reftype_map_num = loader_ctx->reftype_map_num; + + /* Temporarily set these values since they may be used in + GET_LOCAL_REFTYPE(), remember they must be restored later */ + param_reftype_maps = tag_type->ref_type_maps; + /* For tag_type function, it shouldn't have result_count = 0 */ + param_reftype_map_count = tag_type->ref_type_map_count; + param_count = tag_type->param_count; +#endif + + for (tti = (int32)tag_type->param_count - 1; tti >= 0; tti--) { +#if WASM_ENABLE_GC != 0 + local_type = tag_type->types[tti]; + local_idx = tti; + /* Get the wasm_ref_type if the local_type is multibyte + type */ + GET_LOCAL_REFTYPE(); +#endif + + if (!check_stack_top_values( + loader_ctx, frame_ref, available_stack_cell, +#if WASM_ENABLE_GC != 0 + frame_reftype_map, frame_reftype_map_num, +#endif + tag_type->types[tti], +#if WASM_ENABLE_GC != 0 + &wasm_ref_type, +#endif + error_buf, error_buf_size)) { + snprintf(error_buf, error_buf_size, + "type mismatch: instruction requires [%s] but " + "stack has [%s]", + tag_type->param_count > 0 + ? type2str(tag_type->types[tti]) + : "", + available_stack_cell > 0 + ? type2str(*(loader_ctx->frame_ref - 1)) + : ""); + goto fail; + } + frame_ref -= wasm_value_type_cell_num(tag_type->types[tti]); + available_stack_cell -= + wasm_value_type_cell_num(tag_type->types[tti]); + } + +#if WASM_ENABLE_GC != 0 + /* Restore the values */ + param_reftype_maps = func->func_type->ref_type_maps; + param_reftype_map_count = func->func_type->ref_type_map_count; + param_count = func->func_type->param_count; +#endif + + /* throw is stack polymorphic */ + (void)label_type; + RESET_STACK(); + + break; + } + case WASM_OP_RETHROW: + { + /* must be done before checking branch block */ + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + + /* check the target catching block: LABEL_TYPE_CATCH */ + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, opcode, + error_buf, error_buf_size))) + goto fail; + + if (frame_csp_tmp->label_type != LABEL_TYPE_CATCH + && frame_csp_tmp->label_type != LABEL_TYPE_CATCH_ALL) { + /* trap according to spectest (rethrow.wast) */ + set_error_buf(error_buf, error_buf_size, + "invalid rethrow label"); + goto fail; + } + + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + uint8 label_type = cur_block->label_type; + (void)label_type; + /* rethrow is stack polymorphic */ + RESET_STACK(); + break; + } + case WASM_OP_DELEGATE: + { + /* check target block is valid */ + if (!(frame_csp_tmp = check_branch_block_for_delegate( + loader_ctx, &p, p_end, error_buf, error_buf_size))) + goto fail; + + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + uint8 label_type = cur_block->label_type; + + (void)label_type; + /* DELEGATE ends the block */ + POP_CSP(); + break; + } + case WASM_OP_CATCH: + { + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + + uint8 label_type = cur_block->label_type; + uint32 tag_index = 0; + pb_read_leb_int32(p, p_end, tag_index); + + /* check validity of tag_index against module->tag_count */ + /* check tag index is within the tag index space */ + if (tag_index >= module->import_tag_count + module->tag_count) { + LOG_VERBOSE("In %s, unknown tag at WASM_OP_CATCH\n", + __FUNCTION__); + set_error_buf(error_buf, error_buf_size, "unknown tag"); + goto fail; + } + + /* the tag_type is stored in either the WASMTag (section tags) + * or WASMTagImport (import tag) */ + WASMFuncType *func_type = NULL; + if (tag_index < module->import_tag_count) { + func_type = module->import_tags[tag_index].u.tag.tag_type; + } + else { + func_type = + module->tags[tag_index - module->import_tag_count] + ->tag_type; + } + + if (func_type->result_count != 0) { + set_error_buf(error_buf, error_buf_size, + "tag type signature does not return void"); + goto fail; + } + + /* check validity of current label (expect LABEL_TYPE_TRY or + * LABEL_TYPE_CATCH) */ + if ((LABEL_TYPE_CATCH != label_type) + && (LABEL_TYPE_TRY != label_type)) { + set_error_buf(error_buf, error_buf_size, + "Unexpected block sequence encountered."); + goto fail; + } + + /* + * replace frame_csp by LABEL_TYPE_CATCH + */ + cur_block->label_type = LABEL_TYPE_CATCH; + + /* RESET_STACK removes the values pushed in TRY or previous + * CATCH Blocks */ + RESET_STACK(); + +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type; + uint32 j = 0; +#endif + + /* push types on the stack according to caught type */ + for (i = 0; i < func_type->param_count; i++) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(func_type->types[i])) { + bh_assert(func_type->ref_type_maps[j].index == i); + ref_type = func_type->ref_type_maps[j].ref_type; + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + j++; + } +#endif + PUSH_TYPE(func_type->types[i]); + } + break; + } + case WASM_OP_CATCH_ALL: + { + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + + /* expecting a TRY or CATCH, anything else will be considered an + * error */ + if ((LABEL_TYPE_CATCH != cur_block->label_type) + && (LABEL_TYPE_TRY != cur_block->label_type)) { + set_error_buf(error_buf, error_buf_size, + "Unexpected block sequence encountered."); + goto fail; + } + + /* no immediates */ + /* replace frame_csp by LABEL_TYPE_CATCH_ALL */ + cur_block->label_type = LABEL_TYPE_CATCH_ALL; + + /* RESET_STACK removes the values pushed in TRY or previous + * CATCH Blocks */ + RESET_STACK(); + + /* catch_all has no tagtype and therefore no parameters */ + break; + } +#endif /* end of WASM_ENABLE_EXCE_HANDLING != 0 */ + case WASM_OP_ELSE: + handle_op_else: + { + BranchBlock *block = NULL; + BlockType block_type; + + if (loader_ctx->csp_num < 2 + /* the matched if isn't found */ + || (loader_ctx->frame_csp - 1)->label_type != LABEL_TYPE_IF + /* duplicated else is found */ + || (loader_ctx->frame_csp - 1)->else_addr) { + set_error_buf( + error_buf, error_buf_size, + "opcode else found without matched opcode if"); + goto fail; + } + block = loader_ctx->frame_csp - 1; + + /* check whether if branch's stack matches its result type */ + if (!check_block_stack(loader_ctx, block, error_buf, + error_buf_size)) + goto fail; + + block->else_addr = p - 1; + block_type = block->block_type; + +#if WASM_ENABLE_GC != 0 + if (!wasm_loader_init_local_use_masks( + loader_ctx, local_count, error_buf, error_buf_size)) { + goto fail; + } +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + /* if the result of if branch is in local or const area, add a + * copy op */ + if (!reserve_block_ret(loader_ctx, opcode, disable_emit, + error_buf, error_buf_size)) { + goto fail; + } + + emit_empty_label_addr_and_frame_ip(PATCH_END); + apply_label_patch(loader_ctx, 1, PATCH_ELSE); +#endif + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(false); + + /* Pass parameters to if-false branch */ + if (BLOCK_HAS_PARAM(block_type)) { + for (i = 0; i < block_type.u.type->param_count; i++) + PUSH_TYPE(block_type.u.type->types[i]); + } + +#if WASM_ENABLE_FAST_INTERP != 0 + /* Recover top param_count values of frame_offset stack */ + if (BLOCK_HAS_PARAM((block_type))) { + uint32 size; + size = sizeof(int16) * block_type.u.type->param_cell_num; + bh_memcpy_s(loader_ctx->frame_offset, size, + block->param_frame_offsets, size); + loader_ctx->frame_offset += (size / sizeof(int16)); + } + loader_ctx->dynamic_offset = block->start_dynamic_offset; +#endif + + break; + } + + case WASM_OP_END: + { + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + + /* check whether block stack matches its result type */ + if (!check_block_stack(loader_ctx, cur_block, error_buf, + error_buf_size)) + goto fail; + + /* if there is no else branch, make a virtual else opcode for + easier integrity check and to copy the correct results to + the block return address for fast-interp mode: + change if block from `if ... end` to `if ... else end` */ + if (cur_block->label_type == LABEL_TYPE_IF + && !cur_block->else_addr) { + opcode = WASM_OP_ELSE; + p--; +#if WASM_ENABLE_FAST_INTERP != 0 + p_org = p; + skip_label(); + disable_emit = false; + emit_label(opcode); +#endif + goto handle_op_else; + } + + POP_CSP(); + +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + /* copy the result to the block return address */ + if (!reserve_block_ret(loader_ctx, opcode, disable_emit, + error_buf, error_buf_size)) { + /* it could be tmp frame_csp allocated from opcode like + * OP_BR and not counted in loader_ctx->csp_num, it won't + * be freed in wasm_loader_ctx_destroy(loader_ctx) so need + * to free the loader_ctx->frame_csp if fails */ + free_label_patch_list(loader_ctx->frame_csp); + goto fail; + } + + apply_label_patch(loader_ctx, 0, PATCH_END); + free_label_patch_list(loader_ctx->frame_csp); + if (loader_ctx->frame_csp->label_type == LABEL_TYPE_FUNCTION) { + int32 idx; + uint8 ret_type; + + emit_label(WASM_OP_RETURN); + for (idx = (int32)func->func_type->result_count - 1; + idx >= 0; idx--) { + ret_type = *(func->func_type->types + + func->func_type->param_count + idx); + POP_OFFSET_TYPE(ret_type); + } + } +#endif + if (loader_ctx->csp_num > 0) { + loader_ctx->frame_csp->end_addr = p - 1; + } + else { + /* end of function block, function will return */ + if (p < p_end) { + set_error_buf(error_buf, error_buf_size, + "section size mismatch"); + goto fail; + } + } + +#if WASM_ENABLE_FAST_INTERP != 0 + if (pending_exception) { + set_error_buf( + error_buf, error_buf_size, + "There is a pending exception needs to be handled"); + goto fail; + } +#endif + + break; + } + + case WASM_OP_BR: + { + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, opcode, + error_buf, error_buf_size))) + goto fail; + + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + break; + } + + case WASM_OP_BR_IF: + { + POP_I32(); + + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, opcode, + error_buf, error_buf_size))) + goto fail; + + break; + } + + case WASM_OP_BR_TABLE: + { + uint32 depth = 0, default_arity, arity = 0; + BranchBlock *target_block; + BlockType *target_block_type; +#if WASM_ENABLE_FAST_INTERP == 0 + BrTableCache *br_table_cache = NULL; + uint8 *p_depth_begin, *p_depth, *p_opcode = p - 1; + uint32 j; +#endif + + pb_read_leb_uint32(p, p_end, count); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, count); +#endif + POP_I32(); + + /* Get each depth and check it */ + p_org = p; + for (i = 0; i <= count; i++) { + pb_read_leb_uint32(p, p_end, depth); + bh_assert(loader_ctx->csp_num > 0); + if (loader_ctx->csp_num - 1 < depth) { + set_error_buf(error_buf, error_buf_size, + "unknown label, " + "unexpected end of section or function"); + goto fail; + } + } + p = p_org; + + /* Get the default block's arity */ + target_block = loader_ctx->frame_csp - (depth + 1); + target_block_type = &target_block->block_type; + default_arity = block_type_get_arity(target_block_type, + target_block->label_type); + +#if WASM_ENABLE_FAST_INTERP == 0 + p_depth_begin = p_depth = p; +#endif + for (i = 0; i <= count; i++) { + p_org = p; + pb_read_leb_uint32(p, p_end, depth); + p = p_org; + + /* Get the target block's arity and check it */ + target_block = loader_ctx->frame_csp - (depth + 1); + target_block_type = &target_block->block_type; + arity = block_type_get_arity(target_block_type, + target_block->label_type); + if (arity != default_arity) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: br_table targets must " + "all use same result type"); + goto fail; + } + + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, opcode, + error_buf, error_buf_size))) { + goto fail; + } + +#if WASM_ENABLE_FAST_INTERP == 0 + if (br_table_cache) { + br_table_cache->br_depths[i] = depth; + } + else { + if (depth > 255) { + /* The depth cannot be stored in one byte, + create br_table cache to store each depth */ +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!record_fast_op(module, p_opcode, *p_opcode, + error_buf, error_buf_size)) { + goto fail; + } +#endif + if (!(br_table_cache = loader_malloc( + offsetof(BrTableCache, br_depths) + + sizeof(uint32) + * (uint64)(count + 1), + error_buf, error_buf_size))) { + goto fail; + } + *p_opcode = EXT_OP_BR_TABLE_CACHE; + br_table_cache->br_table_op_addr = p_opcode; + br_table_cache->br_count = count; + /* Copy previous depths which are one byte */ + for (j = 0; j < i; j++) { + br_table_cache->br_depths[j] = p_depth_begin[j]; + } + br_table_cache->br_depths[i] = depth; + bh_list_insert(module->br_table_cache_list, + br_table_cache); + } + else { + /* The depth can be stored in one byte, use the + byte of the leb to store it */ + *p_depth++ = (uint8)depth; + } + } +#endif + } + +#if WASM_ENABLE_FAST_INTERP == 0 + /* Set the tailing bytes to nop */ + if (br_table_cache) + p_depth = p_depth_begin; + while (p_depth < p) + *p_depth++ = WASM_OP_NOP; +#endif + + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + break; + } + + case WASM_OP_RETURN: + { + WASMFuncType *func_type = func->func_type; + int32 idx; + uint8 ret_type; + +#if WASM_ENABLE_GC != 0 + uint32 j = func_type->ref_type_map_count - 1; +#endif + for (idx = (int32)func_type->result_count - 1; idx >= 0; + idx--) { + ret_type = + *(func_type->types + func_type->param_count + idx); +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(ret_type)) { + WASMRefType *ref_type = + func_type->ref_type_maps[j].ref_type; + bh_assert(func_type->ref_type_maps[j].index + == func_type->param_count + idx); + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + j--; + } +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + /* emit the offset after return opcode */ + POP_OFFSET_TYPE(ret_type); +#endif + POP_TYPE(ret_type); + } + + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + + break; + } + + case WASM_OP_CALL: +#if WASM_ENABLE_TAIL_CALL != 0 + case WASM_OP_RETURN_CALL: +#endif +#if WASM_ENABLE_GC != 0 + case WASM_OP_CALL_REF: + case WASM_OP_RETURN_CALL_REF: +#endif + { + WASMFuncType *func_type; + uint8 type; + int32 idx; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type; + uint32 type_idx1; + int32 j; +#endif + +#if WASM_ENABLE_GC != 0 + if (opcode == WASM_OP_CALL_REF + || opcode == WASM_OP_RETURN_CALL_REF) { + pb_read_leb_uint32(p, p_end, type_idx1); + if (!check_type_index(module, module->type_count, type_idx1, + error_buf, error_buf_size)) { + goto fail; + } + if (module->types[type_idx1]->type_flag != WASM_TYPE_FUNC) { + set_error_buf(error_buf, error_buf_size, + "unknown function type"); + goto fail; + } + if (!wasm_loader_pop_nullable_typeidx(loader_ctx, &type, + &type_idx, error_buf, + error_buf_size)) { + goto fail; + } + if (type == VALUE_TYPE_ANY) { + type_idx = type_idx1; + } + if (!check_type_index(module, module->type_count, type_idx, + error_buf, error_buf_size)) { + goto fail; + } + if (module->types[type_idx]->type_flag != WASM_TYPE_FUNC) { + set_error_buf(error_buf, error_buf_size, + "unknown function type"); + goto fail; + } + if (!wasm_func_type_is_super_of( + (WASMFuncType *)module->types[type_idx1], + (WASMFuncType *)module->types[type_idx])) { + set_error_buf(error_buf, error_buf_size, + "function type mismatch"); + goto fail; + } + func_type = (WASMFuncType *)module->types[type_idx]; + } + else +#endif + { + pb_read_leb_uint32(p, p_end, func_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + /* we need to emit func_idx before arguments */ + emit_uint32(loader_ctx, func_idx); +#endif + + if (!check_function_index(module, func_idx, error_buf, + error_buf_size)) { + goto fail; + } + + if (func_idx < module->import_function_count) + func_type = module->import_functions[func_idx] + .u.function.func_type; + else + func_type = + module + ->functions[func_idx + - module->import_function_count] + ->func_type; + } + + if (func_type->param_count > 0) { +#if WASM_ENABLE_GC != 0 + j = (int32)(func_type->result_ref_type_maps + - func_type->ref_type_maps - 1); +#endif + for (idx = (int32)(func_type->param_count - 1); idx >= 0; + idx--) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type( + func_type->types[idx])) { + ref_type = func_type->ref_type_maps[j].ref_type; + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + j--; + } +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(func_type->types[idx]); +#endif + POP_TYPE(func_type->types[idx]); + } + } + +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + if (opcode == WASM_OP_CALL || opcode == WASM_OP_CALL_REF) { +#endif +#if WASM_ENABLE_GC != 0 + j = (int32)(func_type->result_ref_type_maps + - func_type->ref_type_maps); +#endif + for (i = 0; i < func_type->result_count; i++) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type( + func_type->types[func_type->param_count + i])) { + ref_type = func_type->ref_type_maps[j].ref_type; + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + j++; + } +#endif + PUSH_TYPE(func_type->types[func_type->param_count + i]); +#if WASM_ENABLE_FAST_INTERP != 0 + /* Here we emit each return value's dynamic_offset. But + * in fact these offsets are continuous, so interpreter + * only need to get the first return value's offset. + */ + PUSH_OFFSET_TYPE( + func_type->types[func_type->param_count + i]); +#endif + } +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + } + else { +#if WASM_ENABLE_GC == 0 + if (func_type->result_count + != func->func_type->result_count) { + set_error_buf_v(error_buf, error_buf_size, "%s%u%s", + "type mismatch: expect ", + func->func_type->result_count, + " return values but got other"); + goto fail; + } + for (i = 0; i < func_type->result_count; i++) { + type = func->func_type + ->types[func->func_type->param_count + i]; + if (func_type->types[func_type->param_count + i] + != type) { + set_error_buf_v(error_buf, error_buf_size, "%s%s%s", + "type mismatch: expect ", + type2str(type), " but got other"); + goto fail; + } + } +#else + if (!wasm_func_type_result_is_subtype_of( + func_type, func->func_type, module->types, + module->type_count)) { + set_error_buf( + error_buf, error_buf_size, + "type mismatch: invalid func result types"); + goto fail; + } +#endif + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + } +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_op_func_call = true; +#endif + (void)type; + break; + } + + /* + * if disable reference type: call_indirect typeidx, 0x00 + * if enable reference type: call_indirect typeidx, tableidx + */ + case WASM_OP_CALL_INDIRECT: +#if WASM_ENABLE_TAIL_CALL != 0 + case WASM_OP_RETURN_CALL_INDIRECT: +#endif + { + int32 idx; + WASMFuncType *func_type; + uint32 tbl_elem_type; +#if WASM_ENABLE_GC != 0 + WASMRefType *elem_ref_type = NULL; +#endif + + pb_read_leb_uint32(p, p_end, type_idx); +#if WASM_ENABLE_CALL_INDIRECT_OVERLONG != 0 || WASM_ENABLE_GC != 0 +#if WASM_ENABLE_WAMR_COMPILER != 0 + if (p + 1 < p_end && *p != 0x00) { + /* + * Any non-0x00 byte requires the ref types proposal. + * This is different from checking the table_idx value + * since `0x80 0x00` etc. are all valid encodings of zero. + */ + module->is_ref_types_used = true; + } +#endif + pb_read_leb_uint32(p, p_end, table_idx); +#else + CHECK_BUF(p, p_end, 1); + table_idx = read_uint8(p); +#endif + if (!check_table_index(module, table_idx, error_buf, + error_buf_size)) { + goto fail; + } + tbl_elem_type = + table_idx < module->import_table_count + ? module->import_tables[table_idx] + .u.table.table_type.elem_type + : module->tables[table_idx - module->import_table_count] + .table_type.elem_type; + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + if (tbl_elem_type != VALUE_TYPE_FUNCREF) { + set_error_buf_v(error_buf, error_buf_size, + "type mismatch: instruction requires table " + "of functions but table %u has externref", + table_idx); + goto fail; + } +#elif WASM_ENABLE_GC != 0 + /* Table element must match type ref null func */ + elem_ref_type = + table_idx < module->import_table_count + ? module->import_tables[table_idx] + .u.table.table_type.elem_ref_type + : module->tables[table_idx - module->import_table_count] + .table_type.elem_ref_type; + + if (!wasm_reftype_is_subtype_of( + tbl_elem_type, elem_ref_type, REF_TYPE_FUNCREF, NULL, + module->types, module->type_count)) { + set_error_buf_v(error_buf, error_buf_size, + "type mismatch: instruction requires " + "reference type t match type ref null func" + "in table %u", + table_idx); + goto fail; + } +#else + (void)tbl_elem_type; +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + /* we need to emit before arguments */ +#if WASM_ENABLE_TAIL_CALL != 0 + emit_byte(loader_ctx, opcode); +#endif + emit_uint32(loader_ctx, type_idx); + emit_uint32(loader_ctx, table_idx); +#endif + +#if WASM_ENABLE_MEMORY64 != 0 + table_elem_idx_type = is_table_64bit(module, table_idx) + ? VALUE_TYPE_I64 + : VALUE_TYPE_I32; +#endif + /* skip elem idx */ + POP_TBL_ELEM_IDX(); + + if (!check_function_type(module, type_idx, error_buf, + error_buf_size)) { + goto fail; + } + + func_type = (WASMFuncType *)module->types[type_idx]; + + if (func_type->param_count > 0) { + for (idx = (int32)(func_type->param_count - 1); idx >= 0; + idx--) { +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(func_type->types[idx]); +#endif + POP_TYPE(func_type->types[idx]); + } + } + +#if WASM_ENABLE_TAIL_CALL != 0 + if (opcode == WASM_OP_CALL_INDIRECT) { +#endif + for (i = 0; i < func_type->result_count; i++) { + PUSH_TYPE(func_type->types[func_type->param_count + i]); +#if WASM_ENABLE_FAST_INTERP != 0 + PUSH_OFFSET_TYPE( + func_type->types[func_type->param_count + i]); +#endif + } +#if WASM_ENABLE_TAIL_CALL != 0 + } + else { + uint8 type; + if (func_type->result_count + != func->func_type->result_count) { + set_error_buf_v(error_buf, error_buf_size, "%s%u%s", + "type mismatch: expect ", + func->func_type->result_count, + " return values but got other"); + goto fail; + } + for (i = 0; i < func_type->result_count; i++) { + type = func->func_type + ->types[func->func_type->param_count + i]; + if (func_type->types[func_type->param_count + i] + != type) { + set_error_buf_v(error_buf, error_buf_size, "%s%s%s", + "type mismatch: expect ", + type2str(type), " but got other"); + goto fail; + } + } + RESET_STACK(); + SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + } +#endif +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_op_func_call = true; +#endif +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_op_call_indirect = true; +#endif + break; + } + + case WASM_OP_DROP: + { + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + int32 available_stack_cell = + (int32)(loader_ctx->stack_cell_num + - cur_block->stack_cell_num); + + if (available_stack_cell <= 0 + && !cur_block->is_stack_polymorphic) { + set_error_buf(error_buf, error_buf_size, + "type mismatch, opcode drop was found " + "but stack was empty"); + goto fail; + } + + if (available_stack_cell > 0) { +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type( + *(loader_ctx->frame_ref - 1))) { + bh_assert((int32)(loader_ctx->reftype_map_num + - cur_block->reftype_map_num) + > 0); + loader_ctx->frame_reftype_map--; + loader_ctx->reftype_map_num--; + } +#endif + if (is_32bit_type(*(loader_ctx->frame_ref - 1))) { + loader_ctx->frame_ref--; + loader_ctx->stack_cell_num--; +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + loader_ctx->frame_offset--; + if ((*(loader_ctx->frame_offset) + > loader_ctx->start_dynamic_offset) + && (*(loader_ctx->frame_offset) + < loader_ctx->max_dynamic_offset)) + loader_ctx->dynamic_offset--; +#endif + } + else if (is_64bit_type(*(loader_ctx->frame_ref - 1))) { + loader_ctx->frame_ref -= 2; + loader_ctx->stack_cell_num -= 2; +#if WASM_ENABLE_FAST_INTERP == 0 + *(p - 1) = WASM_OP_DROP_64; +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + loader_ctx->frame_offset -= 2; + if ((*(loader_ctx->frame_offset) + > loader_ctx->start_dynamic_offset) + && (*(loader_ctx->frame_offset) + < loader_ctx->max_dynamic_offset)) + loader_ctx->dynamic_offset -= 2; +#endif + } +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) \ + || (WASM_ENABLE_FAST_INTERP != 0) + else if (*(loader_ctx->frame_ref - 1) == VALUE_TYPE_V128) { + loader_ctx->frame_ref -= 4; + loader_ctx->stack_cell_num -= 4; +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + loader_ctx->frame_offset -= 4; + if ((*(loader_ctx->frame_offset) + > loader_ctx->start_dynamic_offset) + && (*(loader_ctx->frame_offset) + < loader_ctx->max_dynamic_offset)) + loader_ctx->dynamic_offset -= 4; +#endif + } +#endif +#endif + else { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + } + else { +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); +#endif + } + break; + } + + case WASM_OP_SELECT: + { + uint8 ref_type; + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + int32 available_stack_cell; +#if WASM_ENABLE_FAST_INTERP != 0 + uint8 *p_code_compiled_tmp = loader_ctx->p_code_compiled; +#endif + + POP_I32(); + + available_stack_cell = (int32)(loader_ctx->stack_cell_num + - cur_block->stack_cell_num); + + if (available_stack_cell <= 0 + && !cur_block->is_stack_polymorphic) { + set_error_buf(error_buf, error_buf_size, + "type mismatch or invalid result arity, " + "opcode select was found " + "but stack was empty"); + goto fail; + } + + if (available_stack_cell > 0) { + switch (*(loader_ctx->frame_ref - 1)) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: + case VALUE_TYPE_ANY: + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: +#if WASM_ENABLE_FAST_INTERP == 0 + *(p - 1) = WASM_OP_SELECT_64; +#else + if (loader_ctx->p_code_compiled) { + uint8 opcode_tmp = WASM_OP_SELECT_64; +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + *(void **)(p_code_compiled_tmp + - sizeof(void *)) = + handle_table[opcode_tmp]; +#elif UINTPTR_MAX == UINT64_MAX + /* emit int32 relative offset in 64-bit target + */ + int32 offset = + (int32)((uint8 *)handle_table[opcode_tmp] + - (uint8 *)handle_table[0]); + *(int32 *)(p_code_compiled_tmp + - sizeof(int32)) = offset; +#else + /* emit uint32 label address in 32-bit target */ + *(uint32 *)(p_code_compiled_tmp + - sizeof(uint32)) = + (uint32)(uintptr_t)handle_table[opcode_tmp]; +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#else /* else of WASM_ENABLE_LABELS_AS_VALUES */ +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + *(p_code_compiled_tmp - 1) = opcode_tmp; +#else + *(p_code_compiled_tmp - 2) = opcode_tmp; +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#endif /* end of WASM_ENABLE_LABELS_AS_VALUES */ + } +#endif /* end of WASM_ENABLE_FAST_INTERP */ + break; +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) \ + || (WASM_ENABLE_FAST_INTERP != 0) + case VALUE_TYPE_V128: +#if WASM_ENABLE_SIMDE != 0 + if (loader_ctx->p_code_compiled) { + uint8 opcode_tmp = WASM_OP_SELECT_128; +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + *(void **)(p_code_compiled_tmp + - sizeof(void *)) = + handle_table[opcode_tmp]; +#elif UINTPTR_MAX == UINT64_MAX + /* emit int32 relative offset in 64-bit target + */ + int32 offset = + (int32)((uint8 *)handle_table[opcode_tmp] + - (uint8 *)handle_table[0]); + *(int32 *)(p_code_compiled_tmp + - sizeof(int32)) = offset; +#else + /* emit uint32 label address in 32-bit target */ + *(uint32 *)(p_code_compiled_tmp + - sizeof(uint32)) = + (uint32)(uintptr_t)handle_table[opcode_tmp]; +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#else /* else of WASM_ENABLE_LABELS_AS_VALUES */ +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + *(p_code_compiled_tmp - 1) = opcode_tmp; +#else + *(p_code_compiled_tmp - 2) = opcode_tmp; +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#endif /* end of WASM_ENABLE_LABELS_AS_VALUES */ + } +#endif /* end of WASM_ENABLE_FAST_INTERP */ + break; +#endif /* (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) || \ + (WASM_ENABLE_FAST_INTERP != 0) */ +#endif /* WASM_ENABLE_SIMD != 0 */ + default: + { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + } + + ref_type = *(loader_ctx->frame_ref - 1); +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(ref_type); + POP_TYPE(ref_type); + POP_OFFSET_TYPE(ref_type); + POP_TYPE(ref_type); + PUSH_OFFSET_TYPE(ref_type); + PUSH_TYPE(ref_type); +#else + POP2_AND_PUSH(ref_type, ref_type); +#endif + } + else { +#if WASM_ENABLE_FAST_INTERP != 0 + PUSH_OFFSET_TYPE(VALUE_TYPE_ANY); +#endif + PUSH_TYPE(VALUE_TYPE_ANY); + } + break; + } + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + case WASM_OP_SELECT_T: + { + uint8 vec_len, type; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type = NULL; +#endif +#if WASM_ENABLE_FAST_INTERP != 0 + uint8 *p_code_compiled_tmp = loader_ctx->p_code_compiled; +#endif + + pb_read_leb_uint32(p, p_end, vec_len); + if (vec_len != 1) { + /* typed select must have exactly one result */ + set_error_buf(error_buf, error_buf_size, + "invalid result arity"); + goto fail; + } + +#if WASM_ENABLE_GC == 0 + CHECK_BUF(p, p_end, 1); + type = read_uint8(p); + if (!is_valid_value_type_for_interpreter(type)) { + set_error_buf(error_buf, error_buf_size, + "unknown value type"); + goto fail; + } +#else + p_org = p + 1; + if (!resolve_value_type((const uint8 **)&p, p_end, module, + module->type_count, &need_ref_type_map, + &wasm_ref_type, false, error_buf, + error_buf_size)) { + goto fail; + } + type = wasm_ref_type.ref_type; + if (need_ref_type_map) { + if (!(ref_type = reftype_set_insert( + module->ref_type_set, &wasm_ref_type, error_buf, + error_buf_size))) { + goto fail; + } + } +#if WASM_ENABLE_FAST_INTERP == 0 + while (p_org < p) { +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!record_fast_op(module, p_org, *p_org, error_buf, + error_buf_size)) { + goto fail; + } +#endif + /* Ignore extra bytes for interpreter */ + *p_org++ = WASM_OP_NOP; + } +#endif +#endif /* end of WASM_ENABLE_GC == 0 */ + + POP_I32(); + +#if WASM_ENABLE_FAST_INTERP != 0 + if (loader_ctx->p_code_compiled) { + uint8 opcode_tmp = WASM_OP_SELECT; + + if (type == VALUE_TYPE_V128) { +#if WASM_ENABLE_SIMDE != 0 + opcode_tmp = WASM_OP_SELECT_128; +#else + set_error_buf(error_buf, error_buf_size, + "v128 value type requires simd feature"); +#endif + } + else { + if (type == VALUE_TYPE_F64 || type == VALUE_TYPE_I64) + opcode_tmp = WASM_OP_SELECT_64; +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_reftype(type)) + opcode_tmp = WASM_OP_SELECT_T; +#endif +#if WASM_ENABLE_LABELS_AS_VALUES != 0 +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + *(void **)(p_code_compiled_tmp - sizeof(void *)) = + handle_table[opcode_tmp]; +#else +#if UINTPTR_MAX == UINT64_MAX + /* emit int32 relative offset in 64-bit target */ + int32 offset = (int32)((uint8 *)handle_table[opcode_tmp] + - (uint8 *)handle_table[0]); + *(int32 *)(p_code_compiled_tmp - sizeof(int32)) = + offset; +#else + /* emit uint32 label address in 32-bit target */ + *(uint32 *)(p_code_compiled_tmp - sizeof(uint32)) = + (uint32)(uintptr_t)handle_table[opcode_tmp]; +#endif +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#else /* else of WASM_ENABLE_LABELS_AS_VALUES */ +#if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 + *(p_code_compiled_tmp - 1) = opcode_tmp; +#else + *(p_code_compiled_tmp - 2) = opcode_tmp; +#endif /* end of WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS */ +#endif /* end of WASM_ENABLE_LABELS_AS_VALUES */ + } + } +#endif /* WASM_ENABLE_FAST_INTERP != 0 */ + + POP_REF(type); + +#if WASM_ENABLE_GC != 0 + if (need_ref_type_map) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), ref_type, + wasm_reftype_struct_size(ref_type)); + } +#endif + POP_REF(type); + +#if WASM_ENABLE_GC != 0 + if (need_ref_type_map) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), ref_type, + wasm_reftype_struct_size(ref_type)); + } +#endif + PUSH_REF(type); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + (void)vec_len; + break; + } + + /* table.get x. tables[x]. [it] -> [t] */ + /* table.set x. tables[x]. [it t] -> [] */ + case WASM_OP_TABLE_GET: + case WASM_OP_TABLE_SET: + { + uint8 decl_ref_type; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type; +#endif + + pb_read_leb_uint32(p, p_end, table_idx); + if (!get_table_elem_type(module, table_idx, &decl_ref_type, +#if WASM_ENABLE_GC != 0 + (void **)&ref_type, +#else + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(decl_ref_type)) { + bh_assert(ref_type); + bh_memcpy_s(&wasm_ref_type, (uint32)sizeof(WASMRefType), + ref_type, wasm_reftype_struct_size(ref_type)); + } +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, table_idx); +#endif + +#if WASM_ENABLE_MEMORY64 != 0 + table_elem_idx_type = is_table_64bit(module, table_idx) + ? VALUE_TYPE_I64 + : VALUE_TYPE_I32; +#endif + if (opcode == WASM_OP_TABLE_GET) { + POP_TBL_ELEM_IDX(); +#if WASM_ENABLE_FAST_INTERP != 0 + PUSH_OFFSET_TYPE(decl_ref_type); +#endif + PUSH_TYPE(decl_ref_type); + } + else { +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(decl_ref_type); +#endif + POP_TYPE(decl_ref_type); + POP_TBL_ELEM_IDX(); + } + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } + case WASM_OP_REF_NULL: + { + uint8 ref_type; + +#if WASM_ENABLE_GC == 0 + CHECK_BUF(p, p_end, 1); + ref_type = read_uint8(p); + + if (ref_type != VALUE_TYPE_FUNCREF + && ref_type != VALUE_TYPE_EXTERNREF) { + set_error_buf(error_buf, error_buf_size, "type mismatch"); + goto fail; + } +#else + pb_read_leb_int32(p, p_end, heap_type); + if (heap_type >= 0) { + if (!check_type_index(module, module->type_count, heap_type, + error_buf, error_buf_size)) { + goto fail; + } + wasm_set_refheaptype_typeidx(&wasm_ref_type.ref_ht_typeidx, + true, heap_type); + ref_type = wasm_ref_type.ref_type; + } + else { + if (!wasm_is_valid_heap_type(heap_type)) { + set_error_buf(error_buf, error_buf_size, + "unknown type"); + goto fail; + } + ref_type = (uint8)((int32)0x80 + heap_type); + } +#endif /* end of WASM_ENABLE_GC == 0 */ + +#if WASM_ENABLE_FAST_INTERP != 0 + PUSH_OFFSET_TYPE(ref_type); +#endif + PUSH_TYPE(ref_type); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } + case WASM_OP_REF_IS_NULL: + { +#if WASM_ENABLE_GC == 0 +#if WASM_ENABLE_FAST_INTERP != 0 + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + int32 block_stack_cell_num = + (int32)(loader_ctx->stack_cell_num + - cur_block->stack_cell_num); + if (block_stack_cell_num <= 0) { + if (!cur_block->is_stack_polymorphic) { + set_error_buf( + error_buf, error_buf_size, + "type mismatch: expect data but stack was empty"); + goto fail; + } + } + else { + if (*(loader_ctx->frame_ref - 1) == VALUE_TYPE_FUNCREF + || *(loader_ctx->frame_ref - 1) == VALUE_TYPE_EXTERNREF + || *(loader_ctx->frame_ref - 1) == VALUE_TYPE_ANY) { + if (!wasm_loader_pop_frame_ref_offset( + loader_ctx, *(loader_ctx->frame_ref - 1), + error_buf, error_buf_size)) { + goto fail; + } + } + else { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + } +#else + if (!wasm_loader_pop_frame_ref(loader_ctx, VALUE_TYPE_FUNCREF, + error_buf, error_buf_size) + && !wasm_loader_pop_frame_ref(loader_ctx, + VALUE_TYPE_EXTERNREF, + error_buf, error_buf_size)) { + goto fail; + } +#endif +#else /* else of WASM_ENABLE_GC == 0 */ + uint8 type; + if (!wasm_loader_pop_heap_obj(loader_ctx, &type, &wasm_ref_type, + error_buf, error_buf_size)) { + goto fail; + } +#endif + PUSH_I32(); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } + case WASM_OP_REF_FUNC: + { + pb_read_leb_uint32(p, p_end, func_idx); + + if (!check_function_index(module, func_idx, error_buf, + error_buf_size)) { + goto fail; + } + + /* Refer to a forward-declared function: + the function must be an import, exported, or present in + a table elem segment or global initializer to be used as + the operand to ref.func */ + if (func_idx >= module->import_function_count) { + WASMTableSeg *table_seg = module->table_segments; + bool func_declared = false; + uint32 j; + + for (i = 0; i < module->global_count; i++) { + if (module->globals[i].type.val_type + == VALUE_TYPE_FUNCREF + && module->globals[i].init_expr.init_expr_type + == INIT_EXPR_TYPE_FUNCREF_CONST + && module->globals[i].init_expr.u.unary.v.u32 + == func_idx) { + func_declared = true; + break; + } + } + + if (!func_declared) { + /* Check whether the function is declared in table segs, + note that it doesn't matter whether the table seg's + mode is passive, active or declarative. */ + for (i = 0; i < module->table_seg_count; + i++, table_seg++) { + if (table_seg->elem_type == VALUE_TYPE_FUNCREF +#if WASM_ENABLE_GC != 0 + /* elem type is (ref null? func) or + (ref null? $t) */ + || ((table_seg->elem_type + == REF_TYPE_HT_NON_NULLABLE + || table_seg->elem_type + == REF_TYPE_HT_NULLABLE) + && (table_seg->elem_ref_type->ref_ht_common + .heap_type + == HEAP_TYPE_FUNC + || table_seg->elem_ref_type + ->ref_ht_common.heap_type + > 0)) +#endif + ) { + for (j = 0; j < table_seg->value_count; j++) { + if (table_seg->init_values[j] + .u.unary.v.ref_index + == func_idx) { + func_declared = true; + break; + } + } + } + } + } + + if (!func_declared) { + /* Check whether the function is exported */ + for (i = 0; i < module->export_count; i++) { + if (module->exports[i].kind == EXPORT_KIND_FUNC + && module->exports[i].index == func_idx) { + func_declared = true; + break; + } + } + } + + if (!func_declared) { + set_error_buf(error_buf, error_buf_size, + "undeclared function reference"); + goto fail; + } + } + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, func_idx); +#endif +#if WASM_ENABLE_GC == 0 + PUSH_FUNCREF(); +#else + if (func_idx < module->import_function_count) + type_idx = + module->import_functions[func_idx].u.function.type_idx; + else + type_idx = module + ->functions[func_idx + - module->import_function_count] + ->type_idx; + wasm_set_refheaptype_typeidx(&wasm_ref_type.ref_ht_typeidx, + false, type_idx); + PUSH_REF(wasm_ref_type.ref_type); +#endif + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_GC != 0 + case WASM_OP_REF_AS_NON_NULL: + case WASM_OP_BR_ON_NULL: + { + uint8 type; + WASMRefType ref_type; + + /* POP (ref null ht) and get the converted (ref ht) */ + if (!wasm_loader_pop_nullable_ht(loader_ctx, &type, &ref_type, + error_buf, error_buf_size)) { + goto fail; + } + + if (opcode == WASM_OP_BR_ON_NULL) { + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, opcode, + error_buf, error_buf_size))) { + goto fail; + } +#if WASM_ENABLE_FAST_INTERP != 0 + disable_emit = true; +#endif + } + + /* PUSH the converted (ref ht) */ + if (type != VALUE_TYPE_ANY) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), &ref_type, + sizeof(WASMRefType)); + } + PUSH_REF(type); + break; + } + + case WASM_OP_BR_ON_NON_NULL: + { + uint8 type; + WASMRefType ref_type; + uint32 available_stack_cell = + loader_ctx->stack_cell_num + - (loader_ctx->frame_csp - 1)->stack_cell_num; + + /* POP (ref null ht) and get the converted (ref ht) */ + if (!wasm_loader_pop_nullable_ht(loader_ctx, &type, &ref_type, + error_buf, error_buf_size)) { + goto fail; + } + +#if WASM_ENABLE_FAST_INTERP != 0 + disable_emit = true; +#endif + + /* Temporarily PUSH back (ref ht), check brach block and + then POP it */ + if (available_stack_cell + > 0) { /* stack isn't in polymorphic state */ + if (type != VALUE_TYPE_ANY) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + &ref_type, sizeof(WASMRefType)); + } + PUSH_REF(type); + } + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, opcode, + error_buf, error_buf_size))) { + goto fail; + } + if (available_stack_cell + > 0) { /* stack isn't in polymorphic state */ + POP_REF(type); +#if WASM_ENABLE_FAST_INTERP != 0 + /* Erase the opnd offset emitted by POP_REF() */ + wasm_loader_emit_backspace(loader_ctx, sizeof(uint16)); +#endif + } + break; + } + + case WASM_OP_REF_EQ: + POP_REF(REF_TYPE_EQREF); + POP_REF(REF_TYPE_EQREF); + PUSH_I32(); + break; +#endif /* end of WASM_ENABLE_GC != 0 */ + + case WASM_OP_GET_LOCAL: + { + p_org = p - 1; + GET_LOCAL_INDEX_TYPE_AND_OFFSET(); + PUSH_TYPE(local_type); + +#if WASM_ENABLE_GC != 0 + /* Cannot get a non-nullable and unset local */ + if (local_idx >= param_count + && wasm_is_reftype_htref_non_nullable(local_type) + && !wasm_loader_get_local_status(loader_ctx, + local_idx - param_count)) { + set_error_buf(error_buf, error_buf_size, + "uninitialized local"); + goto fail; + } +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + /* Get Local is optimized out */ + skip_label(); + disable_emit = true; + operand_offset = local_offset; + PUSH_OFFSET_TYPE(local_type); +#else +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) \ + && (WASM_ENABLE_FAST_JIT == 0) && (WASM_ENABLE_DEBUG_INTERP == 0) + if (local_offset < 0x80 +#if WASM_ENABLE_GC != 0 + && !wasm_is_type_reftype(local_type) +#endif + ) { + *p_org++ = EXT_OP_GET_LOCAL_FAST; + if (is_32bit_type(local_type)) { + *p_org++ = (uint8)local_offset; + } + else { + *p_org++ = (uint8)(local_offset | 0x80); + } + while (p_org < p) { + *p_org++ = WASM_OP_NOP; + } + } +#endif +#endif /* end of WASM_ENABLE_FAST_INTERP != 0 */ + break; + } + + case WASM_OP_SET_LOCAL: + { + p_org = p - 1; + GET_LOCAL_INDEX_TYPE_AND_OFFSET(); + +#if WASM_ENABLE_FAST_INTERP != 0 + if (!(preserve_referenced_local( + loader_ctx, opcode, local_offset, local_type, + &preserve_local, error_buf, error_buf_size))) + goto fail; + + if (local_offset < 256 +#if WASM_ENABLE_GC != 0 + && !wasm_is_type_reftype(local_type) +#endif + ) { + skip_label(); + if ((!preserve_local) && (LAST_OP_OUTPUT_I32())) { + if (loader_ctx->p_code_compiled) + STORE_U16(loader_ctx->p_code_compiled - 2, + local_offset); + loader_ctx->frame_offset--; + loader_ctx->dynamic_offset--; + } + else if ((!preserve_local) && (LAST_OP_OUTPUT_I64())) { + if (loader_ctx->p_code_compiled) + STORE_U16(loader_ctx->p_code_compiled - 2, + local_offset); + loader_ctx->frame_offset -= 2; + loader_ctx->dynamic_offset -= 2; + } + else { + if (is_32bit_type(local_type)) { + emit_label(EXT_OP_SET_LOCAL_FAST); + emit_byte(loader_ctx, (uint8)local_offset); + } + else if (is_64bit_type(local_type)) { + emit_label(EXT_OP_SET_LOCAL_FAST_I64); + emit_byte(loader_ctx, (uint8)local_offset); + } +#if WASM_ENABLE_SIMDE != 0 + else if (local_type == VALUE_TYPE_V128) { + emit_label(EXT_OP_SET_LOCAL_FAST_V128); + emit_byte(loader_ctx, (uint8)local_offset); + } +#endif + else { + set_error_buf(error_buf, error_buf_size, + "unknown local type"); + goto fail; + } + POP_OFFSET_TYPE(local_type); + } + } + else { /* local index larger than 255, reserve leb */ + emit_uint32(loader_ctx, local_idx); + POP_OFFSET_TYPE(local_type); + } +#else +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) \ + && (WASM_ENABLE_FAST_JIT == 0) && (WASM_ENABLE_DEBUG_INTERP == 0) + if (local_offset < 0x80 +#if WASM_ENABLE_GC != 0 + && !wasm_is_type_reftype(local_type) +#endif + ) { + *p_org++ = EXT_OP_SET_LOCAL_FAST; + if (is_32bit_type(local_type)) { + *p_org++ = (uint8)local_offset; + } + else { + *p_org++ = (uint8)(local_offset | 0x80); + } + while (p_org < p) { + *p_org++ = WASM_OP_NOP; + } + } +#endif +#endif /* end of WASM_ENABLE_FAST_INTERP != 0 */ + +#if WASM_ENABLE_GC != 0 + if (local_idx >= param_count) { + wasm_loader_mask_local(loader_ctx, local_idx - param_count); + } +#endif + + POP_TYPE(local_type); + break; + } + + case WASM_OP_TEE_LOCAL: + { + p_org = p - 1; + GET_LOCAL_INDEX_TYPE_AND_OFFSET(); +#if WASM_ENABLE_FAST_INTERP != 0 + /* If the stack is in polymorphic state, do fake pop and push on + offset stack to keep the depth of offset stack to be the + same with ref stack */ + BranchBlock *cur_block = loader_ctx->frame_csp - 1; + if (cur_block->is_stack_polymorphic) { + POP_OFFSET_TYPE(local_type); + PUSH_OFFSET_TYPE(local_type); + } +#endif + POP_TYPE(local_type); + PUSH_TYPE(local_type); + +#if WASM_ENABLE_FAST_INTERP != 0 + if (!(preserve_referenced_local( + loader_ctx, opcode, local_offset, local_type, + &preserve_local, error_buf, error_buf_size))) + goto fail; + + if (local_offset < 256 +#if WASM_ENABLE_GC != 0 + && !wasm_is_type_reftype(local_type) +#endif + ) { + skip_label(); + if (is_32bit_type(local_type)) { + emit_label(EXT_OP_TEE_LOCAL_FAST); + emit_byte(loader_ctx, (uint8)local_offset); + } +#if WASM_ENABLE_SIMDE != 0 + else if (local_type == VALUE_TYPE_V128) { + emit_label(EXT_OP_TEE_LOCAL_FAST_V128); + emit_byte(loader_ctx, (uint8)local_offset); + } +#endif + else { + emit_label(EXT_OP_TEE_LOCAL_FAST_I64); + emit_byte(loader_ctx, (uint8)local_offset); + } + } + else { /* local index larger than 255, reserve leb */ + emit_uint32(loader_ctx, local_idx); + } + emit_operand(loader_ctx, + *(loader_ctx->frame_offset + - wasm_value_type_cell_num(local_type))); +#else +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) \ + && (WASM_ENABLE_FAST_JIT == 0) && (WASM_ENABLE_DEBUG_INTERP == 0) + if (local_offset < 0x80 +#if WASM_ENABLE_GC != 0 + && !wasm_is_type_reftype(local_type) +#endif + ) { + *p_org++ = EXT_OP_TEE_LOCAL_FAST; + if (is_32bit_type(local_type)) { + *p_org++ = (uint8)local_offset; + } + else { + *p_org++ = (uint8)(local_offset | 0x80); + } + while (p_org < p) { + *p_org++ = WASM_OP_NOP; + } + } +#endif +#endif /* end of WASM_ENABLE_FAST_INTERP != 0 */ + +#if WASM_ENABLE_GC != 0 + if (local_idx >= param_count) { + wasm_loader_mask_local(loader_ctx, local_idx - param_count); + } +#endif + break; + } + + case WASM_OP_GET_GLOBAL: + { +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type; +#endif + + p_org = p - 1; + pb_read_leb_uint32(p, p_end, global_idx); + if (global_idx >= global_count) { + set_error_buf(error_buf, error_buf_size, "unknown global"); + goto fail; + } + + global_type = global_idx < module->import_global_count + ? module->import_globals[global_idx] + .u.global.type.val_type + : module + ->globals[global_idx + - module->import_global_count] + .type.val_type; +#if WASM_ENABLE_GC != 0 + ref_type = + global_idx < module->import_global_count + ? module->import_globals[global_idx].u.global.ref_type + : module + ->globals[global_idx + - module->import_global_count] + .ref_type; + if (wasm_is_type_multi_byte_type(global_type)) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), ref_type, + wasm_reftype_struct_size(ref_type)); + } +#endif + + PUSH_TYPE(global_type); + +#if WASM_ENABLE_FAST_INTERP == 0 + if (global_type == VALUE_TYPE_I64 + || global_type == VALUE_TYPE_F64) { +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!record_fast_op(module, p_org, *p_org, error_buf, + error_buf_size)) { + goto fail; + } +#endif + *p_org = WASM_OP_GET_GLOBAL_64; + } +#else /* else of WASM_ENABLE_FAST_INTERP */ + if (global_type == VALUE_TYPE_I64 + || global_type == VALUE_TYPE_F64) { + skip_label(); + emit_label(WASM_OP_GET_GLOBAL_64); + } +#if WASM_ENABLE_SIMDE != 0 + if (global_type == VALUE_TYPE_V128) { + skip_label(); + emit_label(WASM_OP_GET_GLOBAL_V128); + } +#endif /* end of WASM_ENABLE_SIMDE */ + emit_uint32(loader_ctx, global_idx); + PUSH_OFFSET_TYPE(global_type); +#endif /* end of WASM_ENABLE_FAST_INTERP */ + break; + } + + case WASM_OP_SET_GLOBAL: + { + bool is_mutable = false; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type; +#endif + + p_org = p - 1; + pb_read_leb_uint32(p, p_end, global_idx); + if (global_idx >= global_count) { + set_error_buf(error_buf, error_buf_size, "unknown global"); + goto fail; + } + + is_mutable = global_idx < module->import_global_count + ? module->import_globals[global_idx] + .u.global.type.is_mutable + : module + ->globals[global_idx + - module->import_global_count] + .type.is_mutable; + if (!is_mutable) { +#if WASM_ENABLE_GC == 0 + set_error_buf(error_buf, error_buf_size, + "global is immutable"); +#else + set_error_buf(error_buf, error_buf_size, + "immutable global"); +#endif + goto fail; + } + + global_type = global_idx < module->import_global_count + ? module->import_globals[global_idx] + .u.global.type.val_type + : module + ->globals[global_idx + - module->import_global_count] + .type.val_type; +#if WASM_ENABLE_GC != 0 + ref_type = + global_idx < module->import_global_count + ? module->import_globals[global_idx].u.global.ref_type + : module + ->globals[global_idx + - module->import_global_count] + .ref_type; + if (wasm_is_type_multi_byte_type(global_type)) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), ref_type, + wasm_reftype_struct_size(ref_type)); + } +#endif + +#if WASM_ENABLE_FAST_INTERP == 0 + if (global_type == VALUE_TYPE_I64 + || global_type == VALUE_TYPE_F64) { +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!record_fast_op(module, p_org, *p_org, error_buf, + error_buf_size)) { + goto fail; + } +#endif + *p_org = WASM_OP_SET_GLOBAL_64; + } + else if (module->aux_stack_size > 0 + && global_idx == module->aux_stack_top_global_index) { +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!record_fast_op(module, p_org, *p_org, error_buf, + error_buf_size)) { + goto fail; + } +#endif + *p_org = WASM_OP_SET_GLOBAL_AUX_STACK; +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_op_set_global_aux_stack = true; +#endif + } +#else /* else of WASM_ENABLE_FAST_INTERP */ + if (global_type == VALUE_TYPE_I64 + || global_type == VALUE_TYPE_F64) { + skip_label(); + emit_label(WASM_OP_SET_GLOBAL_64); + } + else if (module->aux_stack_size > 0 + && global_idx == module->aux_stack_top_global_index) { + skip_label(); + emit_label(WASM_OP_SET_GLOBAL_AUX_STACK); + } +#if WASM_ENABLE_SIMDE != 0 + else if (global_type == VALUE_TYPE_V128) { + skip_label(); + emit_label(WASM_OP_SET_GLOBAL_V128); + } +#endif /* end of WASM_ENABLE_SIMDE */ + emit_uint32(loader_ctx, global_idx); + POP_OFFSET_TYPE(global_type); +#endif /* end of WASM_ENABLE_FAST_INTERP */ + + POP_TYPE(global_type); + + break; + } + + /* load */ + case WASM_OP_I32_LOAD: + case WASM_OP_I32_LOAD8_S: + case WASM_OP_I32_LOAD8_U: + case WASM_OP_I32_LOAD16_S: + case WASM_OP_I32_LOAD16_U: + case WASM_OP_I64_LOAD: + case WASM_OP_I64_LOAD8_S: + case WASM_OP_I64_LOAD8_U: + case WASM_OP_I64_LOAD16_S: + case WASM_OP_I64_LOAD16_U: + case WASM_OP_I64_LOAD32_S: + case WASM_OP_I64_LOAD32_U: + case WASM_OP_F32_LOAD: + case WASM_OP_F64_LOAD: + /* store */ + case WASM_OP_I32_STORE: + case WASM_OP_I32_STORE8: + case WASM_OP_I32_STORE16: + case WASM_OP_I64_STORE: + case WASM_OP_I64_STORE8: + case WASM_OP_I64_STORE16: + case WASM_OP_I64_STORE32: + case WASM_OP_F32_STORE: + case WASM_OP_F64_STORE: + { +#if WASM_ENABLE_FAST_INTERP != 0 + /* change F32/F64 into I32/I64 */ + if (opcode == WASM_OP_F32_LOAD) { + skip_label(); + emit_label(WASM_OP_I32_LOAD); + } + else if (opcode == WASM_OP_F64_LOAD) { + skip_label(); + emit_label(WASM_OP_I64_LOAD); + } + else if (opcode == WASM_OP_F32_STORE) { + skip_label(); + emit_label(WASM_OP_I32_STORE); + } + else if (opcode == WASM_OP_F64_STORE) { + skip_label(); + emit_label(WASM_OP_I64_STORE); + } +#endif + CHECK_MEMORY(); + pb_read_leb_memarg(p, p_end, align); /* align */ + pb_read_leb_mem_offset(p, p_end, mem_offset); /* offset */ + if (!check_memory_access_align(opcode, align, error_buf, + error_buf_size)) { + goto fail; + } +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, mem_offset); +#endif +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + switch (opcode) { + /* load */ + case WASM_OP_I32_LOAD: + case WASM_OP_I32_LOAD8_S: + case WASM_OP_I32_LOAD8_U: + case WASM_OP_I32_LOAD16_S: + case WASM_OP_I32_LOAD16_U: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I32); + break; + case WASM_OP_I64_LOAD: + case WASM_OP_I64_LOAD8_S: + case WASM_OP_I64_LOAD8_U: + case WASM_OP_I64_LOAD16_S: + case WASM_OP_I64_LOAD16_U: + case WASM_OP_I64_LOAD32_S: + case WASM_OP_I64_LOAD32_U: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I64); + break; + case WASM_OP_F32_LOAD: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_F32); + break; + case WASM_OP_F64_LOAD: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_F64); + break; + /* store */ + case WASM_OP_I32_STORE: + case WASM_OP_I32_STORE8: + case WASM_OP_I32_STORE16: + POP_I32(); + POP_MEM_OFFSET(); + break; + case WASM_OP_I64_STORE: + case WASM_OP_I64_STORE8: + case WASM_OP_I64_STORE16: + case WASM_OP_I64_STORE32: + POP_I64(); + POP_MEM_OFFSET(); + break; + case WASM_OP_F32_STORE: + POP_F32(); + POP_MEM_OFFSET(); + break; + case WASM_OP_F64_STORE: + POP_F64(); + POP_MEM_OFFSET(); + break; + default: + break; + } + break; + } + + case WASM_OP_MEMORY_SIZE: + CHECK_MEMORY(); + pb_read_leb_uint32(p, p_end, memidx); + check_memidx(module, memidx); + PUSH_PAGE_COUNT(); + + module->possible_memory_grow = true; +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + + case WASM_OP_MEMORY_GROW: + CHECK_MEMORY(); + pb_read_leb_uint32(p, p_end, memidx); + check_memidx(module, memidx); + POP_AND_PUSH(mem_offset_type, mem_offset_type); + + module->possible_memory_grow = true; +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_op_memory_grow = true; +#endif +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + + case WASM_OP_I32_CONST: + pb_read_leb_int32(p, p_end, i32_const); +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + disable_emit = true; + GET_CONST_OFFSET(VALUE_TYPE_I32, i32_const); + + if (operand_offset == 0) { + disable_emit = false; + emit_label(WASM_OP_I32_CONST); + emit_uint32(loader_ctx, i32_const); + } +#else + (void)i32_const; +#endif + PUSH_I32(); + break; + + case WASM_OP_I64_CONST: + pb_read_leb_int64(p, p_end, i64_const); +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + disable_emit = true; + GET_CONST_OFFSET(VALUE_TYPE_I64, i64_const); + + if (operand_offset == 0) { + disable_emit = false; + emit_label(WASM_OP_I64_CONST); + emit_uint64(loader_ctx, i64_const); + } +#endif + PUSH_I64(); + break; + + case WASM_OP_F32_CONST: + CHECK_BUF(p, p_end, sizeof(float32)); + p += sizeof(float32); +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + disable_emit = true; + bh_memcpy_s((uint8 *)&f32_const, sizeof(float32), p_org, + sizeof(float32)); + GET_CONST_F32_OFFSET(VALUE_TYPE_F32, f32_const); + + if (operand_offset == 0) { + disable_emit = false; + emit_label(WASM_OP_F32_CONST); + emit_float32(loader_ctx, f32_const); + } +#endif + PUSH_F32(); + break; + + case WASM_OP_F64_CONST: + CHECK_BUF(p, p_end, sizeof(float64)); + p += sizeof(float64); +#if WASM_ENABLE_FAST_INTERP != 0 + skip_label(); + disable_emit = true; + /* Some MCU may require 8-byte align */ + bh_memcpy_s((uint8 *)&f64_const, sizeof(float64), p_org, + sizeof(float64)); + GET_CONST_F64_OFFSET(VALUE_TYPE_F64, f64_const); + + if (operand_offset == 0) { + disable_emit = false; + emit_label(WASM_OP_F64_CONST); + emit_float64(loader_ctx, f64_const); + } +#endif + PUSH_F64(); + break; + + case WASM_OP_I32_EQZ: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + break; + + case WASM_OP_I32_EQ: + case WASM_OP_I32_NE: + case WASM_OP_I32_LT_S: + case WASM_OP_I32_LT_U: + case WASM_OP_I32_GT_S: + case WASM_OP_I32_GT_U: + case WASM_OP_I32_LE_S: + case WASM_OP_I32_LE_U: + case WASM_OP_I32_GE_S: + case WASM_OP_I32_GE_U: + POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_EQZ: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_EQ: + case WASM_OP_I64_NE: + case WASM_OP_I64_LT_S: + case WASM_OP_I64_LT_U: + case WASM_OP_I64_GT_S: + case WASM_OP_I64_GT_U: + case WASM_OP_I64_LE_S: + case WASM_OP_I64_LE_U: + case WASM_OP_I64_GE_S: + case WASM_OP_I64_GE_U: + POP2_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I32); + break; + + case WASM_OP_F32_EQ: + case WASM_OP_F32_NE: + case WASM_OP_F32_LT: + case WASM_OP_F32_GT: + case WASM_OP_F32_LE: + case WASM_OP_F32_GE: + POP2_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I32); + break; + + case WASM_OP_F64_EQ: + case WASM_OP_F64_NE: + case WASM_OP_F64_LT: + case WASM_OP_F64_GT: + case WASM_OP_F64_LE: + case WASM_OP_F64_GE: + POP2_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I32); + break; + + case WASM_OP_I32_CLZ: + case WASM_OP_I32_CTZ: + case WASM_OP_I32_POPCNT: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + break; + + case WASM_OP_I32_ADD: + case WASM_OP_I32_SUB: + case WASM_OP_I32_MUL: + case WASM_OP_I32_DIV_S: + case WASM_OP_I32_DIV_U: + case WASM_OP_I32_REM_S: + case WASM_OP_I32_REM_U: + case WASM_OP_I32_AND: + case WASM_OP_I32_OR: + case WASM_OP_I32_XOR: + case WASM_OP_I32_SHL: + case WASM_OP_I32_SHR_S: + case WASM_OP_I32_SHR_U: + case WASM_OP_I32_ROTL: + case WASM_OP_I32_ROTR: + POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_CLZ: + case WASM_OP_I64_CTZ: + case WASM_OP_I64_POPCNT: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I64); + break; + + case WASM_OP_I64_ADD: + case WASM_OP_I64_SUB: + case WASM_OP_I64_MUL: + case WASM_OP_I64_DIV_S: + case WASM_OP_I64_DIV_U: + case WASM_OP_I64_REM_S: + case WASM_OP_I64_REM_U: + case WASM_OP_I64_AND: + case WASM_OP_I64_OR: + case WASM_OP_I64_XOR: + case WASM_OP_I64_SHL: + case WASM_OP_I64_SHR_S: + case WASM_OP_I64_SHR_U: + case WASM_OP_I64_ROTL: + case WASM_OP_I64_ROTR: + POP2_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I64); + break; + + case WASM_OP_F32_ABS: + case WASM_OP_F32_NEG: + case WASM_OP_F32_CEIL: + case WASM_OP_F32_FLOOR: + case WASM_OP_F32_TRUNC: + case WASM_OP_F32_NEAREST: + case WASM_OP_F32_SQRT: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_F32); + break; + + case WASM_OP_F32_ADD: + case WASM_OP_F32_SUB: + case WASM_OP_F32_MUL: + case WASM_OP_F32_DIV: + case WASM_OP_F32_MIN: + case WASM_OP_F32_MAX: + case WASM_OP_F32_COPYSIGN: + POP2_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_F32); + break; + + case WASM_OP_F64_ABS: + case WASM_OP_F64_NEG: + case WASM_OP_F64_CEIL: + case WASM_OP_F64_FLOOR: + case WASM_OP_F64_TRUNC: + case WASM_OP_F64_NEAREST: + case WASM_OP_F64_SQRT: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_F64); + break; + + case WASM_OP_F64_ADD: + case WASM_OP_F64_SUB: + case WASM_OP_F64_MUL: + case WASM_OP_F64_DIV: + case WASM_OP_F64_MIN: + case WASM_OP_F64_MAX: + case WASM_OP_F64_COPYSIGN: + POP2_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_F64); + break; + + case WASM_OP_I32_WRAP_I64: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I32); + break; + + case WASM_OP_I32_TRUNC_S_F32: + case WASM_OP_I32_TRUNC_U_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I32); + break; + + case WASM_OP_I32_TRUNC_S_F64: + case WASM_OP_I32_TRUNC_U_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_EXTEND_S_I32: + case WASM_OP_I64_EXTEND_U_I32: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I64); + break; + + case WASM_OP_I64_TRUNC_S_F32: + case WASM_OP_I64_TRUNC_U_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I64); + break; + + case WASM_OP_I64_TRUNC_S_F64: + case WASM_OP_I64_TRUNC_U_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I64); + break; + + case WASM_OP_F32_CONVERT_S_I32: + case WASM_OP_F32_CONVERT_U_I32: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F32); + break; + + case WASM_OP_F32_CONVERT_S_I64: + case WASM_OP_F32_CONVERT_U_I64: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_F32); + break; + + case WASM_OP_F32_DEMOTE_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_F32); + break; + + case WASM_OP_F64_CONVERT_S_I32: + case WASM_OP_F64_CONVERT_U_I32: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F64); + break; + + case WASM_OP_F64_CONVERT_S_I64: + case WASM_OP_F64_CONVERT_U_I64: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_F64); + break; + + case WASM_OP_F64_PROMOTE_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_F64); + break; + + case WASM_OP_I32_REINTERPRET_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_REINTERPRET_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I64); + break; + + case WASM_OP_F32_REINTERPRET_I32: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F32); + break; + + case WASM_OP_F64_REINTERPRET_I64: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_F64); + break; + + case WASM_OP_I32_EXTEND8_S: + case WASM_OP_I32_EXTEND16_S: + POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + break; + + case WASM_OP_I64_EXTEND8_S: + case WASM_OP_I64_EXTEND16_S: + case WASM_OP_I64_EXTEND32_S: + POP_AND_PUSH(VALUE_TYPE_I64, VALUE_TYPE_I64); + break; + +#if WASM_ENABLE_GC != 0 + case WASM_OP_GC_PREFIX: + { + uint32 opcode1; + + pb_read_leb_uint32(p, p_end, opcode1); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_byte(loader_ctx, ((uint8)opcode1)); +#endif + + switch (opcode1) { + case WASM_OP_STRUCT_NEW: + case WASM_OP_STRUCT_NEW_DEFAULT: + { + pb_read_leb_uint32(p, p_end, type_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, type_idx); +#endif + if (!check_type_index(module, module->type_count, + type_idx, error_buf, + error_buf_size)) { + goto fail; + } + if (module->types[type_idx]->type_flag + != WASM_TYPE_STRUCT) { + set_error_buf(error_buf, error_buf_size, + "unknown struct type"); + goto fail; + } + + if (opcode1 == WASM_OP_STRUCT_NEW) { + int32 j, k; + uint8 value_type; + uint32 ref_type_struct_size; + WASMStructType *struct_type = + (WASMStructType *)module->types[type_idx]; + + k = struct_type->ref_type_map_count - 1; + for (j = struct_type->field_count - 1; j >= 0; + j--) { + value_type = struct_type->fields[j].field_type; + if (wasm_is_type_reftype(value_type)) { + if (wasm_is_type_multi_byte_type( + value_type)) { + ref_type_struct_size = + wasm_reftype_struct_size( + struct_type->ref_type_maps[k] + .ref_type); + bh_memcpy_s( + &wasm_ref_type, + (uint32)sizeof(WASMRefType), + struct_type->ref_type_maps[k] + .ref_type, + ref_type_struct_size); + k--; + } + POP_REF(value_type); + } + else { + switch (value_type) { + case VALUE_TYPE_I32: + case PACKED_TYPE_I8: + case PACKED_TYPE_I16: + POP_I32(); + break; + case VALUE_TYPE_I64: + POP_I64(); + break; + case VALUE_TYPE_F32: + POP_F32(); + break; + case VALUE_TYPE_F64: + POP_F64(); + break; + default: + set_error_buf(error_buf, + error_buf_size, + "unknown type"); + goto fail; + } + } + } + } + + /* PUSH struct obj, (ref $t) */ + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, false, type_idx); + PUSH_REF(wasm_ref_type.ref_type); + break; + } + + case WASM_OP_STRUCT_GET: + case WASM_OP_STRUCT_GET_S: + case WASM_OP_STRUCT_GET_U: + case WASM_OP_STRUCT_SET: + { + WASMStructType *struct_type; + WASMRefType *ref_type = NULL; + uint32 field_idx; + uint8 field_type; + + pb_read_leb_uint32(p, p_end, type_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, type_idx); +#endif + if (!check_type_index(module, module->type_count, + type_idx, error_buf, + error_buf_size)) { + goto fail; + } + if (module->types[type_idx]->type_flag + != WASM_TYPE_STRUCT) { + set_error_buf(error_buf, error_buf_size, + "unknown struct type"); + goto fail; + } + struct_type = (WASMStructType *)module->types[type_idx]; + + pb_read_leb_uint32(p, p_end, field_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, field_idx); +#endif + if (field_idx >= struct_type->field_count) { + set_error_buf(error_buf, error_buf_size, + "unknown struct field"); + goto fail; + } + + if (opcode1 == WASM_OP_STRUCT_SET + && !(struct_type->fields[field_idx].field_flags + & 1)) { + set_error_buf(error_buf, error_buf_size, + "field is immutable"); + goto fail; + } + + field_type = struct_type->fields[field_idx].field_type; + if (is_packed_type(field_type)) { + if (opcode1 == WASM_OP_STRUCT_GET) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + else { + field_type = VALUE_TYPE_I32; + } + } + if (wasm_is_type_multi_byte_type(field_type)) { + ref_type = wasm_reftype_map_find( + struct_type->ref_type_maps, + struct_type->ref_type_map_count, field_idx); + bh_assert(ref_type); + } + if (opcode1 == WASM_OP_STRUCT_SET) { + /* POP field */ + if (wasm_is_type_multi_byte_type(field_type)) { + bh_memcpy_s(&wasm_ref_type, + (uint32)sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + } + POP_REF(field_type); + /* POP struct obj, (ref null $t) */ + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, true, type_idx); + POP_REF(wasm_ref_type.ref_type); + } + else { + /* POP struct obj, (ref null $t) */ + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, true, type_idx); + POP_REF(wasm_ref_type.ref_type); + /* PUSH field */ + if (wasm_is_type_multi_byte_type(field_type)) { + bh_memcpy_s(&wasm_ref_type, + (uint32)sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + } + PUSH_REF(field_type); + } + break; + } + + case WASM_OP_ARRAY_NEW: + case WASM_OP_ARRAY_NEW_DEFAULT: + case WASM_OP_ARRAY_NEW_FIXED: + case WASM_OP_ARRAY_NEW_DATA: + case WASM_OP_ARRAY_NEW_ELEM: + { + WASMArrayType *array_type; + uint8 elem_type; + uint32 u32 = 0; + + pb_read_leb_uint32(p, p_end, type_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, type_idx); +#endif + if (opcode1 == WASM_OP_ARRAY_NEW_FIXED + || opcode1 == WASM_OP_ARRAY_NEW_DATA + || opcode1 == WASM_OP_ARRAY_NEW_ELEM) { + pb_read_leb_uint32(p, p_end, u32); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, u32); +#endif + } + + if (!check_array_type(module, type_idx, error_buf, + error_buf_size)) { + goto fail; + } + + if (opcode1 != WASM_OP_ARRAY_NEW_FIXED) { + /* length */ + POP_I32(); + } + + array_type = (WASMArrayType *)module->types[type_idx]; + elem_type = array_type->elem_type; + + if (opcode1 == WASM_OP_ARRAY_NEW + || opcode1 == WASM_OP_ARRAY_NEW_FIXED) { + if (wasm_is_type_multi_byte_type(elem_type)) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + array_type->elem_ref_type, + wasm_reftype_struct_size( + array_type->elem_ref_type)); + } + if (is_packed_type(elem_type)) { + elem_type = VALUE_TYPE_I32; + } + + if (opcode1 == WASM_OP_ARRAY_NEW_FIXED) { + uint32 N = u32; + for (i = 0; i < N; i++) { + if (wasm_is_type_multi_byte_type( + elem_type)) { + bh_memcpy_s( + &wasm_ref_type, sizeof(WASMRefType), + array_type->elem_ref_type, + wasm_reftype_struct_size( + array_type->elem_ref_type)); + } + POP_REF(elem_type); + } + } + else + POP_REF(elem_type); + } + else if (opcode1 == WASM_OP_ARRAY_NEW_DATA) { + /* offset of data segment */ + POP_I32(); + + if (u32 >= module->data_seg_count) { + set_error_buf(error_buf, error_buf_size, + "unknown data segment"); + goto fail; + } + + if (wasm_is_type_reftype(elem_type)) { + set_error_buf(error_buf, error_buf_size, + "array elem type mismatch"); + goto fail; + } + } + else if (opcode1 == WASM_OP_ARRAY_NEW_ELEM) { + WASMTableSeg *table_seg = + module->table_segments + u32; + + /* offset of element segment */ + POP_I32(); + + if (u32 >= module->table_seg_count) { + set_error_buf(error_buf, error_buf_size, + "unknown element segment"); + goto fail; + } + if (!wasm_reftype_is_subtype_of( + table_seg->elem_type, + table_seg->elem_ref_type, elem_type, + array_type->elem_ref_type, module->types, + module->type_count)) { + set_error_buf(error_buf, error_buf_size, + "array elem type mismatch"); + goto fail; + } + } + + /* PUSH array obj, (ref $t) */ + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, false, type_idx); + PUSH_REF(wasm_ref_type.ref_type); + break; + } + + case WASM_OP_ARRAY_GET: + case WASM_OP_ARRAY_GET_S: + case WASM_OP_ARRAY_GET_U: + case WASM_OP_ARRAY_SET: + { + uint8 elem_type; + WASMArrayType *array_type; + WASMRefType *ref_type = NULL; + + pb_read_leb_uint32(p, p_end, type_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, type_idx); +#endif + if (!check_array_type(module, type_idx, error_buf, + error_buf_size)) { + goto fail; + } + array_type = (WASMArrayType *)module->types[type_idx]; + + if (opcode1 == WASM_OP_ARRAY_SET + && !(array_type->elem_flags & 1)) { + set_error_buf(error_buf, error_buf_size, + "array is immutable"); + goto fail; + } + + elem_type = array_type->elem_type; + if (is_packed_type(elem_type)) { + if (opcode1 != WASM_OP_ARRAY_GET_S + && opcode1 != WASM_OP_ARRAY_GET_U + && opcode1 != WASM_OP_ARRAY_SET) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + else { + elem_type = VALUE_TYPE_I32; + } + } + ref_type = array_type->elem_ref_type; + + if (opcode1 == WASM_OP_ARRAY_SET) { + /* POP elem to set */ + if (wasm_is_type_multi_byte_type(elem_type)) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + } + POP_REF(elem_type); + } + /* elem idx */ + POP_I32(); + /* POP array obj, (ref null $t) */ + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, true, type_idx); + POP_REF(wasm_ref_type.ref_type); + if (opcode1 != WASM_OP_ARRAY_SET) { + /* PUSH elem */ + if (wasm_is_type_multi_byte_type(elem_type)) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + } + PUSH_REF(elem_type); + } + break; + } + + case WASM_OP_ARRAY_LEN: + { + POP_REF(REF_TYPE_ARRAYREF); + /* length */ + PUSH_I32(); + break; + } + + case WASM_OP_ARRAY_FILL: + { + WASMArrayType *array_type; + uint8 elem_type; + /* typeidx */ + pb_read_leb_uint32(p, p_end, type_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, type_idx); +#endif + if (!check_array_type(module, type_idx, error_buf, + error_buf_size)) { + goto fail; + } + + array_type = (WASMArrayType *)module->types[type_idx]; + if (!(array_type->elem_flags & 1)) { + set_error_buf(error_buf, error_buf_size, + "array is immutable"); + goto fail; + } + + elem_type = array_type->elem_type; + if (is_packed_type(elem_type)) { + elem_type = VALUE_TYPE_I32; + } + + POP_I32(); /* length */ +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(elem_type); +#endif + POP_TYPE(elem_type); + POP_I32(); /* start */ + /* POP array obj, (ref null $t) */ + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, true, type_idx); + POP_REF(wasm_ref_type.ref_type); + + break; + } + + case WASM_OP_ARRAY_COPY: + { + uint32 src_type_idx; + uint8 src_elem_type, dst_elem_type; + WASMRefType src_ref_type = { 0 }, + *src_elem_ref_type = NULL; + WASMRefType dst_ref_type = { 0 }, + *dst_elem_ref_type = NULL; + WASMArrayType *array_type; + + /* typeidx1 */ + pb_read_leb_uint32(p, p_end, type_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, type_idx); +#endif + /* typeidx2 */ + pb_read_leb_uint32(p, p_end, src_type_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, src_type_idx); +#endif + if (!check_array_type(module, type_idx, error_buf, + error_buf_size)) { + goto fail; + } + + if (!check_array_type(module, src_type_idx, error_buf, + error_buf_size)) { + goto fail; + } + + POP_I32(); + POP_I32(); + /* POP array obj, (ref null $t) */ + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, true, src_type_idx); + POP_REF(wasm_ref_type.ref_type); + bh_memcpy_s(&src_ref_type, (uint32)sizeof(WASMRefType), + &wasm_ref_type, + wasm_reftype_struct_size(&wasm_ref_type)); + POP_I32(); + /* POP array obj, (ref null $t) */ + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, true, type_idx); + POP_REF(wasm_ref_type.ref_type); + bh_memcpy_s(&dst_ref_type, (uint32)sizeof(WASMRefType), + &wasm_ref_type, + wasm_reftype_struct_size(&wasm_ref_type)); + + array_type = (WASMArrayType *)module->types[type_idx]; + if (!(array_type->elem_flags & 1)) { + set_error_buf(error_buf, error_buf_size, + "destination array is immutable"); + goto fail; + } + + dst_elem_type = array_type->elem_type; + if (wasm_is_type_multi_byte_type(dst_elem_type)) { + dst_elem_ref_type = array_type->elem_ref_type; + } + + array_type = + (WASMArrayType *)module->types[src_type_idx]; + src_elem_type = array_type->elem_type; + if (wasm_is_type_multi_byte_type(src_elem_type)) { + src_elem_ref_type = array_type->elem_ref_type; + } + + if (!wasm_reftype_is_subtype_of( + src_elem_type, src_elem_ref_type, dst_elem_type, + dst_elem_ref_type, module->types, + module->type_count)) { + set_error_buf(error_buf, error_buf_size, + "array types do not match"); + goto fail; + } + + break; + } + + case WASM_OP_REF_I31: + { + POP_I32(); + wasm_set_refheaptype_common( + &wasm_ref_type.ref_ht_common, false, HEAP_TYPE_I31); + PUSH_REF(wasm_ref_type.ref_type); + break; + } + + case WASM_OP_I31_GET_S: + case WASM_OP_I31_GET_U: + { + POP_REF(REF_TYPE_I31REF); + PUSH_I32(); + break; + } + + case WASM_OP_REF_TEST: + case WASM_OP_REF_CAST: + case WASM_OP_REF_TEST_NULLABLE: + case WASM_OP_REF_CAST_NULLABLE: + { + uint8 type; + + pb_read_leb_int32(p, p_end, heap_type); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, (uint32)heap_type); +#endif + if (heap_type >= 0) { + if (!check_type_index(module, module->type_count, + heap_type, error_buf, + error_buf_size)) { + goto fail; + } + } + else { + if (!wasm_is_valid_heap_type(heap_type)) { + set_error_buf(error_buf, error_buf_size, + "unknown type"); + goto fail; + } + } + if (!wasm_loader_pop_heap_obj(loader_ctx, &type, + &wasm_ref_type, error_buf, + error_buf_size)) { + goto fail; + } + if (opcode1 == WASM_OP_REF_TEST + || opcode1 == WASM_OP_REF_TEST_NULLABLE) + PUSH_I32(); + else { + bool nullable = + (opcode1 == WASM_OP_REF_CAST_NULLABLE) ? true + : false; + if (heap_type >= 0 || !nullable) { + wasm_set_refheaptype_typeidx( + &wasm_ref_type.ref_ht_typeidx, nullable, + heap_type); + PUSH_REF(wasm_ref_type.ref_type); + } + else { + PUSH_REF((uint8)((int32)0x80 + heap_type)); + } + } + break; + } + + case WASM_OP_BR_ON_CAST: + case WASM_OP_BR_ON_CAST_FAIL: + { + WASMRefType ref_type_tmp = { 0 }, ref_type1 = { 0 }, + ref_type2 = { 0 }, ref_type_diff = { 0 }; + uint8 type_tmp, castflags; + uint32 depth; + int32 heap_type_dst; + bool src_nullable, dst_nullable; + + CHECK_BUF(p, p_end, 1); + castflags = read_uint8(p); +#if WASM_ENABLE_FAST_INTERP != 0 + /* Emit heap_type firstly */ + emit_byte(loader_ctx, castflags); +#endif + + p_org = p; + pb_read_leb_uint32(p, p_end, depth); + pb_read_leb_int32(p, p_end, heap_type); +#if WASM_ENABLE_FAST_INTERP != 0 + /* Emit heap_type firstly */ + emit_uint32(loader_ctx, (uint32)heap_type); +#endif + pb_read_leb_int32(p, p_end, heap_type_dst); +#if WASM_ENABLE_FAST_INTERP != 0 + /* Emit heap_type firstly */ + emit_uint32(loader_ctx, (uint32)heap_type_dst); +#endif + (void)depth; + + /* + * castflags should be 0~3: + * 0: (non-null, non-null) + * 1: (null, non-null) + * 2: (non-null, null) + * 3: (null, null) + */ + if (castflags > 3) { + set_error_buf(error_buf, error_buf_size, + "invalid castflags"); + break; + } + src_nullable = + (castflags == 1) || (castflags == 3) ? true : false; + dst_nullable = + (castflags == 2) || (castflags == 3) ? true : false; + + /* Pop and backup the stack top's ref type */ + if (!wasm_loader_pop_heap_obj(loader_ctx, &type_tmp, + &ref_type_tmp, error_buf, + error_buf_size)) { + goto fail; + } + + /* The reference type rt1 must be valid */ + if (!init_ref_type(module, &ref_type1, src_nullable, + heap_type, error_buf, + error_buf_size)) { + goto fail; + } + + /* The reference type rt2 must be valid. */ + if (!init_ref_type(module, &ref_type2, dst_nullable, + heap_type_dst, error_buf, + error_buf_size)) { + goto fail; + } + + calculate_reftype_diff(&ref_type_diff, &ref_type1, + &ref_type2); + + /* The reference type rt2 must match rt1. */ + if (!wasm_reftype_is_subtype_of( + ref_type2.ref_type, &ref_type2, + ref_type1.ref_type, &ref_type1, module->types, + module->type_count)) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + + p = p_org; + /* Push ref type casted for branch block check */ + if (opcode1 == WASM_OP_BR_ON_CAST) { + /* The reference type rt2 must match rt′. */ + type_tmp = ref_type2.ref_type; + if (wasm_is_type_multi_byte_type(type_tmp)) { + bh_memcpy_s( + &wasm_ref_type, + wasm_reftype_struct_size(&ref_type2), + &ref_type2, + wasm_reftype_struct_size(&ref_type2)); + } + } + else { + /* The reference type rt′1 must match rt′. */ + type_tmp = ref_type_diff.ref_type; + if (wasm_is_type_multi_byte_type(type_tmp)) { + bh_memcpy_s( + &wasm_ref_type, + wasm_reftype_struct_size(&ref_type_diff), + &ref_type_diff, + wasm_reftype_struct_size(&ref_type_diff)); + } + } + PUSH_REF(type_tmp); + if (!(frame_csp_tmp = check_branch_block( + loader_ctx, &p, p_end, opcode, error_buf, + error_buf_size))) { + goto fail; + } + /* Ignore heap_types */ + skip_leb_uint32(p, p_end); + skip_leb_uint32(p, p_end); + + /* Restore the original stack top's ref type */ + POP_REF(type_tmp); +#if WASM_ENABLE_FAST_INTERP != 0 + /* Erase the opnd offset emitted by POP_REF() */ + wasm_loader_emit_backspace(loader_ctx, sizeof(uint16)); +#endif + if (opcode1 == WASM_OP_BR_ON_CAST) { + type_tmp = ref_type_diff.ref_type; + if (wasm_is_type_multi_byte_type(type_tmp)) { + bh_memcpy_s( + &wasm_ref_type, + wasm_reftype_struct_size(&ref_type_diff), + &ref_type_diff, + wasm_reftype_struct_size(&ref_type_diff)); + } + } + else { + type_tmp = ref_type2.ref_type; + if (wasm_is_type_multi_byte_type(type_tmp)) { + bh_memcpy_s( + &wasm_ref_type, + wasm_reftype_struct_size(&ref_type2), + &ref_type2, + wasm_reftype_struct_size(&ref_type2)); + } + } + PUSH_REF(type_tmp); + +#if WASM_ENABLE_FAST_INTERP != 0 + /* Erase the opnd offset emitted by PUSH_REF() */ + wasm_loader_emit_backspace(loader_ctx, sizeof(uint16)); +#endif + break; + } + + case WASM_OP_ANY_CONVERT_EXTERN: + { + uint8 type; + + if (!wasm_loader_pop_heap_obj(loader_ctx, &type, + &wasm_ref_type, error_buf, + error_buf_size)) { + goto fail; + } + if (!(type == REF_TYPE_EXTERNREF + || (type == REF_TYPE_HT_NON_NULLABLE + && wasm_ref_type.ref_ht_common.heap_type + == HEAP_TYPE_EXTERN) + || type == VALUE_TYPE_ANY)) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + + if (type == REF_TYPE_EXTERNREF) + type = REF_TYPE_ANYREF; + else { + wasm_ref_type.ref_ht_common.heap_type = + HEAP_TYPE_ANY; + } + PUSH_REF(type); + break; + } + + case WASM_OP_EXTERN_CONVERT_ANY: + { + uint8 type; + + if (!wasm_loader_pop_heap_obj(loader_ctx, &type, + &wasm_ref_type, error_buf, + error_buf_size)) { + goto fail; + } + if (type == REF_TYPE_EXTERNREF + || ((type == REF_TYPE_HT_NULLABLE + || type == REF_TYPE_HT_NON_NULLABLE) + && wasm_ref_type.ref_ht_common.heap_type + == HEAP_TYPE_EXTERN)) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } + + if (type != REF_TYPE_HT_NON_NULLABLE) { + /* push (ref null extern) */ + type = REF_TYPE_EXTERNREF; + } + else { + /* push (ref extern) */ + type = REF_TYPE_HT_NON_NULLABLE; + wasm_set_refheaptype_common( + &wasm_ref_type.ref_ht_common, false, + HEAP_TYPE_EXTERN); + } + PUSH_REF(type); + break; + } + +#if WASM_ENABLE_STRINGREF != 0 + case WASM_OP_STRING_NEW_UTF8: + case WASM_OP_STRING_NEW_WTF16: + case WASM_OP_STRING_NEW_LOSSY_UTF8: + case WASM_OP_STRING_NEW_WTF8: + { +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + + pb_read_leb_uint32(p, p_end, memidx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, (uint32)memidx); +#endif + POP_I32(); + POP_I32(); + PUSH_REF(REF_TYPE_STRINGREF); + break; + } + case WASM_OP_STRING_CONST: + { + uint32 contents; + + pb_read_leb_uint32(p, p_end, contents); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, (uint32)contents); +#endif + PUSH_REF(REF_TYPE_STRINGREF); + (void)contents; + break; + } + case WASM_OP_STRING_MEASURE_UTF8: + case WASM_OP_STRING_MEASURE_WTF8: + case WASM_OP_STRING_MEASURE_WTF16: + { + POP_STRINGREF(); + PUSH_I32(); + break; + } + case WASM_OP_STRING_ENCODE_UTF8: + case WASM_OP_STRING_ENCODE_WTF16: + case WASM_OP_STRING_ENCODE_LOSSY_UTF8: + case WASM_OP_STRING_ENCODE_WTF8: + { +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + + pb_read_leb_uint32(p, p_end, memidx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, (uint32)memidx); +#endif + POP_I32(); + POP_STRINGREF(); + PUSH_I32(); + break; + } + case WASM_OP_STRING_CONCAT: + { + POP_STRINGREF(); + POP_STRINGREF(); + PUSH_REF(REF_TYPE_STRINGREF); + break; + } + case WASM_OP_STRING_EQ: + { + POP_STRINGREF(); + POP_STRINGREF(); + PUSH_I32(); + break; + } + case WASM_OP_STRING_IS_USV_SEQUENCE: + { + POP_STRINGREF(); + PUSH_I32(); + break; + } + case WASM_OP_STRING_AS_WTF8: + { + POP_STRINGREF(); + PUSH_REF(REF_TYPE_STRINGVIEWWTF8); + break; + } + case WASM_OP_STRINGVIEW_WTF8_ADVANCE: + { + POP_I32(); + POP_I32(); + POP_REF(REF_TYPE_STRINGVIEWWTF8); + PUSH_I32(); + break; + } + case WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8: + case WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8: + case WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8: + { +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + + pb_read_leb_uint32(p, p_end, memidx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, (uint32)memidx); +#endif + POP_I32(); + POP_I32(); + POP_I32(); + POP_REF(REF_TYPE_STRINGVIEWWTF8); + PUSH_I32(); + PUSH_I32(); + break; + } + case WASM_OP_STRINGVIEW_WTF8_SLICE: + { + POP_I32(); + POP_I32(); + POP_REF(REF_TYPE_STRINGVIEWWTF8); + PUSH_REF(REF_TYPE_STRINGREF); + break; + } + case WASM_OP_STRING_AS_WTF16: + { + POP_STRINGREF(); + PUSH_REF(REF_TYPE_STRINGVIEWWTF16); + break; + } + case WASM_OP_STRINGVIEW_WTF16_LENGTH: + { + POP_REF(REF_TYPE_STRINGVIEWWTF16); + PUSH_I32(); + break; + } + case WASM_OP_STRINGVIEW_WTF16_GET_CODEUNIT: + { + POP_I32(); + POP_REF(REF_TYPE_STRINGVIEWWTF16); + PUSH_I32(); + break; + } + case WASM_OP_STRINGVIEW_WTF16_ENCODE: + { +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + + pb_read_leb_uint32(p, p_end, memidx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, (uint32)memidx); +#endif + POP_I32(); + POP_I32(); + POP_I32(); + POP_REF(REF_TYPE_STRINGVIEWWTF16); + PUSH_I32(); + break; + } + case WASM_OP_STRINGVIEW_WTF16_SLICE: + { + POP_I32(); + POP_I32(); + POP_REF(REF_TYPE_STRINGVIEWWTF16); + PUSH_REF(REF_TYPE_STRINGREF); + break; + } + case WASM_OP_STRING_AS_ITER: + { + POP_STRINGREF(); + PUSH_REF(REF_TYPE_STRINGVIEWITER); + break; + } + case WASM_OP_STRINGVIEW_ITER_NEXT: + { + POP_REF(REF_TYPE_STRINGVIEWITER); + PUSH_I32(); + break; + } + case WASM_OP_STRINGVIEW_ITER_ADVANCE: + case WASM_OP_STRINGVIEW_ITER_REWIND: + { + POP_I32(); + POP_REF(REF_TYPE_STRINGVIEWITER); + PUSH_I32(); + break; + } + case WASM_OP_STRINGVIEW_ITER_SLICE: + { + POP_I32(); + POP_REF(REF_TYPE_STRINGVIEWITER); + PUSH_REF(REF_TYPE_STRINGREF); + break; + } + case WASM_OP_STRING_NEW_UTF8_ARRAY: + case WASM_OP_STRING_NEW_WTF16_ARRAY: + case WASM_OP_STRING_NEW_LOSSY_UTF8_ARRAY: + case WASM_OP_STRING_NEW_WTF8_ARRAY: + { + POP_I32(); + POP_I32(); + POP_REF(REF_TYPE_ARRAYREF); + PUSH_REF(REF_TYPE_STRINGREF); + break; + } + case WASM_OP_STRING_ENCODE_UTF8_ARRAY: + case WASM_OP_STRING_ENCODE_WTF16_ARRAY: + case WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY: + case WASM_OP_STRING_ENCODE_WTF8_ARRAY: + { + POP_I32(); + POP_REF(REF_TYPE_ARRAYREF); + POP_STRINGREF(); + PUSH_I32(); + break; + } +#endif /* end of WASM_ENABLE_STRINGREF != 0 */ + default: + set_error_buf_v(error_buf, error_buf_size, + "%s %02x %02x", "unsupported opcode", + 0xfb, opcode1); + goto fail; + } + break; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + + case WASM_OP_MISC_PREFIX: + { + uint32 opcode1; + + pb_read_leb_uint32(p, p_end, opcode1); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_byte(loader_ctx, ((uint8)opcode1)); +#endif + switch (opcode1) { + case WASM_OP_I32_TRUNC_SAT_S_F32: + case WASM_OP_I32_TRUNC_SAT_U_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I32); + break; + case WASM_OP_I32_TRUNC_SAT_S_F64: + case WASM_OP_I32_TRUNC_SAT_U_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I32); + break; + case WASM_OP_I64_TRUNC_SAT_S_F32: + case WASM_OP_I64_TRUNC_SAT_U_F32: + POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I64); + break; + case WASM_OP_I64_TRUNC_SAT_S_F64: + case WASM_OP_I64_TRUNC_SAT_U_F64: + POP_AND_PUSH(VALUE_TYPE_F64, VALUE_TYPE_I64); + break; +#if WASM_ENABLE_BULK_MEMORY != 0 + case WASM_OP_MEMORY_INIT: + { + pb_read_leb_uint32(p, p_end, data_seg_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, data_seg_idx); +#endif + if (module->import_memory_count == 0 + && module->memory_count == 0) { + set_error_buf(error_buf, error_buf_size, + "unknown memory 0"); + goto fail; + } + + pb_read_leb_uint32(p, p_end, memidx); + check_memidx(module, memidx); + + if (data_seg_idx >= module->data_seg_count) { + set_error_buf_v(error_buf, error_buf_size, + "unknown data segment %d", + data_seg_idx); + goto fail; + } + + if (module->data_seg_count1 == 0) + goto fail_data_cnt_sec_require; + + POP_I32(); + POP_I32(); + POP_MEM_OFFSET(); +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_bulk_memory_used = true; +#endif + break; + } + case WASM_OP_DATA_DROP: + { + pb_read_leb_uint32(p, p_end, data_seg_idx); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, data_seg_idx); +#endif + if (data_seg_idx >= module->data_seg_count) { + set_error_buf(error_buf, error_buf_size, + "unknown data segment"); + goto fail; + } + + if (module->data_seg_count1 == 0) + goto fail_data_cnt_sec_require; + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_bulk_memory_used = true; +#endif + break; + } + fail_data_cnt_sec_require: + set_error_buf(error_buf, error_buf_size, + "data count section required"); + goto fail; +#endif /* WASM_ENABLE_BULK_MEMORY */ +#if WASM_ENABLE_BULK_MEMORY_OPT != 0 + case WASM_OP_MEMORY_COPY: + { + CHECK_BUF(p, p_end, sizeof(int16)); + /* check both src and dst memory index */ + pb_read_leb_uint32(p, p_end, memidx); + check_memidx(module, memidx); + pb_read_leb_uint32(p, p_end, memidx); + check_memidx(module, memidx); + + if (module->import_memory_count == 0 + && module->memory_count == 0) { + set_error_buf(error_buf, error_buf_size, + "unknown memory 0"); + goto fail; + } + + POP_MEM_OFFSET(); + POP_MEM_OFFSET(); + POP_MEM_OFFSET(); +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_bulk_memory_used = true; +#endif + break; + } + case WASM_OP_MEMORY_FILL: + { + pb_read_leb_uint32(p, p_end, memidx); + check_memidx(module, memidx); + if (module->import_memory_count == 0 + && module->memory_count == 0) { + set_error_buf(error_buf, error_buf_size, + "unknown memory 0"); + goto fail; + } + POP_MEM_OFFSET(); + POP_I32(); + POP_MEM_OFFSET(); +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_bulk_memory_used = true; +#endif + break; + } +#endif /* WASM_ENABLE_BULK_MEMORY_OPT */ +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + case WASM_OP_TABLE_INIT: + { + uint8 seg_type = 0, tbl_type = 0; +#if WASM_ENABLE_GC != 0 + WASMRefType *seg_ref_type = NULL, *tbl_ref_type = NULL; +#endif + + pb_read_leb_uint32(p, p_end, table_seg_idx); + pb_read_leb_uint32(p, p_end, table_idx); + + if (!get_table_elem_type(module, table_idx, &tbl_type, +#if WASM_ENABLE_GC != 0 + (void **)&tbl_ref_type, +#else + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + + if (!get_table_seg_elem_type(module, table_seg_idx, + &seg_type, +#if WASM_ENABLE_GC != 0 + (void **)&seg_ref_type, +#else + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + +#if WASM_ENABLE_GC == 0 + if (seg_type != tbl_type) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } +#else + if (!wasm_reftype_is_subtype_of( + seg_type, seg_ref_type, tbl_type, tbl_ref_type, + module->types, module->type_count)) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, table_seg_idx); + emit_uint32(loader_ctx, table_idx); +#endif + POP_I32(); + POP_I32(); +#if WASM_ENABLE_MEMORY64 != 0 + table_elem_idx_type = is_table_64bit(module, table_idx) + ? VALUE_TYPE_I64 + : VALUE_TYPE_I32; +#endif + POP_TBL_ELEM_IDX(); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } + case WASM_OP_ELEM_DROP: + { + pb_read_leb_uint32(p, p_end, table_seg_idx); + if (!get_table_seg_elem_type(module, table_seg_idx, + NULL, NULL, error_buf, + error_buf_size)) + goto fail; +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, table_seg_idx); +#endif + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } + case WASM_OP_TABLE_COPY: + { + uint8 src_type, dst_type, src_tbl_idx_type, + dst_tbl_idx_type, min_tbl_idx_type; +#if WASM_ENABLE_GC != 0 + WASMRefType *src_ref_type = NULL, *dst_ref_type = NULL; +#endif + uint32 src_tbl_idx, dst_tbl_idx; + + pb_read_leb_uint32(p, p_end, dst_tbl_idx); + if (!get_table_elem_type(module, dst_tbl_idx, &dst_type, +#if WASM_ENABLE_GC != 0 + (void **)&dst_ref_type, +#else + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + + pb_read_leb_uint32(p, p_end, src_tbl_idx); + if (!get_table_elem_type(module, src_tbl_idx, &src_type, +#if WASM_ENABLE_GC != 0 + (void **)&src_ref_type, +#else + NULL, +#endif + error_buf, error_buf_size)) + goto fail; + +#if WASM_ENABLE_GC == 0 + if (src_type != dst_type) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } +#else + if (!wasm_reftype_is_subtype_of( + src_type, src_ref_type, dst_type, dst_ref_type, + module->types, module->type_count)) { + set_error_buf(error_buf, error_buf_size, + "type mismatch"); + goto fail; + } +#endif + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, dst_tbl_idx); + emit_uint32(loader_ctx, src_tbl_idx); +#endif + +#if WASM_ENABLE_MEMORY64 != 0 + src_tbl_idx_type = is_table_64bit(module, src_tbl_idx) + ? VALUE_TYPE_I64 + : VALUE_TYPE_I32; + dst_tbl_idx_type = is_table_64bit(module, dst_tbl_idx) + ? VALUE_TYPE_I64 + : VALUE_TYPE_I32; + min_tbl_idx_type = + (src_tbl_idx_type == VALUE_TYPE_I32 + || dst_tbl_idx_type == VALUE_TYPE_I32) + ? VALUE_TYPE_I32 + : VALUE_TYPE_I64; +#else + src_tbl_idx_type = VALUE_TYPE_I32; + dst_tbl_idx_type = VALUE_TYPE_I32; + min_tbl_idx_type = VALUE_TYPE_I32; +#endif + + table_elem_idx_type = min_tbl_idx_type; + POP_TBL_ELEM_IDX(); + table_elem_idx_type = src_tbl_idx_type; + POP_TBL_ELEM_IDX(); + table_elem_idx_type = dst_tbl_idx_type; + POP_TBL_ELEM_IDX(); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } + case WASM_OP_TABLE_SIZE: + { + pb_read_leb_uint32(p, p_end, table_idx); + /* TODO: shall we create a new function to check + table idx instead of using below function? */ + if (!get_table_elem_type(module, table_idx, NULL, NULL, + error_buf, error_buf_size)) + goto fail; + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, table_idx); +#endif + +#if WASM_ENABLE_MEMORY64 != 0 + table_elem_idx_type = is_table_64bit(module, table_idx) + ? VALUE_TYPE_I64 + : VALUE_TYPE_I32; +#endif + PUSH_TBL_ELEM_IDX(); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } + case WASM_OP_TABLE_GROW: + case WASM_OP_TABLE_FILL: + { + uint8 decl_type; +#if WASM_ENABLE_GC != 0 + WASMRefType *ref_type = NULL; +#endif + + pb_read_leb_uint32(p, p_end, table_idx); + if (!get_table_elem_type(module, table_idx, &decl_type, +#if WASM_ENABLE_GC != 0 + (void **)&ref_type, +#else + NULL, +#endif + error_buf, error_buf_size)) + goto fail; +#if WASM_ENABLE_GC != 0 + if (wasm_is_type_multi_byte_type(decl_type)) { + bh_memcpy_s(&wasm_ref_type, sizeof(WASMRefType), + ref_type, + wasm_reftype_struct_size(ref_type)); + } +#endif + + if (opcode1 == WASM_OP_TABLE_GROW) { + if (table_idx < module->import_table_count) { + module->import_tables[table_idx] + .u.table.table_type.possible_grow = true; + } + else { + module + ->tables[table_idx + - module->import_table_count] + .table_type.possible_grow = true; + } + } + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, table_idx); +#endif + +#if WASM_ENABLE_MEMORY64 != 0 + table_elem_idx_type = is_table_64bit(module, table_idx) + ? VALUE_TYPE_I64 + : VALUE_TYPE_I32; +#endif + POP_TBL_ELEM_IDX(); +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(decl_type); +#endif + POP_TYPE(decl_type); + if (opcode1 == WASM_OP_TABLE_GROW) + PUSH_TBL_ELEM_IDX(); + else + POP_TBL_ELEM_IDX(); + +#if WASM_ENABLE_WAMR_COMPILER != 0 + module->is_ref_types_used = true; +#endif + break; + } +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + default: + set_error_buf_v(error_buf, error_buf_size, + "%s %02x %02x", "unsupported opcode", + 0xfc, opcode1); + goto fail; + } + break; + } + +#if WASM_ENABLE_SIMD != 0 +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) \ + || (WASM_ENABLE_FAST_INTERP != 0) + case WASM_OP_SIMD_PREFIX: + { + uint32 opcode1; + +#if WASM_ENABLE_WAMR_COMPILER != 0 + /* Mark the SIMD instruction is used in this module */ + module->is_simd_used = true; +#endif + + pb_read_leb_uint32(p, p_end, opcode1); + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_byte(loader_ctx, opcode1); +#endif + + /* follow the order of enum WASMSimdEXTOpcode in wasm_opcode.h + */ + switch (opcode1) { + /* memory instruction */ + case SIMD_v128_load: + case SIMD_v128_load8x8_s: + case SIMD_v128_load8x8_u: + case SIMD_v128_load16x4_s: + case SIMD_v128_load16x4_u: + case SIMD_v128_load32x2_s: + case SIMD_v128_load32x2_u: + case SIMD_v128_load8_splat: + case SIMD_v128_load16_splat: + case SIMD_v128_load32_splat: + case SIMD_v128_load64_splat: + { + CHECK_MEMORY(); + + pb_read_leb_uint32(p, p_end, align); /* align */ + if (!check_simd_memory_access_align( + opcode1, align, error_buf, error_buf_size)) { + goto fail; + } + + pb_read_leb_mem_offset(p, p_end, + mem_offset); /* offset */ + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, mem_offset); +#endif + + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_V128); +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + } + + case SIMD_v128_store: + { + CHECK_MEMORY(); + + pb_read_leb_uint32(p, p_end, align); /* align */ + if (!check_simd_memory_access_align( + opcode1, align, error_buf, error_buf_size)) { + goto fail; + } + + pb_read_leb_mem_offset(p, p_end, + mem_offset); /* offset */ + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, mem_offset); +#endif + + POP_V128(); + POP_MEM_OFFSET(); +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + } + + /* basic operation */ + case SIMD_v128_const: + { +#if WASM_ENABLE_FAST_INTERP != 0 + uint64 high, low; +#endif + CHECK_BUF1(p, p_end, 16); +#if WASM_ENABLE_FAST_INTERP != 0 + wasm_runtime_read_v128(p, &high, &low); + emit_uint64(loader_ctx, high); + emit_uint64(loader_ctx, low); +#endif + p += 16; + PUSH_V128(); + break; + } + + case SIMD_v8x16_shuffle: + { + V128 mask; + + CHECK_BUF1(p, p_end, 16); + mask = read_i8x16(p, error_buf, error_buf_size); + if (!check_simd_shuffle_mask(mask, error_buf, + error_buf_size)) { + goto fail; + } +#if WASM_ENABLE_FAST_INTERP != 0 + uint64 high, low; + wasm_runtime_read_v128(p, &high, &low); + emit_uint64(loader_ctx, high); + emit_uint64(loader_ctx, low); +#endif + p += 16; + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_v8x16_swizzle: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + /* splat operation */ + case SIMD_i8x16_splat: + case SIMD_i16x8_splat: + case SIMD_i32x4_splat: + case SIMD_i64x2_splat: + case SIMD_f32x4_splat: + case SIMD_f64x2_splat: + { + uint8 pop_type[] = { VALUE_TYPE_I32, VALUE_TYPE_I32, + VALUE_TYPE_I32, VALUE_TYPE_I64, + VALUE_TYPE_F32, VALUE_TYPE_F64 }; + POP_AND_PUSH(pop_type[opcode1 - SIMD_i8x16_splat], + VALUE_TYPE_V128); + break; + } + + /* lane operation */ + case SIMD_i8x16_extract_lane_s: + case SIMD_i8x16_extract_lane_u: + case SIMD_i8x16_replace_lane: + case SIMD_i16x8_extract_lane_s: + case SIMD_i16x8_extract_lane_u: + case SIMD_i16x8_replace_lane: + case SIMD_i32x4_extract_lane: + case SIMD_i32x4_replace_lane: + case SIMD_i64x2_extract_lane: + case SIMD_i64x2_replace_lane: + case SIMD_f32x4_extract_lane: + case SIMD_f32x4_replace_lane: + case SIMD_f64x2_extract_lane: + case SIMD_f64x2_replace_lane: + { + uint8 lane; + /* clang-format off */ + uint8 replace[] = { + /*i8x16*/ 0x0, 0x0, VALUE_TYPE_I32, + /*i16x8*/ 0x0, 0x0, VALUE_TYPE_I32, + /*i32x4*/ 0x0, VALUE_TYPE_I32, + /*i64x2*/ 0x0, VALUE_TYPE_I64, + /*f32x4*/ 0x0, VALUE_TYPE_F32, + /*f64x2*/ 0x0, VALUE_TYPE_F64, + }; + uint8 push_type[] = { + /*i8x16*/ VALUE_TYPE_I32, VALUE_TYPE_I32, + VALUE_TYPE_V128, + /*i16x8*/ VALUE_TYPE_I32, VALUE_TYPE_I32, + VALUE_TYPE_V128, + /*i32x4*/ VALUE_TYPE_I32, VALUE_TYPE_V128, + /*i64x2*/ VALUE_TYPE_I64, VALUE_TYPE_V128, + /*f32x4*/ VALUE_TYPE_F32, VALUE_TYPE_V128, + /*f64x2*/ VALUE_TYPE_F64, VALUE_TYPE_V128, + }; + /* clang-format on */ + + CHECK_BUF(p, p_end, 1); + lane = read_uint8(p); + if (!check_simd_access_lane(opcode1, lane, error_buf, + error_buf_size)) { + goto fail; + } +#if WASM_ENABLE_FAST_INTERP != 0 + emit_byte(loader_ctx, lane); +#endif + if (replace[opcode1 - SIMD_i8x16_extract_lane_s]) { +#if WASM_ENABLE_FAST_INTERP != 0 + if (!(wasm_loader_pop_frame_ref_offset( + loader_ctx, + replace[opcode1 + - SIMD_i8x16_extract_lane_s], + error_buf, error_buf_size))) + goto fail; +#else + if (!(wasm_loader_pop_frame_ref( + loader_ctx, + replace[opcode1 + - SIMD_i8x16_extract_lane_s], + error_buf, error_buf_size))) + goto fail; +#endif /* end of WASM_ENABLE_FAST_INTERP != 0 */ + } + + POP_AND_PUSH( + VALUE_TYPE_V128, + push_type[opcode1 - SIMD_i8x16_extract_lane_s]); + break; + } + + /* i8x16 compare operation */ + case SIMD_i8x16_eq: + case SIMD_i8x16_ne: + case SIMD_i8x16_lt_s: + case SIMD_i8x16_lt_u: + case SIMD_i8x16_gt_s: + case SIMD_i8x16_gt_u: + case SIMD_i8x16_le_s: + case SIMD_i8x16_le_u: + case SIMD_i8x16_ge_s: + case SIMD_i8x16_ge_u: + /* i16x8 compare operation */ + case SIMD_i16x8_eq: + case SIMD_i16x8_ne: + case SIMD_i16x8_lt_s: + case SIMD_i16x8_lt_u: + case SIMD_i16x8_gt_s: + case SIMD_i16x8_gt_u: + case SIMD_i16x8_le_s: + case SIMD_i16x8_le_u: + case SIMD_i16x8_ge_s: + case SIMD_i16x8_ge_u: + /* i32x4 compare operation */ + case SIMD_i32x4_eq: + case SIMD_i32x4_ne: + case SIMD_i32x4_lt_s: + case SIMD_i32x4_lt_u: + case SIMD_i32x4_gt_s: + case SIMD_i32x4_gt_u: + case SIMD_i32x4_le_s: + case SIMD_i32x4_le_u: + case SIMD_i32x4_ge_s: + case SIMD_i32x4_ge_u: + /* f32x4 compare operation */ + case SIMD_f32x4_eq: + case SIMD_f32x4_ne: + case SIMD_f32x4_lt: + case SIMD_f32x4_gt: + case SIMD_f32x4_le: + case SIMD_f32x4_ge: + /* f64x2 compare operation */ + case SIMD_f64x2_eq: + case SIMD_f64x2_ne: + case SIMD_f64x2_lt: + case SIMD_f64x2_gt: + case SIMD_f64x2_le: + case SIMD_f64x2_ge: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + /* v128 operation */ + case SIMD_v128_not: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_v128_and: + case SIMD_v128_andnot: + case SIMD_v128_or: + case SIMD_v128_xor: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_v128_bitselect: + { + POP_V128(); + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_v128_any_true: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_I32); + break; + } + + /* Load Lane Operation */ + case SIMD_v128_load8_lane: + case SIMD_v128_load16_lane: + case SIMD_v128_load32_lane: + case SIMD_v128_load64_lane: + case SIMD_v128_store8_lane: + case SIMD_v128_store16_lane: + case SIMD_v128_store32_lane: + case SIMD_v128_store64_lane: + { + uint8 lane; + + CHECK_MEMORY(); + + pb_read_leb_uint32(p, p_end, align); /* align */ + if (!check_simd_memory_access_align( + opcode1, align, error_buf, error_buf_size)) { + goto fail; + } + + pb_read_leb_mem_offset(p, p_end, + mem_offset); /* offset */ + + CHECK_BUF(p, p_end, 1); + lane = read_uint8(p); + if (!check_simd_access_lane(opcode1, lane, error_buf, + error_buf_size)) { + goto fail; + } +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, mem_offset); +#endif + POP_V128(); + POP_MEM_OFFSET(); +#if WASM_ENABLE_FAST_INTERP != 0 + emit_byte(loader_ctx, lane); +#endif + if (opcode1 < SIMD_v128_store8_lane) { + PUSH_V128(); + } +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + } + + case SIMD_v128_load32_zero: + case SIMD_v128_load64_zero: + { + CHECK_MEMORY(); + + pb_read_leb_uint32(p, p_end, align); /* align */ + if (!check_simd_memory_access_align( + opcode1, align, error_buf, error_buf_size)) { + goto fail; + } + + pb_read_leb_mem_offset(p, p_end, + mem_offset); /* offset */ +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, mem_offset); +#endif + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_V128); +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + break; + } + + /* Float conversion */ + case SIMD_f32x4_demote_f64x2_zero: + case SIMD_f64x2_promote_low_f32x4_zero: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + /* i8x16 Operation */ + case SIMD_i8x16_abs: + case SIMD_i8x16_neg: + case SIMD_i8x16_popcnt: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i8x16_all_true: + case SIMD_i8x16_bitmask: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_I32); + break; + } + + case SIMD_i8x16_narrow_i16x8_s: + case SIMD_i8x16_narrow_i16x8_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_f32x4_ceil: + case SIMD_f32x4_floor: + case SIMD_f32x4_trunc: + case SIMD_f32x4_nearest: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i8x16_shl: + case SIMD_i8x16_shr_s: + case SIMD_i8x16_shr_u: + { + POP_I32(); + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i8x16_add: + case SIMD_i8x16_add_sat_s: + case SIMD_i8x16_add_sat_u: + case SIMD_i8x16_sub: + case SIMD_i8x16_sub_sat_s: + case SIMD_i8x16_sub_sat_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_f64x2_ceil: + case SIMD_f64x2_floor: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i8x16_min_s: + case SIMD_i8x16_min_u: + case SIMD_i8x16_max_s: + case SIMD_i8x16_max_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_f64x2_trunc: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i8x16_avgr_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i16x8_extadd_pairwise_i8x16_s: + case SIMD_i16x8_extadd_pairwise_i8x16_u: + case SIMD_i32x4_extadd_pairwise_i16x8_s: + case SIMD_i32x4_extadd_pairwise_i16x8_u: + /* i16x8 operation */ + case SIMD_i16x8_abs: + case SIMD_i16x8_neg: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i16x8_q15mulr_sat_s: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i16x8_all_true: + case SIMD_i16x8_bitmask: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_I32); + break; + } + + case SIMD_i16x8_narrow_i32x4_s: + case SIMD_i16x8_narrow_i32x4_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i16x8_extend_low_i8x16_s: + case SIMD_i16x8_extend_high_i8x16_s: + case SIMD_i16x8_extend_low_i8x16_u: + case SIMD_i16x8_extend_high_i8x16_u: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i16x8_shl: + case SIMD_i16x8_shr_s: + case SIMD_i16x8_shr_u: + { + POP_I32(); + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i16x8_add: + case SIMD_i16x8_add_sat_s: + case SIMD_i16x8_add_sat_u: + case SIMD_i16x8_sub: + case SIMD_i16x8_sub_sat_s: + case SIMD_i16x8_sub_sat_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_f64x2_nearest: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i16x8_mul: + case SIMD_i16x8_min_s: + case SIMD_i16x8_min_u: + case SIMD_i16x8_max_s: + case SIMD_i16x8_max_u: + case SIMD_i16x8_avgr_u: + case SIMD_i16x8_extmul_low_i8x16_s: + case SIMD_i16x8_extmul_high_i8x16_s: + case SIMD_i16x8_extmul_low_i8x16_u: + case SIMD_i16x8_extmul_high_i8x16_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + /* i32x4 operation */ + case SIMD_i32x4_abs: + case SIMD_i32x4_neg: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i32x4_all_true: + case SIMD_i32x4_bitmask: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_I32); + break; + } + + case SIMD_i32x4_extend_low_i16x8_s: + case SIMD_i32x4_extend_high_i16x8_s: + case SIMD_i32x4_extend_low_i16x8_u: + case SIMD_i32x4_extend_high_i16x8_u: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i32x4_shl: + case SIMD_i32x4_shr_s: + case SIMD_i32x4_shr_u: + { + POP_I32(); + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i32x4_add: + case SIMD_i32x4_sub: + case SIMD_i32x4_mul: + case SIMD_i32x4_min_s: + case SIMD_i32x4_min_u: + case SIMD_i32x4_max_s: + case SIMD_i32x4_max_u: + case SIMD_i32x4_dot_i16x8_s: + case SIMD_i32x4_extmul_low_i16x8_s: + case SIMD_i32x4_extmul_high_i16x8_s: + case SIMD_i32x4_extmul_low_i16x8_u: + case SIMD_i32x4_extmul_high_i16x8_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + /* i64x2 operation */ + case SIMD_i64x2_abs: + case SIMD_i64x2_neg: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i64x2_all_true: + case SIMD_i64x2_bitmask: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_I32); + break; + } + + case SIMD_i64x2_extend_low_i32x4_s: + case SIMD_i64x2_extend_high_i32x4_s: + case SIMD_i64x2_extend_low_i32x4_u: + case SIMD_i64x2_extend_high_i32x4_u: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i64x2_shl: + case SIMD_i64x2_shr_s: + case SIMD_i64x2_shr_u: + { + POP_I32(); + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i64x2_add: + case SIMD_i64x2_sub: + case SIMD_i64x2_mul: + case SIMD_i64x2_eq: + case SIMD_i64x2_ne: + case SIMD_i64x2_lt_s: + case SIMD_i64x2_gt_s: + case SIMD_i64x2_le_s: + case SIMD_i64x2_ge_s: + case SIMD_i64x2_extmul_low_i32x4_s: + case SIMD_i64x2_extmul_high_i32x4_s: + case SIMD_i64x2_extmul_low_i32x4_u: + case SIMD_i64x2_extmul_high_i32x4_u: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + /* f32x4 operation */ + case SIMD_f32x4_abs: + case SIMD_f32x4_neg: + case SIMD_f32x4_sqrt: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_f32x4_add: + case SIMD_f32x4_sub: + case SIMD_f32x4_mul: + case SIMD_f32x4_div: + case SIMD_f32x4_min: + case SIMD_f32x4_max: + case SIMD_f32x4_pmin: + case SIMD_f32x4_pmax: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + /* f64x2 operation */ + case SIMD_f64x2_abs: + case SIMD_f64x2_neg: + case SIMD_f64x2_sqrt: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_f64x2_add: + case SIMD_f64x2_sub: + case SIMD_f64x2_mul: + case SIMD_f64x2_div: + case SIMD_f64x2_min: + case SIMD_f64x2_max: + case SIMD_f64x2_pmin: + case SIMD_f64x2_pmax: + { + POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + case SIMD_i32x4_trunc_sat_f32x4_s: + case SIMD_i32x4_trunc_sat_f32x4_u: + case SIMD_f32x4_convert_i32x4_s: + case SIMD_f32x4_convert_i32x4_u: + case SIMD_i32x4_trunc_sat_f64x2_s_zero: + case SIMD_i32x4_trunc_sat_f64x2_u_zero: + case SIMD_f64x2_convert_low_i32x4_s: + case SIMD_f64x2_convert_low_i32x4_u: + { + POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); + break; + } + + default: + { + if (error_buf != NULL) { + snprintf(error_buf, error_buf_size, + "WASM module load failed: " + "invalid opcode 0xfd %02x.", + opcode1); + } + goto fail; + } + } + break; + } +#endif /* end of (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) || \ + (WASM_ENABLE_FAST_INTERP != 0) */ +#endif /* end of WASM_ENABLE_SIMD */ + +#if WASM_ENABLE_SHARED_MEMORY != 0 + case WASM_OP_ATOMIC_PREFIX: + { + uint32 opcode1; + + pb_read_leb_uint32(p, p_end, opcode1); + +#if WASM_ENABLE_FAST_INTERP != 0 + emit_byte(loader_ctx, opcode1); +#endif + if (opcode1 != WASM_OP_ATOMIC_FENCE) { + CHECK_MEMORY(); + pb_read_leb_uint32(p, p_end, align); /* align */ + pb_read_leb_mem_offset(p, p_end, mem_offset); /* offset */ + if (!check_memory_align_equal(opcode1, align, error_buf, + error_buf_size)) { + goto fail; + } +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, mem_offset); +#endif + } +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + func->has_memory_operations = true; +#endif + switch (opcode1) { + case WASM_OP_ATOMIC_NOTIFY: + POP_I32(); + POP_MEM_OFFSET(); + PUSH_I32(); + break; + case WASM_OP_ATOMIC_WAIT32: + POP_I64(); + POP_I32(); + POP_MEM_OFFSET(); + PUSH_I32(); + break; + case WASM_OP_ATOMIC_WAIT64: + POP_I64(); + POP_I64(); + POP_MEM_OFFSET(); + PUSH_I32(); + break; + case WASM_OP_ATOMIC_FENCE: + /* reserved byte 0x00 */ + if (*p++ != 0x00) { + set_error_buf(error_buf, error_buf_size, + "zero byte expected"); + goto fail; + } + break; + case WASM_OP_ATOMIC_I32_LOAD: + case WASM_OP_ATOMIC_I32_LOAD8_U: + case WASM_OP_ATOMIC_I32_LOAD16_U: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I32); + break; + case WASM_OP_ATOMIC_I32_STORE: + case WASM_OP_ATOMIC_I32_STORE8: + case WASM_OP_ATOMIC_I32_STORE16: + POP_I32(); + POP_MEM_OFFSET(); + break; + case WASM_OP_ATOMIC_I64_LOAD: + case WASM_OP_ATOMIC_I64_LOAD8_U: + case WASM_OP_ATOMIC_I64_LOAD16_U: + case WASM_OP_ATOMIC_I64_LOAD32_U: + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I64); + break; + case WASM_OP_ATOMIC_I64_STORE: + case WASM_OP_ATOMIC_I64_STORE8: + case WASM_OP_ATOMIC_I64_STORE16: + case WASM_OP_ATOMIC_I64_STORE32: + POP_I64(); + POP_MEM_OFFSET(); + break; + case WASM_OP_ATOMIC_RMW_I32_ADD: + case WASM_OP_ATOMIC_RMW_I32_ADD8_U: + case WASM_OP_ATOMIC_RMW_I32_ADD16_U: + case WASM_OP_ATOMIC_RMW_I32_SUB: + case WASM_OP_ATOMIC_RMW_I32_SUB8_U: + case WASM_OP_ATOMIC_RMW_I32_SUB16_U: + case WASM_OP_ATOMIC_RMW_I32_AND: + case WASM_OP_ATOMIC_RMW_I32_AND8_U: + case WASM_OP_ATOMIC_RMW_I32_AND16_U: + case WASM_OP_ATOMIC_RMW_I32_OR: + case WASM_OP_ATOMIC_RMW_I32_OR8_U: + case WASM_OP_ATOMIC_RMW_I32_OR16_U: + case WASM_OP_ATOMIC_RMW_I32_XOR: + case WASM_OP_ATOMIC_RMW_I32_XOR8_U: + case WASM_OP_ATOMIC_RMW_I32_XOR16_U: + case WASM_OP_ATOMIC_RMW_I32_XCHG: + case WASM_OP_ATOMIC_RMW_I32_XCHG8_U: + case WASM_OP_ATOMIC_RMW_I32_XCHG16_U: + POP_I32(); + POP_MEM_OFFSET(); + PUSH_I32(); + break; + case WASM_OP_ATOMIC_RMW_I64_ADD: + case WASM_OP_ATOMIC_RMW_I64_ADD8_U: + case WASM_OP_ATOMIC_RMW_I64_ADD16_U: + case WASM_OP_ATOMIC_RMW_I64_ADD32_U: + case WASM_OP_ATOMIC_RMW_I64_SUB: + case WASM_OP_ATOMIC_RMW_I64_SUB8_U: + case WASM_OP_ATOMIC_RMW_I64_SUB16_U: + case WASM_OP_ATOMIC_RMW_I64_SUB32_U: + case WASM_OP_ATOMIC_RMW_I64_AND: + case WASM_OP_ATOMIC_RMW_I64_AND8_U: + case WASM_OP_ATOMIC_RMW_I64_AND16_U: + case WASM_OP_ATOMIC_RMW_I64_AND32_U: + case WASM_OP_ATOMIC_RMW_I64_OR: + case WASM_OP_ATOMIC_RMW_I64_OR8_U: + case WASM_OP_ATOMIC_RMW_I64_OR16_U: + case WASM_OP_ATOMIC_RMW_I64_OR32_U: + case WASM_OP_ATOMIC_RMW_I64_XOR: + case WASM_OP_ATOMIC_RMW_I64_XOR8_U: + case WASM_OP_ATOMIC_RMW_I64_XOR16_U: + case WASM_OP_ATOMIC_RMW_I64_XOR32_U: + case WASM_OP_ATOMIC_RMW_I64_XCHG: + case WASM_OP_ATOMIC_RMW_I64_XCHG8_U: + case WASM_OP_ATOMIC_RMW_I64_XCHG16_U: + case WASM_OP_ATOMIC_RMW_I64_XCHG32_U: + POP_I64(); + POP_MEM_OFFSET(); + PUSH_I64(); + break; + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG: + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U: + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U: + POP_I32(); + POP_I32(); + POP_MEM_OFFSET(); + PUSH_I32(); + break; + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U: + POP_I64(); + POP_I64(); + POP_MEM_OFFSET(); + PUSH_I64(); + break; + default: + set_error_buf_v(error_buf, error_buf_size, + "%s %02x %02x", "unsupported opcode", + 0xfe, opcode1); + goto fail; + } + break; + } +#endif /* end of WASM_ENABLE_SHARED_MEMORY */ + + default: + set_error_buf_v(error_buf, error_buf_size, "%s %02x", + "unsupported opcode", opcode); + goto fail; + } + +#if WASM_ENABLE_FAST_INTERP != 0 + last_op = opcode; +#endif + } + + if (loader_ctx->csp_num > 0) { + /* unmatched end opcodes result from unbalanced control flow structures, + * for example, br_table with inconsistent target count (1 declared, 2 + * given), or simply superfluous end opcodes */ + set_error_buf( + error_buf, error_buf_size, + "unexpected end opcodes from unbalanced control flow structures"); + goto fail; + } + +#if WASM_ENABLE_FAST_INTERP != 0 + if (loader_ctx->p_code_compiled == NULL) + goto re_scan; + + func->const_cell_num = loader_ctx->i64_const_num * 2 + + loader_ctx->v128_const_num * 4 + + loader_ctx->i32_const_num; + if (func->const_cell_num > 0) { + if (!(func->consts = + loader_malloc((uint64)sizeof(uint32) * func->const_cell_num, + error_buf, error_buf_size))) + goto fail; + if (loader_ctx->i64_const_num > 0) { + bh_memcpy_s(func->consts, + (uint32)sizeof(int64) * loader_ctx->i64_const_num, + loader_ctx->i64_consts, + (uint32)sizeof(int64) * loader_ctx->i64_const_num); + } + if (loader_ctx->i32_const_num > 0) { + bh_memcpy_s(func->consts + + sizeof(int64) * loader_ctx->i64_const_num, + (uint32)sizeof(int32) * loader_ctx->i32_const_num, + loader_ctx->i32_consts, + (uint32)sizeof(int32) * loader_ctx->i32_const_num); + } + if (loader_ctx->v128_const_num > 0) { + bh_memcpy_s(func->consts, + (uint32)sizeof(V128) * loader_ctx->v128_const_num, + loader_ctx->v128_consts, + (uint32)sizeof(V128) * loader_ctx->v128_const_num); + } + } + + func->max_stack_cell_num = loader_ctx->preserved_local_offset + - loader_ctx->start_dynamic_offset + 1; +#else + func->max_stack_cell_num = loader_ctx->max_stack_cell_num; +#endif + func->max_block_num = loader_ctx->max_csp_num; + return_value = true; + +fail: + wasm_loader_ctx_destroy(loader_ctx); + + (void)table_idx; + (void)table_seg_idx; + (void)data_seg_idx; + (void)i64_const; + (void)local_offset; + (void)p_org; + (void)mem_offset; + (void)align; + return return_value; +} diff --git a/wamr/src/wasm_loader_common.c b/wamr/src/wasm_loader_common.c new file mode 100644 index 0000000..861c473 --- /dev/null +++ b/wamr/src/wasm_loader_common.c @@ -0,0 +1,254 @@ +/* + * Copyright (C) 2024 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#include "wasm_loader_common.h" // wasm_loader_common.h +#include "bh_leb128.h" // bh_leb128.h +#include "bh_log.h" // bh_log.h + + +void +wasm_loader_set_error_buf(char *error_buf, uint32 error_buf_size, + const char *string, bool is_aot) +{ + if (error_buf != NULL) { + snprintf(error_buf, error_buf_size, "%s module load failed: %s", + is_aot ? "AOT" : "WASM", string); + } +} + +#if WASM_ENABLE_MEMORY64 != 0 +bool +check_memory64_flags_consistency(WASMModule *module, char *error_buf, + uint32 error_buf_size, bool is_aot) +{ + uint32 i; + bool wasm64_flag, all_wasm64 = true, none_wasm64 = true; + + for (i = 0; i < module->import_memory_count; ++i) { + wasm64_flag = + module->import_memories[i].u.memory.mem_type.flags & MEMORY64_FLAG; + all_wasm64 &= wasm64_flag; + none_wasm64 &= !wasm64_flag; + } + + for (i = 0; i < module->memory_count; ++i) { + wasm64_flag = module->memories[i].flags & MEMORY64_FLAG; + all_wasm64 &= wasm64_flag; + none_wasm64 &= !wasm64_flag; + } + + if (!(all_wasm64 || none_wasm64)) { + wasm_loader_set_error_buf( + error_buf, error_buf_size, + "inconsistent limits wasm64 flags for memory sections", is_aot); + return false; + } + return true; +} +#endif + +bool +wasm_memory_check_flags(const uint8 mem_flag, char *error_buf, + uint32 error_buf_size, bool is_aot) +{ + /* Check whether certain features indicated by mem_flag are enabled in + * runtime */ + if (mem_flag > MAX_PAGE_COUNT_FLAG) { +#if WASM_ENABLE_SHARED_MEMORY == 0 + if (mem_flag & SHARED_MEMORY_FLAG) { + LOG_VERBOSE("shared memory flag was found, please enable shared " + "memory, lib-pthread or lib-wasi-threads"); + wasm_loader_set_error_buf(error_buf, error_buf_size, + "invalid limits flags", is_aot); + return false; + } +#endif +#if WASM_ENABLE_MEMORY64 == 0 + if (mem_flag & MEMORY64_FLAG) { + LOG_VERBOSE("memory64 flag was found, please enable memory64"); + wasm_loader_set_error_buf(error_buf, error_buf_size, + "invalid limits flags", is_aot); + return false; + } +#endif + } + + if (mem_flag > MAX_PAGE_COUNT_FLAG + SHARED_MEMORY_FLAG + MEMORY64_FLAG) { + wasm_loader_set_error_buf(error_buf, error_buf_size, + "invalid limits flags", is_aot); + return false; + } + else if ((mem_flag & SHARED_MEMORY_FLAG) + && !(mem_flag & MAX_PAGE_COUNT_FLAG)) { + wasm_loader_set_error_buf(error_buf, error_buf_size, + "shared memory must have maximum", is_aot); + return false; + } + + return true; +} + +bool +wasm_table_check_flags(const uint8 table_flag, char *error_buf, + uint32 error_buf_size, bool is_aot) +{ + /* Check whether certain features indicated by mem_flag are enabled in + * runtime */ + if (table_flag > MAX_TABLE_SIZE_FLAG) { + if (table_flag & SHARED_TABLE_FLAG) { + wasm_loader_set_error_buf(error_buf, error_buf_size, + "tables cannot be shared", is_aot); + } +#if WASM_ENABLE_MEMORY64 == 0 + if (table_flag & TABLE64_FLAG) { + wasm_loader_set_error_buf(error_buf, error_buf_size, + "invalid limits flags(table64 flag was " + "found, please enable memory64)", + is_aot); + return false; + } +#endif + } + + if (table_flag > MAX_TABLE_SIZE_FLAG + TABLE64_FLAG) { + wasm_loader_set_error_buf(error_buf, error_buf_size, + "invalid limits flags", is_aot); + return false; + } + + return true; +} + +/* + * compare with a bigger type set in `wasm_value_type_size_internal()`, + * this function will only cover global value type, function's param + * value type and function's result value type. + * + * please feel free to add more if there are more requirements + */ +bool +is_valid_value_type(uint8 type) +{ + if (/* I32/I64/F32/F64, 0x7C to 0x7F */ + (type >= VALUE_TYPE_F64 && type <= VALUE_TYPE_I32) +#if WASM_ENABLE_GC != 0 + /* reference types, 0x65 to 0x70 */ + || wasm_is_type_reftype(type) +#elif WASM_ENABLE_REF_TYPES != 0 + || (type == VALUE_TYPE_FUNCREF || type == VALUE_TYPE_EXTERNREF) +#endif +#if WASM_ENABLE_SIMD != 0 + || type == VALUE_TYPE_V128 /* 0x7B */ +#endif + ) + return true; + return false; +} + +bool +is_valid_value_type_for_interpreter(uint8 value_type) +{ +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) \ + && (WASM_ENABLE_SIMDE == 0) + /* + * Note: regardless of WASM_ENABLE_SIMD, our classic interpreters don't + * have SIMD implemented. + * + * WASM_ENABLE_SIMDE is used to control SIMD feaure in fast interpreter + */ + if (value_type == VALUE_TYPE_V128) + return false; +#endif + return is_valid_value_type(value_type); +} + +bool +is_valid_func_type(const WASMFuncType *func_type) +{ + unsigned i; + for (i = 0; + i < (unsigned)(func_type->param_count + func_type->result_count); + i++) { + if (!is_valid_value_type(func_type->types[i])) + return false; + } + + return true; +} + +bool +is_valid_packed_type(uint8 packed_type) +{ + return packed_type == PACKED_TYPE_I8 || packed_type == PACKED_TYPE_I16; +} + +bool +is_valid_field_type(uint8 field_type) +{ + if (is_valid_value_type(field_type) || is_valid_packed_type(field_type)) + return true; + return false; +} + +/* + * Indices are represented as a u32. + */ +bool +is_indices_overflow(uint32 import, uint32 other, char *error_buf, + uint32 error_buf_size) +{ + if (import > UINT32_MAX - other) { + snprintf(error_buf, error_buf_size, + "too many items in the index space(%" PRIu32 "+%" PRIu32 ").", + import, other); + return true; + } + + return false; +} + +bool +read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign, + uint64 *p_result, char *error_buf, uint32 error_buf_size) +{ + size_t offset = 0; + bh_leb_read_status_t status = + bh_leb_read(*p_buf, buf_end, maxbits, sign, p_result, &offset); + + switch (status) { + case BH_LEB_READ_SUCCESS: + *p_buf += offset; + return true; + case BH_LEB_READ_TOO_LONG: + wasm_loader_set_error_buf(error_buf, error_buf_size, + "integer representation too long", false); + return false; + case BH_LEB_READ_OVERFLOW: + wasm_loader_set_error_buf(error_buf, error_buf_size, + "integer too large", false); + return false; + case BH_LEB_READ_UNEXPECTED_END: + wasm_loader_set_error_buf(error_buf, error_buf_size, + "unexpected end", false); + return false; + default: + bh_assert(false); + return false; + } +} + +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 +void +destroy_init_expr_recursive(InitializerExpression *expr) +{ + if (expr == NULL) { + return; + } + if (is_expr_binary_op(expr->init_expr_type)) { + destroy_init_expr_recursive(expr->u.binary.l_expr); + destroy_init_expr_recursive(expr->u.binary.r_expr); + } + wasm_runtime_free(expr); +} +#endif /* end of WASM_ENABLE_EXTENDED_CONST_EXPR != 0 */ diff --git a/wamr/src/wasm_memory.c b/wamr/src/wasm_memory.c new file mode 100644 index 0000000..52ca7bb --- /dev/null +++ b/wamr/src/wasm_memory.c @@ -0,0 +1,2002 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_runtime_common.h" // wasm_runtime_common.h +#include "wasm_runtime.h" // ../interpreter/wasm_runtime.h + +#include "mem_alloc.h" // mem_alloc.h +#include "wasm_memory.h" // wasm_memory.h + + + + + +typedef enum Memory_Mode { + MEMORY_MODE_UNKNOWN = 0, + MEMORY_MODE_POOL, + MEMORY_MODE_ALLOCATOR, + MEMORY_MODE_SYSTEM_ALLOCATOR +} Memory_Mode; + +static Memory_Mode memory_mode = MEMORY_MODE_UNKNOWN; + +static mem_allocator_t pool_allocator = NULL; + +#if WASM_ENABLE_SHARED_HEAP != 0 +static WASMSharedHeap *shared_heap_list = NULL; +static korp_mutex shared_heap_list_lock; +#endif + +static enlarge_memory_error_callback_t enlarge_memory_error_cb; +static void *enlarge_memory_error_user_data; + +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 +static void *allocator_user_data = NULL; +#endif + +static void *(*malloc_func)( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + mem_alloc_usage_t usage, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + void *user_data, +#endif + unsigned int size) = NULL; + +static void *(*realloc_func)( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + mem_alloc_usage_t usage, bool full_size_mmaped, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + void *user_data, +#endif + void *ptr, unsigned int size) = NULL; + +static void (*free_func)( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + mem_alloc_usage_t usage, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + void *user_data, +#endif + void *ptr) = NULL; + +static unsigned int global_pool_size; + +static uint64 +align_as_and_cast(uint64 size, uint64 alignment) +{ + uint64 aligned_size = (size + alignment - 1) & ~(alignment - 1); + + return aligned_size; +} + +static bool +wasm_memory_init_with_pool(void *mem, unsigned int bytes) +{ + mem_allocator_t allocator = mem_allocator_create(mem, bytes); + + if (allocator) { + memory_mode = MEMORY_MODE_POOL; + pool_allocator = allocator; + global_pool_size = bytes; + return true; + } + LOG_ERROR("Init memory with pool (%p, %u) failed.\n", mem, bytes); + return false; +} + +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 +static bool +wasm_memory_init_with_allocator(void *_user_data, void *_malloc_func, + void *_realloc_func, void *_free_func) +{ + if (_malloc_func && _free_func && _malloc_func != _free_func) { + memory_mode = MEMORY_MODE_ALLOCATOR; + allocator_user_data = _user_data; + malloc_func = _malloc_func; + realloc_func = _realloc_func; + free_func = _free_func; + return true; + } + LOG_ERROR("Init memory with allocator (%p, %p, %p, %p) failed.\n", + _user_data, _malloc_func, _realloc_func, _free_func); + return false; +} +#else +static bool +wasm_memory_init_with_allocator(void *malloc_func_ptr, void *realloc_func_ptr, + void *free_func_ptr) +{ + if (malloc_func_ptr && free_func_ptr && malloc_func_ptr != free_func_ptr) { + memory_mode = MEMORY_MODE_ALLOCATOR; + malloc_func = malloc_func_ptr; + realloc_func = realloc_func_ptr; + free_func = free_func_ptr; + return true; + } + LOG_ERROR("Init memory with allocator (%p, %p, %p) failed.\n", + malloc_func_ptr, realloc_func_ptr, free_func_ptr); + return false; +} +#endif + +static inline bool +is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 + if (!module_inst) { + return true; + } + + return wasm_runtime_is_bounds_checks_enabled(module_inst); +#else + return true; +#endif +} + +#if WASM_ENABLE_SHARED_HEAP != 0 +static void * +wasm_mmap_linear_memory(uint64 map_size, uint64 commit_size); +static void +wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, + uint64 map_size); + +static void * +runtime_malloc(uint64 size) +{ + void *mem; + + if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) { + LOG_WARNING("Allocate memory failed"); + return NULL; + } + + memset(mem, 0, (uint32)size); + return mem; +} + +WASMSharedHeap * +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) +{ + uint64 heap_struct_size = sizeof(WASMSharedHeap), map_size; + uint32 size = init_args->size; + WASMSharedHeap *heap; + + if (size == 0) { + goto fail1; + } + + if (!(heap = runtime_malloc(heap_struct_size))) { + goto fail1; + } + + size = align_uint(size, os_getpagesize()); + if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { + LOG_WARNING("Invalid size of shared heap"); + goto fail2; + } + + heap->size = size; + heap->start_off_mem64 = UINT64_MAX - heap->size + 1; + heap->start_off_mem32 = UINT32_MAX - heap->size + 1; + heap->attached_count = 0; + + if (init_args->pre_allocated_addr != NULL) { + /* Create shared heap from a pre allocated buffer, its size need to + * align with system page */ + if (size != init_args->size) { + LOG_WARNING("Pre allocated size need to be aligned with system " + "page size to create shared heap"); + goto fail2; + } + + heap->heap_handle = NULL; + heap->base_addr = init_args->pre_allocated_addr; + LOG_VERBOSE("Create preallocated shared heap %p with size %u", + heap->base_addr, size); + } + else { + if (!(heap->heap_handle = + runtime_malloc(mem_allocator_get_heap_struct_size()))) { + goto fail2; + } + +#ifndef OS_ENABLE_HW_BOUND_CHECK + map_size = size; +#else + /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: + * ea = i + memarg.offset + * both i and memarg.offset are u32 in range 0 to 4G + * so the range of ea is 0 to 8G + */ + map_size = 8 * (uint64)BH_GB; +#endif + + if (!(heap->base_addr = wasm_mmap_linear_memory(map_size, size))) { + goto fail3; + } + if (!mem_allocator_create_with_struct_and_pool( + heap->heap_handle, heap_struct_size, heap->base_addr, size)) { + LOG_WARNING("init share heap failed"); + goto fail4; + } + LOG_VERBOSE("Create pool shared heap %p with size %u", heap->base_addr, + size); + } + + os_mutex_lock(&shared_heap_list_lock); + if (shared_heap_list == NULL) { + shared_heap_list = heap; + } + else { + heap->next = shared_heap_list; + shared_heap_list = heap; + } + os_mutex_unlock(&shared_heap_list_lock); + return heap; + +fail4: + wasm_munmap_linear_memory(heap->base_addr, size, map_size); +fail3: + wasm_runtime_free(heap->heap_handle); +fail2: + wasm_runtime_free(heap); +fail1: + return NULL; +} + +WASMSharedHeap * +wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body) +{ + WASMSharedHeap *cur; + bool heap_handle_exist = false; + + if (!head || !body) { + LOG_WARNING("Invalid shared heap to chain."); + return NULL; + } + heap_handle_exist = head->heap_handle != NULL; + + os_mutex_lock(&shared_heap_list_lock); + if (head->attached_count != 0 || body->attached_count != 0) { + LOG_WARNING("To create shared heap chain, all shared heap need to be " + "detached first."); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + for (cur = shared_heap_list; cur; cur = cur->next) { + if (cur->chain_next == body || cur->chain_next == head) { + LOG_WARNING( + "To create shared heap chain, both the 'head' and 'body' " + "shared heap can't already be the 'body' in another a chain"); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + if (cur == head && cur->chain_next) { + LOG_WARNING( + "To create shared heap chain, the 'head' shared heap can't " + "already be the 'head' in another a chain"); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + } + for (cur = body; cur; cur = cur->chain_next) { + if (cur->heap_handle && heap_handle_exist) { + LOG_WARNING( + "To create shared heap chain, only one of shared heap can " + "dynamically shared_heap_malloc and shared_heap_free, the rest " + "can only be pre-allocated shared heap"); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + if (cur->heap_handle) + heap_handle_exist = true; + } + + head->start_off_mem64 = body->start_off_mem64 - head->size; + head->start_off_mem32 = body->start_off_mem32 - head->size; + head->chain_next = body; + os_mutex_unlock(&shared_heap_list_lock); + return head; +} + +WASMSharedHeap * +wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain) +{ + WASMSharedHeap *cur, *tmp; + + if (!head || !head->chain_next) { + LOG_WARNING("Invalid shared heap chain to disconnect the head from."); + return NULL; + } + + os_mutex_lock(&shared_heap_list_lock); + if (head->attached_count != 0) { + LOG_WARNING("To disconnect the shared heap head from the shared heap " + "chain, the shared heap chain needs to be detached first."); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + + cur = head; + while (cur && cur->chain_next) { + cur->start_off_mem64 = UINT64_MAX - cur->size + 1; + cur->start_off_mem32 = UINT32_MAX - cur->size + 1; + tmp = cur; + cur = cur->chain_next; + tmp->chain_next = NULL; + if (!entire_chain) + break; + } + os_mutex_unlock(&shared_heap_list_lock); + return cur; +} + +static uint8 * +get_last_used_shared_heap_base_addr_adj(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; + return e->shared_heap_base_addr_adj; + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + return e->shared_heap_base_addr_adj; + } +#endif /* end of WASM_ENABLE_AOT != 0 */ + return 0; +} + +static uintptr_t +get_last_used_shared_heap_start_offset(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + return e->shared_heap_start_off.u64; +#else + return e->shared_heap_start_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + return e->shared_heap_start_off.u64; +#else + return e->shared_heap_start_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_AOT != 0 */ + return 0; +} + +static uintptr_t +get_last_used_shared_heap_end_offset(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + return e->shared_heap_end_off.u64; +#else + return e->shared_heap_end_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + return e->shared_heap_end_off.u64; +#else + return e->shared_heap_end_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_AOT != 0 */ + return 0; +} + +static void +update_last_used_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap, bool is_memory64) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + if (is_memory64) + e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; + else + e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; + e->shared_heap_end_off.u64 = + e->shared_heap_start_off.u64 - 1 + shared_heap->size; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u64; +#else + e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; + e->shared_heap_end_off.u32[0] = + e->shared_heap_start_off.u32[0] - 1 + shared_heap->size; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + if (is_memory64) + e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; + else + e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; + e->shared_heap_end_off.u64 = + e->shared_heap_start_off.u64 - 1 + shared_heap->size; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u64; +#else + e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; + e->shared_heap_end_off.u32[0] = + e->shared_heap_start_off.u32[0] - 1 + shared_heap->size; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_AOT != 0 */ +} + +bool +wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap) +{ + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module_inst); + uint64 linear_mem_size; + + if (!memory) + return false; + + linear_mem_size = memory->memory_data_size; + + /* check if linear memory and shared heap are overlapped */ + if ((memory->is_memory64 && linear_mem_size > shared_heap->start_off_mem64) + || (!memory->is_memory64 + && linear_mem_size > shared_heap->start_off_mem32)) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + return false; + } + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; + if (e->shared_heap) { + LOG_WARNING("A shared heap is already attached"); + return false; + } + e->shared_heap = shared_heap; + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + if (e->shared_heap) { + LOG_WARNING("A shared heap is already attached"); + return false; + } + e->shared_heap = shared_heap; + } +#endif /* end of WASM_ENABLE_AOT != 0 */ + update_last_used_shared_heap(module_inst, shared_heap, memory->is_memory64); + + os_mutex_lock(&shared_heap_list_lock); + shared_heap->attached_count++; + os_mutex_unlock(&shared_heap_list_lock); + LOG_VERBOSE("Shared heap %p is attached to module instance %p", shared_heap, + module_inst); + return true; +} + +bool +wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + return wasm_cluster_attach_shared_heap(module_inst, shared_heap); +#else + return wasm_runtime_attach_shared_heap_internal(module_inst, shared_heap); +#endif +} + +void +wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) +{ + /* Reset shared_heap_end_off = UINT64/32_MAX - 1 to handling a corner case, + app_offset >= shared_heap_start && app_offset <= shared_heap_end-bytes+1 + when bytes=1 and both e->shared_heap_start_off and e->shared_heap_end_off + is 0xffffffff */ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; + if (e->shared_heap != NULL) { + os_mutex_lock(&shared_heap_list_lock); + e->shared_heap->attached_count--; + os_mutex_unlock(&shared_heap_list_lock); + } + e->shared_heap = NULL; +#if UINTPTR_MAX == UINT64_MAX + e->shared_heap_start_off.u64 = UINT64_MAX; + e->shared_heap_end_off.u64 = UINT64_MAX - 1; +#else + e->shared_heap_start_off.u32[0] = UINT32_MAX; + e->shared_heap_end_off.u32[0] = UINT32_MAX - 1; +#endif + e->shared_heap_base_addr_adj = NULL; + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + if (e->shared_heap != NULL) { + os_mutex_lock(&shared_heap_list_lock); + e->shared_heap->attached_count--; + os_mutex_unlock(&shared_heap_list_lock); + } + e->shared_heap = NULL; +#if UINTPTR_MAX == UINT64_MAX + e->shared_heap_start_off.u64 = UINT64_MAX; + e->shared_heap_end_off.u64 = UINT64_MAX - 1; +#else + e->shared_heap_start_off.u32[0] = UINT32_MAX; + e->shared_heap_end_off.u32[0] = UINT32_MAX - 1; +#endif + e->shared_heap_base_addr_adj = NULL; + } +#endif /* end of WASM_ENABLE_AOT != 0 */ + LOG_VERBOSE("Shared heap is detached from module instance %p", module_inst); +} + +void +wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_detach_shared_heap(module_inst); +#else + wasm_runtime_detach_shared_heap_internal(module_inst); +#endif +} + +static WASMSharedHeap * +get_shared_heap(WASMModuleInstanceCommon *module_inst_comm) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + return ((WASMModuleInstance *)module_inst_comm)->e->shared_heap; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst_comm->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst_comm) + ->e; + return e->shared_heap; + } +#endif + return NULL; +} + +WASMSharedHeap * +wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm) +{ + return get_shared_heap(module_inst_comm); +} + +bool +is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, + bool is_memory64, uint64 app_offset, uint32 bytes) +{ + WASMSharedHeap *heap = get_shared_heap(module_inst), *cur; + uint64 shared_heap_start, shared_heap_end; + + if (!heap) { + goto fail; + } + + if (bytes == 0) { + bytes = 1; + } + + shared_heap_start = + (uint64)get_last_used_shared_heap_start_offset(module_inst); + shared_heap_end = (uint64)get_last_used_shared_heap_end_offset(module_inst); + if (bytes - 1 <= shared_heap_end && app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1) { + return true; + } + + /* Early stop for app start address not in the shared heap(chain) at all */ + shared_heap_start = + is_memory64 ? heap->start_off_mem64 : heap->start_off_mem32; + shared_heap_end = is_memory64 ? UINT64_MAX : UINT32_MAX; + if (bytes - 1 > shared_heap_end || app_offset < shared_heap_start + || app_offset > shared_heap_end - bytes + 1) { + goto fail; + } + + /* Find the exact shared heap that app addr is in, and update last used + * shared heap info in module inst extra */ + for (cur = heap; cur; cur = cur->chain_next) { + shared_heap_start = + is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; + shared_heap_end = shared_heap_start - 1 + cur->size; + if (bytes - 1 <= shared_heap_end && app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1) { + update_last_used_shared_heap(module_inst, cur, is_memory64); + return true; + } + } + +fail: + return false; +} + +static bool +is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, + bool is_memory64, uint8 *addr, uint32 bytes) +{ + WASMSharedHeap *cur, *heap = get_shared_heap(module_inst); + uintptr_t base_addr, addr_int, end_addr; + + if (!heap) { + goto fail; + } + + /* Iterate through shared heap chain to find whether native addr in one of + * shared heap */ + for (cur = heap; cur != NULL; cur = cur->chain_next) { + base_addr = (uintptr_t)cur->base_addr; + addr_int = (uintptr_t)addr; + if (addr_int < base_addr) + continue; + + end_addr = addr_int + bytes; + /* Check for overflow */ + if (end_addr <= addr_int) + continue; + + if (end_addr > base_addr + cur->size) + continue; + + update_last_used_shared_heap(module_inst, cur, is_memory64); + return true; + } + +fail: + return false; +} + +uint64 +wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, + uint64 size, void **p_native_addr) +{ + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module_inst); + WASMSharedHeap *shared_heap = get_shared_heap(module_inst); + void *native_addr = NULL; + + if (!memory || !shared_heap) + return 0; + + while (shared_heap && !shared_heap->heap_handle) { + shared_heap = shared_heap->chain_next; + } + if (!shared_heap) { + LOG_WARNING("Can't allocate from pre allocated shared heap"); + return 0; + } + + native_addr = mem_allocator_malloc(shared_heap->heap_handle, size); + if (!native_addr) + return 0; + + if (p_native_addr) { + *p_native_addr = native_addr; + } + + return memory->is_memory64 + ? shared_heap->start_off_mem64 + : shared_heap->start_off_mem32 + + ((uint8 *)native_addr - shared_heap->base_addr); +} + +void +wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, uint64 ptr) +{ + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module_inst); + WASMSharedHeap *shared_heap = get_shared_heap(module_inst); + uint8 *addr = NULL; + + if (!memory || !shared_heap) { + return; + } + + while (shared_heap && !shared_heap->heap_handle) { + shared_heap = shared_heap->chain_next; + } + if (!shared_heap) { + LOG_WARNING("The address to free is from pre allocated shared heap"); + return; + } + + if (memory->is_memory64) { + if (ptr < shared_heap->start_off_mem64) { /* ptr can not > UINT64_MAX */ + LOG_WARNING("The address to free isn't in shared heap"); + return; + } + addr = shared_heap->base_addr + (ptr - shared_heap->start_off_mem64); + } + else { + if (ptr < shared_heap->start_off_mem32 || ptr > UINT32_MAX) { + LOG_WARNING("The address to free isn't in shared heap"); + return; + } + addr = shared_heap->base_addr + (ptr - shared_heap->start_off_mem32); + } + + mem_allocator_free(shared_heap->heap_handle, addr); +} +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ + +bool +wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, + const MemAllocOption *alloc_option) +{ + bool ret = false; + +#if WASM_ENABLE_SHARED_HEAP != 0 + if (os_mutex_init(&shared_heap_list_lock)) { + return false; + } +#endif + + if (mem_alloc_type == Alloc_With_Pool) { + ret = wasm_memory_init_with_pool(alloc_option->pool.heap_buf, + alloc_option->pool.heap_size); + } + else if (mem_alloc_type == Alloc_With_Allocator) { + ret = wasm_memory_init_with_allocator( +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + alloc_option->allocator.user_data, +#endif + alloc_option->allocator.malloc_func, + alloc_option->allocator.realloc_func, + alloc_option->allocator.free_func); + } + else if (mem_alloc_type == Alloc_With_System_Allocator) { + memory_mode = MEMORY_MODE_SYSTEM_ALLOCATOR; + ret = true; + } + else { + ret = false; + } + +#if WASM_ENABLE_SHARED_HEAP != 0 + if (!ret) { + os_mutex_destroy(&shared_heap_list_lock); + } +#endif + + return ret; +} + +#if WASM_ENABLE_SHARED_HEAP != 0 +static void +destroy_shared_heaps() +{ + WASMSharedHeap *heap; + WASMSharedHeap *cur; + uint64 map_size; + + os_mutex_lock(&shared_heap_list_lock); + heap = shared_heap_list; + shared_heap_list = NULL; + os_mutex_unlock(&shared_heap_list_lock); + + while (heap) { + cur = heap; + heap = heap->next; + if (cur->heap_handle) { + mem_allocator_destroy(cur->heap_handle); + wasm_runtime_free(cur->heap_handle); +#ifndef OS_ENABLE_HW_BOUND_CHECK + map_size = cur->size; +#else + map_size = 8 * (uint64)BH_GB; +#endif + wasm_munmap_linear_memory(cur->base_addr, cur->size, map_size); + } + wasm_runtime_free(cur); + } + os_mutex_destroy(&shared_heap_list_lock); +} +#endif + +void +wasm_runtime_memory_destroy(void) +{ +#if WASM_ENABLE_SHARED_HEAP != 0 + destroy_shared_heaps(); +#endif + + if (memory_mode == MEMORY_MODE_POOL) { +#if BH_ENABLE_GC_VERIFY == 0 + (void)mem_allocator_destroy(pool_allocator); +#else + int ret = mem_allocator_destroy(pool_allocator); + if (ret != 0) { + /* Memory leak detected */ + exit(-1); + } +#endif + } + memory_mode = MEMORY_MODE_UNKNOWN; +} + +unsigned +wasm_runtime_memory_pool_size(void) +{ + if (memory_mode == MEMORY_MODE_POOL) + return global_pool_size; + else + return UINT32_MAX; +} + +static inline void * +wasm_runtime_malloc_internal(unsigned int size) +{ + if (memory_mode == MEMORY_MODE_UNKNOWN) { + LOG_WARNING( + "wasm_runtime_malloc failed: memory hasn't been initialized.\n"); + return NULL; + } + else if (memory_mode == MEMORY_MODE_POOL) { + return mem_allocator_malloc(pool_allocator, size); + } + else if (memory_mode == MEMORY_MODE_ALLOCATOR) { + return malloc_func( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + Alloc_For_Runtime, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + allocator_user_data, +#endif + size); + } + else { + return os_malloc(size); + } +} + +static inline void * +wasm_runtime_realloc_internal(void *ptr, unsigned int size) +{ + if (memory_mode == MEMORY_MODE_UNKNOWN) { + LOG_WARNING( + "wasm_runtime_realloc failed: memory hasn't been initialized.\n"); + return NULL; + } + else if (memory_mode == MEMORY_MODE_POOL) { + return mem_allocator_realloc(pool_allocator, ptr, size); + } + else if (memory_mode == MEMORY_MODE_ALLOCATOR) { + if (realloc_func) + return realloc_func( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + Alloc_For_Runtime, false, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + allocator_user_data, +#endif + ptr, size); + else + return NULL; + } + else { + return os_realloc(ptr, size); + } +} + +static inline void +wasm_runtime_free_internal(void *ptr) +{ + if (!ptr) { + LOG_WARNING("warning: wasm_runtime_free with NULL pointer\n"); +#if BH_ENABLE_GC_VERIFY != 0 + exit(-1); +#endif + return; + } + + if (memory_mode == MEMORY_MODE_UNKNOWN) { + LOG_WARNING("warning: wasm_runtime_free failed: " + "memory hasn't been initialize.\n"); + } + else if (memory_mode == MEMORY_MODE_POOL) { + mem_allocator_free(pool_allocator, ptr); + } + else if (memory_mode == MEMORY_MODE_ALLOCATOR) { + free_func( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + Alloc_For_Runtime, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + allocator_user_data, +#endif + ptr); + } + else { + os_free(ptr); + } +} + +void * +wasm_runtime_malloc(unsigned int size) +{ + if (size == 0) { + LOG_WARNING("warning: wasm_runtime_malloc with size zero\n"); + /* At lease alloc 1 byte to avoid malloc failed */ + size = 1; +#if BH_ENABLE_GC_VERIFY != 0 + exit(-1); +#endif + } + +#if WASM_ENABLE_FUZZ_TEST != 0 + if (size >= WASM_MEM_ALLOC_MAX_SIZE) { + LOG_WARNING("warning: wasm_runtime_malloc with too large size\n"); + return NULL; + } +#endif + + return wasm_runtime_malloc_internal(size); +} + +void * +wasm_runtime_realloc(void *ptr, unsigned int size) +{ + return wasm_runtime_realloc_internal(ptr, size); +} + +void +wasm_runtime_free(void *ptr) +{ + wasm_runtime_free_internal(ptr); +} + +bool +wasm_runtime_get_mem_alloc_info(mem_alloc_info_t *mem_alloc_info) +{ + if (memory_mode == MEMORY_MODE_POOL) { + return mem_allocator_get_alloc_info(pool_allocator, mem_alloc_info); + } + return false; +} + +bool +wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, + uint64 app_offset, uint64 size) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint64 max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + if (!is_bounds_checks_enabled(module_inst_comm)) { + return true; + } + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + goto fail; + } + +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, + app_offset, size)) { + return true; + } +#endif + +#if WASM_ENABLE_MEMORY64 != 0 + if (memory_inst->is_memory64) + max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; +#endif + /* boundary overflow check */ + if (size > max_linear_memory_size + || app_offset > max_linear_memory_size - size) { + goto fail; + } + + SHARED_MEMORY_LOCK(memory_inst); + + if (app_offset + size <= memory_inst->memory_data_size) { + SHARED_MEMORY_UNLOCK(memory_inst); + return true; + } + + SHARED_MEMORY_UNLOCK(memory_inst); + +fail: + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; +} + +bool +wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, + uint64 app_str_offset) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint64 app_end_offset, max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE; + char *str, *str_end; +#if WASM_ENABLE_SHARED_HEAP != 0 + uintptr_t shared_heap_end_off; + char *shared_heap_base_addr_adj; +#endif + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + if (!is_bounds_checks_enabled(module_inst_comm)) { + return true; + } + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + goto fail; + } + +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, + app_str_offset, 1)) { + shared_heap_end_off = + get_last_used_shared_heap_end_offset(module_inst_comm); + shared_heap_base_addr_adj = + (char *)get_last_used_shared_heap_base_addr_adj(module_inst_comm); + str = shared_heap_base_addr_adj + app_str_offset; + str_end = shared_heap_base_addr_adj + shared_heap_end_off + 1; + } + else +#endif + { + if (!wasm_runtime_get_app_addr_range(module_inst_comm, app_str_offset, + NULL, &app_end_offset)) + goto fail; + +#if WASM_ENABLE_MEMORY64 != 0 + if (memory_inst->is_memory64) + max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; +#endif + /* boundary overflow check, max start offset can be size - 1, while end + offset can be size */ + if (app_str_offset >= max_linear_memory_size + || app_end_offset > max_linear_memory_size) + goto fail; + + str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset); + str_end = str + (app_end_offset - app_str_offset); + } + + while (str < str_end && *str != '\0') + str++; + if (str == str_end) + goto fail; + + return true; +fail: + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; +} + +bool +wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, + void *native_ptr, uint64 size) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint8 *addr = (uint8 *)native_ptr; + uint64 max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + if (!is_bounds_checks_enabled(module_inst_comm)) { + return true; + } + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + goto fail; + } + +#if WASM_ENABLE_MEMORY64 != 0 + if (memory_inst->is_memory64) + max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; +#endif + /* boundary overflow check */ + if (size > max_linear_memory_size || (uintptr_t)addr > UINTPTR_MAX - size) { + goto fail; + } + +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_native_addr_in_shared_heap( + module_inst_comm, memory_inst->is_memory64, native_ptr, size)) { + return true; + } +#endif + + SHARED_MEMORY_LOCK(memory_inst); + + if (memory_inst->memory_data <= addr + && addr + size <= memory_inst->memory_data_end) { + SHARED_MEMORY_UNLOCK(memory_inst); + return true; + } + + SHARED_MEMORY_UNLOCK(memory_inst); + +fail: + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; +} + +void * +wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, + uint64 app_offset) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint8 *addr; + bool bounds_checks; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + bounds_checks = is_bounds_checks_enabled(module_inst_comm); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + return NULL; + } + +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, + app_offset, 1)) { + return get_last_used_shared_heap_base_addr_adj(module_inst_comm) + + app_offset; + } +#endif + + SHARED_MEMORY_LOCK(memory_inst); + + addr = memory_inst->memory_data + (uintptr_t)app_offset; + + if (bounds_checks) { + if (memory_inst->memory_data <= addr + && addr < memory_inst->memory_data_end) { + SHARED_MEMORY_UNLOCK(memory_inst); + return addr; + } + SHARED_MEMORY_UNLOCK(memory_inst); + return NULL; + } + + /* If bounds checks is disabled, return the address directly */ + SHARED_MEMORY_UNLOCK(memory_inst); + return addr; +} + +uint64 +wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, + void *native_ptr) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint8 *addr = (uint8 *)native_ptr; + bool bounds_checks; + uint64 ret; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + bounds_checks = is_bounds_checks_enabled(module_inst_comm); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + return 0; + } + +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_native_addr_in_shared_heap(module_inst_comm, + memory_inst->is_memory64, addr, 1)) { + return (uint64)(uintptr_t)(addr + - get_last_used_shared_heap_base_addr_adj( + module_inst_comm)); + } +#endif + + SHARED_MEMORY_LOCK(memory_inst); + + if (bounds_checks) { + if (memory_inst->memory_data <= addr + && addr < memory_inst->memory_data_end) { + ret = (uint64)(addr - memory_inst->memory_data); + SHARED_MEMORY_UNLOCK(memory_inst); + return ret; + } + } + /* If bounds checks is disabled, return the offset directly */ + else if (addr != NULL) { + ret = (uint64)(addr - memory_inst->memory_data); + SHARED_MEMORY_UNLOCK(memory_inst); + return ret; + } + + SHARED_MEMORY_UNLOCK(memory_inst); + return 0; +} + +bool +wasm_runtime_get_app_addr_range(WASMModuleInstanceCommon *module_inst_comm, + uint64 app_offset, uint64 *p_app_start_offset, + uint64 *p_app_end_offset) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint64 memory_data_size; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + return false; + } + + SHARED_MEMORY_LOCK(memory_inst); + + memory_data_size = memory_inst->memory_data_size; + + if (app_offset < memory_data_size) { + if (p_app_start_offset) + *p_app_start_offset = 0; + if (p_app_end_offset) + *p_app_end_offset = memory_data_size; + SHARED_MEMORY_UNLOCK(memory_inst); + return true; + } + + SHARED_MEMORY_UNLOCK(memory_inst); + return false; +} + +bool +wasm_runtime_get_native_addr_range(WASMModuleInstanceCommon *module_inst_comm, + uint8 *native_ptr, + uint8 **p_native_start_addr, + uint8 **p_native_end_addr) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; + uint8 *addr = (uint8 *)native_ptr; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { + return false; + } + + SHARED_MEMORY_LOCK(memory_inst); + + if (memory_inst->memory_data <= addr + && addr < memory_inst->memory_data_end) { + if (p_native_start_addr) + *p_native_start_addr = memory_inst->memory_data; + if (p_native_end_addr) + *p_native_end_addr = memory_inst->memory_data_end; + SHARED_MEMORY_UNLOCK(memory_inst); + return true; + } + + SHARED_MEMORY_UNLOCK(memory_inst); + return false; +} + +bool +wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, + uint64 app_buf_addr, uint64 app_buf_size, + void **p_native_addr) +{ + WASMMemoryInstance *memory_inst = wasm_get_default_memory(module_inst); + uint8 *native_addr; + bool bounds_checks; +#if WASM_ENABLE_SHARED_HEAP != 0 + uint8 *shared_heap_base_addr_adj = NULL; + uintptr_t shared_heap_end_off = 0; +#endif + + bh_assert(app_buf_addr <= UINTPTR_MAX && app_buf_size <= UINTPTR_MAX); + + if (!memory_inst) { + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; + } + +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst, + memory_inst->is_memory64, app_buf_addr, + app_buf_size)) { + const char *str, *str_end; + shared_heap_base_addr_adj = get_last_used_shared_heap_base_addr_adj( + (WASMModuleInstanceCommon *)module_inst); + shared_heap_end_off = get_last_used_shared_heap_end_offset( + (WASMModuleInstanceCommon *)module_inst); + native_addr = shared_heap_base_addr_adj + (uintptr_t)app_buf_addr; + + /* The whole string must be in the shared heap */ + str = (const char *)native_addr; + str_end = + (const char *)shared_heap_base_addr_adj + shared_heap_end_off + 1; + while (str < str_end && *str != '\0') + str++; + if (str == str_end) { + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; + } + else + goto success; + } +#endif + + native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr; + bounds_checks = + is_bounds_checks_enabled((WASMModuleInstanceCommon *)module_inst); + + if (!bounds_checks) { + if (app_buf_addr == 0) { + native_addr = NULL; + } + goto success; + } + + /* No need to check the app_offset and buf_size if memory access + boundary check with hardware trap is enabled */ +#ifndef OS_ENABLE_HW_BOUND_CHECK + SHARED_MEMORY_LOCK(memory_inst); + + if (app_buf_addr >= memory_inst->memory_data_size) { + goto fail; + } + + if (!is_str) { + if (app_buf_size > memory_inst->memory_data_size - app_buf_addr) { + goto fail; + } + } + else { + const char *str, *str_end; + + /* The whole string must be in the linear memory */ + str = (const char *)native_addr; + str_end = (const char *)memory_inst->memory_data_end; + while (str < str_end && *str != '\0') + str++; + if (str == str_end) + goto fail; + } + + SHARED_MEMORY_UNLOCK(memory_inst); +#endif + +success: + *p_native_addr = (void *)native_addr; + return true; + +#ifndef OS_ENABLE_HW_BOUND_CHECK +fail: + SHARED_MEMORY_UNLOCK(memory_inst); + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; +#endif +} + +WASMMemoryInstance * +wasm_get_default_memory(WASMModuleInstance *module_inst) +{ + if (module_inst->memories) + return module_inst->memories[0]; + else + return NULL; +} + +WASMMemoryInstance * +wasm_get_memory_with_idx(WASMModuleInstance *module_inst, uint32 index) +{ + if ((index >= module_inst->memory_count) || !module_inst->memories) + return NULL; + return module_inst->memories[index]; +} + +void +wasm_runtime_set_mem_bound_check_bytes(WASMMemoryInstance *memory, + uint64 memory_data_size) +{ +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 || WASM_ENABLE_AOT != 0 +#if UINTPTR_MAX == UINT64_MAX + memory->mem_bound_check_1byte.u64 = memory_data_size - 1; + memory->mem_bound_check_2bytes.u64 = memory_data_size - 2; + memory->mem_bound_check_4bytes.u64 = memory_data_size - 4; + memory->mem_bound_check_8bytes.u64 = memory_data_size - 8; + memory->mem_bound_check_16bytes.u64 = memory_data_size - 16; +#else + memory->mem_bound_check_1byte.u32[0] = (uint32)memory_data_size - 1; + memory->mem_bound_check_2bytes.u32[0] = (uint32)memory_data_size - 2; + memory->mem_bound_check_4bytes.u32[0] = (uint32)memory_data_size - 4; + memory->mem_bound_check_8bytes.u32[0] = (uint32)memory_data_size - 8; + memory->mem_bound_check_16bytes.u32[0] = (uint32)memory_data_size - 16; +#endif +#endif +} + +static void +wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, uint64 map_size) +{ +#ifdef BH_PLATFORM_WINDOWS + os_mem_decommit(mapped_mem, commit_size); +#else + (void)commit_size; +#endif + os_munmap(mapped_mem, map_size); +} + +static void * +wasm_mremap_linear_memory(void *mapped_mem, uint64 old_size, uint64 new_size, + uint64 commit_size) +{ + void *new_mem; + + bh_assert(new_size > 0); + bh_assert(new_size > old_size); + +#if UINTPTR_MAX == UINT32_MAX + if (new_size == 4 * (uint64)BH_GB) { + LOG_WARNING("On 32 bit platform, linear memory can't reach maximum " + "size of 4GB\n"); + return NULL; + } +#endif + + if (mapped_mem) { + new_mem = os_mremap(mapped_mem, old_size, new_size); + } + else { + new_mem = os_mmap(NULL, new_size, MMAP_PROT_NONE, MMAP_MAP_NONE, + os_get_invalid_handle()); + } + if (!new_mem) { + return NULL; + } + +#ifdef BH_PLATFORM_WINDOWS + if (commit_size > 0 + && !os_mem_commit(new_mem, commit_size, + MMAP_PROT_READ | MMAP_PROT_WRITE)) { + os_munmap(new_mem, new_size); + return NULL; + } +#endif + + if (os_mprotect(new_mem, commit_size, MMAP_PROT_READ | MMAP_PROT_WRITE) + != 0) { + wasm_munmap_linear_memory(new_mem, new_size, new_size); + return NULL; + } + + return new_mem; +} + +static void * +wasm_mmap_linear_memory(uint64 map_size, uint64 commit_size) +{ + return wasm_mremap_linear_memory(NULL, 0, map_size, commit_size); +} + +static bool +wasm_enlarge_memory_internal(WASMModuleInstanceCommon *module, + WASMMemoryInstance *memory, uint32 inc_page_count) +{ +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; +#endif + uint8 *memory_data_old, *memory_data_new, *heap_data_old; + uint32 num_bytes_per_page, heap_size; + uint32 cur_page_count, max_page_count, total_page_count; + uint64 total_size_old = 0, total_size_new; + bool ret = true, full_size_mmaped; + enlarge_memory_error_reason_t failure_reason = INTERNAL_ERROR; + + if (!memory) { + ret = false; + goto return_func; + } + +#ifdef OS_ENABLE_HW_BOUND_CHECK + full_size_mmaped = true; +#elif WASM_ENABLE_SHARED_MEMORY != 0 + full_size_mmaped = shared_memory_is_shared(memory); +#else + full_size_mmaped = false; +#endif + + memory_data_old = memory->memory_data; + total_size_old = memory->memory_data_size; + + heap_data_old = memory->heap_data; + heap_size = (uint32)(memory->heap_data_end - memory->heap_data); + + num_bytes_per_page = memory->num_bytes_per_page; + cur_page_count = memory->cur_page_count; + max_page_count = memory->max_page_count; + total_page_count = inc_page_count + cur_page_count; + total_size_new = num_bytes_per_page * (uint64)total_page_count; + + if (inc_page_count <= 0) + /* No need to enlarge memory */ + return true; + + if (total_page_count < cur_page_count) { /* integer overflow */ + ret = false; + goto return_func; + } + + if (total_page_count > max_page_count) { + failure_reason = MAX_SIZE_REACHED; + ret = false; + goto return_func; + } + +#if WASM_ENABLE_SHARED_HEAP != 0 + shared_heap = get_shared_heap(module); + if (shared_heap) { + if (memory->is_memory64 + && total_size_new > shared_heap->start_off_mem64) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + ret = false; + goto return_func; + } + else if (!memory->is_memory64 + && total_size_new > shared_heap->start_off_mem32) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + ret = false; + goto return_func; + } + } +#endif + + bh_assert(total_size_new + <= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)); + +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + if (!(memory_data_new = + realloc_func(Alloc_For_LinearMemory, full_size_mmaped, +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + allocator_user_data, +#endif + memory_data_old, total_size_new))) { + ret = false; + goto return_func; + } + if (heap_size > 0) { + if (mem_allocator_migrate(memory->heap_handle, + (char *)heap_data_old + + (memory_data_new - memory_data_old), + heap_size) + != 0) { + ret = false; + } + } + memory->heap_data = memory_data_new + (heap_data_old - memory_data_old); + memory->heap_data_end = memory->heap_data + heap_size; + memory->memory_data = memory_data_new; +#else + if (full_size_mmaped) { +#ifdef BH_PLATFORM_WINDOWS + if (!os_mem_commit(memory->memory_data_end, + total_size_new - total_size_old, + MMAP_PROT_READ | MMAP_PROT_WRITE)) { + ret = false; + goto return_func; + } +#endif + + if (os_mprotect(memory->memory_data_end, + total_size_new - total_size_old, + MMAP_PROT_READ | MMAP_PROT_WRITE) + != 0) { +#ifdef BH_PLATFORM_WINDOWS + os_mem_decommit(memory->memory_data_end, + total_size_new - total_size_old); +#endif + ret = false; + goto return_func; + } + } + else { + if (heap_size > 0) { + if (mem_allocator_is_heap_corrupted(memory->heap_handle)) { + wasm_runtime_show_app_heap_corrupted_prompt(); + ret = false; + goto return_func; + } + } + + if (!(memory_data_new = + wasm_mremap_linear_memory(memory_data_old, total_size_old, + total_size_new, total_size_new))) { + ret = false; + goto return_func; + } + + if (heap_size > 0) { + if (mem_allocator_migrate(memory->heap_handle, + (char *)heap_data_old + + (memory_data_new - memory_data_old), + heap_size) + != 0) { + /* Don't return here as memory->memory_data is obsolete and + must be updated to be correctly used later. */ + ret = false; + } + } + + memory->heap_data = memory_data_new + (heap_data_old - memory_data_old); + memory->heap_data_end = memory->heap_data + heap_size; + memory->memory_data = memory_data_new; +#if defined(os_writegsbase) + /* write base addr of linear memory to GS segment register */ + os_writegsbase(memory_data_new); +#endif + } +#endif /* end of WASM_MEM_ALLOC_WITH_USAGE */ + + /* + * AOT compiler assumes at least 8 byte alignment. + * see aot_check_memory_overflow. + */ + bh_assert(((uintptr_t)memory->memory_data & 0x7) == 0); + + memory->num_bytes_per_page = num_bytes_per_page; + memory->cur_page_count = total_page_count; + memory->max_page_count = max_page_count; + SET_LINEAR_MEMORY_SIZE(memory, total_size_new); + memory->memory_data_end = memory->memory_data + total_size_new; + + wasm_runtime_set_mem_bound_check_bytes(memory, total_size_new); + +return_func: + if (!ret && module && enlarge_memory_error_cb) { + WASMExecEnv *exec_env = NULL; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) + exec_env = ((WASMModuleInstance *)module)->cur_exec_env; +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) + exec_env = ((AOTModuleInstance *)module)->cur_exec_env; +#endif + + enlarge_memory_error_cb(inc_page_count, total_size_old, 0, + failure_reason, module, exec_env, + enlarge_memory_error_user_data); + } + + return ret; +} + +bool +wasm_runtime_enlarge_memory(WASMModuleInstanceCommon *module_inst, + uint64 inc_page_count) +{ + if (inc_page_count > UINT32_MAX) { + return false; + } + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + return aot_enlarge_memory((AOTModuleInstance *)module_inst, + (uint32)inc_page_count); + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + return wasm_enlarge_memory((WASMModuleInstance *)module_inst, + (uint32)inc_page_count); + } +#endif + + return false; +} + +void +wasm_runtime_set_enlarge_mem_error_callback( + const enlarge_memory_error_callback_t callback, void *user_data) +{ + enlarge_memory_error_cb = callback; + enlarge_memory_error_user_data = user_data; +} + +bool +wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) +{ + bool ret = false; + + if (module->memory_count > 0) { +#if WASM_ENABLE_SHARED_MEMORY != 0 + shared_memory_lock(module->memories[0]); +#endif + ret = wasm_enlarge_memory_internal((WASMModuleInstanceCommon *)module, + module->memories[0], inc_page_count); +#if WASM_ENABLE_SHARED_MEMORY != 0 + shared_memory_unlock(module->memories[0]); +#endif + } + + return ret; +} + +bool +wasm_enlarge_memory_with_idx(WASMModuleInstance *module, uint32 inc_page_count, + uint32 memidx) +{ + bool ret = false; + + if (memidx < module->memory_count) { +#if WASM_ENABLE_SHARED_MEMORY != 0 + shared_memory_lock(module->memories[memidx]); +#endif + ret = wasm_enlarge_memory_internal((WASMModuleInstanceCommon *)module, + module->memories[memidx], + inc_page_count); +#if WASM_ENABLE_SHARED_MEMORY != 0 + shared_memory_unlock(module->memories[memidx]); +#endif + } + + return ret; +} + +WASMMemoryInstance * +wasm_runtime_lookup_memory(WASMModuleInstanceCommon *module_inst, + const char *name) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return wasm_lookup_memory((WASMModuleInstance *)module_inst, name); +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return aot_lookup_memory((WASMModuleInstance *)module_inst, name); +#endif + + return NULL; +} + +WASMMemoryInstance * +wasm_runtime_get_default_memory(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return wasm_get_default_memory((WASMModuleInstance *)module_inst); +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return aot_get_default_memory((AOTModuleInstance *)module_inst); +#endif + + return NULL; +} + +WASMMemoryInstance * +wasm_runtime_get_memory(WASMModuleInstanceCommon *module_inst, uint32 index) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return wasm_get_memory_with_idx((WASMModuleInstance *)module_inst, + index); +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return aot_get_memory_with_idx((AOTModuleInstance *)module_inst, index); +#endif + + return NULL; +} + +uint64 +wasm_memory_get_cur_page_count(WASMMemoryInstance *memory) +{ + return memory->cur_page_count; +} + +uint64 +wasm_memory_get_max_page_count(WASMMemoryInstance *memory) +{ + return memory->max_page_count; +} + +uint64 +wasm_memory_get_bytes_per_page(WASMMemoryInstance *memory) +{ + return memory->num_bytes_per_page; +} + +bool +wasm_memory_get_shared(WASMMemoryInstance *memory) +{ + return memory->is_shared_memory; +} + +void * +wasm_memory_get_base_address(WASMMemoryInstance *memory) +{ + return memory->memory_data; +} + +bool +wasm_memory_enlarge(WASMMemoryInstance *memory, uint64 inc_page_count) +{ + bool ret = false; + + if (memory) { +#if WASM_ENABLE_SHARED_MEMORY != 0 + shared_memory_lock(memory); +#endif + ret = + wasm_enlarge_memory_internal(NULL, memory, (uint32)inc_page_count); +#if WASM_ENABLE_SHARED_MEMORY != 0 + shared_memory_unlock(memory); +#endif + } + + return ret; +} + +void +wasm_deallocate_linear_memory(WASMMemoryInstance *memory_inst) +{ + uint64 map_size; + + bh_assert(memory_inst); + bh_assert(memory_inst->memory_data); + +#ifndef OS_ENABLE_HW_BOUND_CHECK +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (shared_memory_is_shared(memory_inst)) { + map_size = (uint64)memory_inst->num_bytes_per_page + * memory_inst->max_page_count; + } + else +#endif + { + map_size = (uint64)memory_inst->num_bytes_per_page + * memory_inst->cur_page_count; + } +#else + map_size = 8 * (uint64)BH_GB; +#endif + +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + (void)map_size; + free_func(Alloc_For_LinearMemory, +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + allocator_user_data, +#endif + memory_inst->memory_data); +#else + wasm_munmap_linear_memory(memory_inst->memory_data, + memory_inst->memory_data_size, map_size); +#endif + + memory_inst->memory_data = NULL; +} + +int +wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory, + bool is_memory64, uint64 num_bytes_per_page, + uint64 init_page_count, uint64 max_page_count, + uint64 *memory_data_size) +{ + uint64 map_size, page_size; + + bh_assert(data); + bh_assert(memory_data_size); + +#ifndef OS_ENABLE_HW_BOUND_CHECK +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (is_shared_memory) { + /* Allocate maximum memory size when memory is shared */ + map_size = max_page_count * num_bytes_per_page; + } + else +#endif + { + map_size = init_page_count * num_bytes_per_page; + } +#else /* else of OS_ENABLE_HW_BOUND_CHECK */ + /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: + * ea = i + memarg.offset + * both i and memarg.offset are u32 in range 0 to 4G + * so the range of ea is 0 to 8G + */ + map_size = 8 * (uint64)BH_GB; +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ + + page_size = os_getpagesize(); + *memory_data_size = init_page_count * num_bytes_per_page; + + bh_assert(*memory_data_size <= GET_MAX_LINEAR_MEMORY_SIZE(is_memory64)); + *memory_data_size = align_as_and_cast(*memory_data_size, page_size); + + if (map_size > 0) { +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + (void)wasm_mmap_linear_memory; + if (!(*data = malloc_func(Alloc_For_LinearMemory, +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + allocator_user_data, +#endif + *memory_data_size))) { + return BHT_ERROR; + } +#else + if (!(*data = wasm_mmap_linear_memory(map_size, *memory_data_size))) { + return BHT_ERROR; + } +#endif + } + + /* + * AOT compiler assumes at least 8 byte alignment. + * see aot_check_memory_overflow. + */ + bh_assert(((uintptr_t)*data & 0x7) == 0); + + return BHT_OK; +} diff --git a/wamr/src/wasm_runtime.c b/wamr/src/wasm_runtime.c new file mode 100644 index 0000000..2967976 --- /dev/null +++ b/wamr/src/wasm_runtime.c @@ -0,0 +1,5140 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_runtime.h" // wasm_runtime.h +#include "wasm.h" // wasm.h +#include "wasm_loader.h" // wasm_loader.h +#include "wasm_interp.h" // wasm_interp.h +#include "bh_common.h" // bh_common.h +#include "bh_log.h" // bh_log.h +#include "mem_alloc.h" // mem_alloc.h +#include "wasm_runtime_common.h" // ../common/wasm_runtime_common.h +#include "wasm_memory.h" // ../common/wasm_memory.h + + + +#if WASM_ENABLE_DEBUG_INTERP != 0 +#include "debug_engine.h" // ../libraries/debug-engine/debug_engine.h +#endif +#if WASM_ENABLE_FAST_JIT != 0 + +#endif + + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) { + snprintf(error_buf, error_buf_size, + "WASM module instantiate failed: %s", string); + } +} + +static void +set_error_buf_v(char *error_buf, uint32 error_buf_size, const char *format, ...) +{ + va_list args; + char buf[128]; + + if (error_buf != NULL) { + va_start(args, format); + vsnprintf(buf, sizeof(buf), format, args); + va_end(args); + snprintf(error_buf, error_buf_size, + "WASM module instantiate failed: %s", buf); + } +} + +WASMModule * +wasm_load(uint8 *buf, uint32 size, +#if WASM_ENABLE_MULTI_MODULE != 0 + bool main_module, +#endif + const LoadArgs *name, char *error_buf, uint32 error_buf_size) +{ + return wasm_loader_load(buf, size, +#if WASM_ENABLE_MULTI_MODULE != 0 + main_module, +#endif + name, error_buf, error_buf_size); +} + +WASMModule * +wasm_load_from_sections(WASMSection *section_list, char *error_buf, + uint32 error_buf_size) +{ + return wasm_loader_load_from_sections(section_list, error_buf, + error_buf_size); +} + +void +wasm_unload(WASMModule *module) +{ + wasm_loader_unload(module); +} + +bool +wasm_resolve_symbols(WASMModule *module) +{ + bool ret = true; + uint32 idx; + for (idx = 0; idx < module->import_function_count; ++idx) { + WASMFunctionImport *import = &module->import_functions[idx].u.function; + bool linked = import->func_ptr_linked; +#if WASM_ENABLE_MULTI_MODULE != 0 + if (import->import_func_linked) { + linked = true; + } +#endif + if (!linked && !wasm_resolve_import_func(module, import)) { + ret = false; + } + } + return ret; +} + +#if WASM_ENABLE_MULTI_MODULE != 0 +static WASMFunction * +wasm_resolve_function(const char *module_name, const char *function_name, + const WASMFuncType *expected_function_type, + char *error_buf, uint32 error_buf_size) +{ + WASMModuleCommon *module_reg; + WASMFunction *function = NULL; + WASMExport *export = NULL; + WASMModule *module = NULL; + WASMFuncType *target_function_type = NULL; + + module_reg = wasm_runtime_find_module_registered(module_name); + if (!module_reg || module_reg->module_type != Wasm_Module_Bytecode) { + LOG_DEBUG("can not find a module named %s for function %s", module_name, + function_name); + set_error_buf(error_buf, error_buf_size, "unknown import"); + return NULL; + } + + module = (WASMModule *)module_reg; + export = loader_find_export((WASMModuleCommon *)module, module_name, + function_name, EXPORT_KIND_FUNC, error_buf, + error_buf_size); + if (!export) { + return NULL; + } + + /* resolve function type and function */ + if (export->index < module->import_function_count) { + target_function_type = + module->import_functions[export->index].u.function.func_type; + function = module->import_functions[export->index] + .u.function.import_func_linked; + } + else { + target_function_type = + module->functions[export->index - module->import_function_count] + ->func_type; + function = + module->functions[export->index - module->import_function_count]; + } + + /* check function type */ + if (!wasm_type_equal((WASMType *)expected_function_type, + (WASMType *)target_function_type, module->types, + module->type_count)) { + LOG_DEBUG("%s.%s failed the type check", module_name, function_name); + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return NULL; + } + + return function; +} +#endif + +bool +wasm_resolve_import_func(const WASMModule *module, WASMFunctionImport *function) +{ +#if WASM_ENABLE_MULTI_MODULE != 0 + char error_buf[128]; + WASMModule *sub_module = NULL; +#endif + +#if WASM_ENABLE_INVOKE_NATIVE != 0 + function->func_ptr_linked = wasm_native_resolve_symbol( + function->module_name, function->field_name, function->func_type, + &function->signature, &function->attachment, &function->call_conv_raw); + + if (function->func_ptr_linked) { + return true; + } +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + if (!wasm_runtime_is_built_in_module(function->module_name)) { + sub_module = (WASMModule *)wasm_runtime_load_depended_module( + (WASMModuleCommon *)module, function->module_name, error_buf, + sizeof(error_buf)); + if (!sub_module) { + LOG_WARNING("failed to load sub module: %s", error_buf); + return false; + } + } + function->import_func_linked = wasm_resolve_function( + function->module_name, function->field_name, function->func_type, + error_buf, sizeof(error_buf)); + + if (function->import_func_linked) { + function->import_module = sub_module; + return true; + } + else { + LOG_WARNING("failed to link function (%s, %s): %s", + function->module_name, function->field_name, error_buf); + } +#endif + + return false; +} + +static void * +runtime_malloc(uint64 size, char *error_buf, uint32 error_buf_size) +{ + void *mem; + + if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + return NULL; + } + + memset(mem, 0, (uint32)size); + return mem; +} + +#if WASM_ENABLE_MULTI_MODULE != 0 +static WASMModuleInstance * +get_sub_module_inst(const WASMModuleInstance *parent_module_inst, + const WASMModule *sub_module) +{ + bh_list *sub_module_inst_list = parent_module_inst->e->sub_module_inst_list; + WASMSubModInstNode *node = bh_list_first_elem(sub_module_inst_list); + + while (node && sub_module != node->module_inst->module) { + node = bh_list_elem_next(node); + } + return node ? node->module_inst : NULL; +} +#endif + +/** + * Destroy memory instances. + */ +static void +memories_deinstantiate(WASMModuleInstance *module_inst, + WASMMemoryInstance **memories, uint32 count) +{ + uint32 i; + if (memories) { + for (i = 0; i < count; i++) { + if (memories[i]) { +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMModule *module = module_inst->module; + if (i < module->import_memory_count + && module->import_memories[i].u.memory.import_module) { + continue; + } +#endif +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (shared_memory_is_shared(memories[i])) { + uint32 ref_count = shared_memory_dec_reference(memories[i]); + /* if the reference count is not zero, + don't free the memory */ + if (ref_count > 0) + continue; + } +#endif + if (memories[i]->heap_handle) { + mem_allocator_destroy(memories[i]->heap_handle); + wasm_runtime_free(memories[i]->heap_handle); + memories[i]->heap_handle = NULL; + } + if (memories[i]->memory_data) { + wasm_deallocate_linear_memory(memories[i]); + } + } + } + wasm_runtime_free(memories); + } + (void)module_inst; +} + +static WASMMemoryInstance * +memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, + WASMMemoryInstance *memory, uint32 memory_idx, + uint32 num_bytes_per_page, uint32 init_page_count, + uint32 max_page_count, uint32 heap_size, uint32 flags, + char *error_buf, uint32 error_buf_size) +{ + WASMModule *module = module_inst->module; + uint32 inc_page_count, global_idx, default_max_page; + uint32 bytes_of_last_page, bytes_to_page_end; + uint64 aux_heap_base, + heap_offset = (uint64)num_bytes_per_page * init_page_count; + uint64 memory_data_size, max_memory_data_size; + uint8 *global_addr; + + bool is_shared_memory = false; +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared_memory = flags & SHARED_MEMORY_FLAG ? true : false; + + /* shared memory */ + if (is_shared_memory && parent != NULL) { + bh_assert(parent->memory_count > memory_idx); + memory = parent->memories[memory_idx]; + shared_memory_inc_reference(memory); + return memory; + } +#else + (void)parent; + (void)memory_idx; + (void)flags; +#endif /* end of WASM_ENABLE_SHARED_MEMORY */ + +#if WASM_ENABLE_MEMORY64 != 0 + if (flags & MEMORY64_FLAG) { + memory->is_memory64 = 1; + } +#endif + default_max_page = + memory->is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; + + /* The app heap should be in the default memory */ + if (memory_idx == 0) { + if (heap_size > 0 && module_inst->module->malloc_function != (uint32)-1 + && module_inst->module->free_function != (uint32)-1) { + /* Disable app heap, use malloc/free function exported + by wasm app to allocate/free memory instead */ + heap_size = 0; + } + + /* If initial memory is the largest size allowed, disallowing insert + * host managed heap */ + if (heap_size > 0 + && heap_offset == GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)) { + set_error_buf(error_buf, error_buf_size, + "failed to insert app heap into linear memory, " + "try using `--heap-size=0` option"); + return NULL; + } + + if (init_page_count == max_page_count && init_page_count == 1) { + /* If only one page and at most one page, we just append + the app heap to the end of linear memory, enlarge the + num_bytes_per_page, and don't change the page count */ + if (heap_size > UINT32_MAX - num_bytes_per_page) { + set_error_buf(error_buf, error_buf_size, + "failed to insert app heap into linear memory, " + "try using `--heap-size=0` option"); + return NULL; + } + heap_offset = num_bytes_per_page; + num_bytes_per_page += heap_size; + } + else if (heap_size > 0) { + if (init_page_count == max_page_count && init_page_count == 0) { + /* If the memory data size is always 0, we resize it to + one page for app heap */ + num_bytes_per_page = heap_size; + heap_offset = 0; + inc_page_count = 1; + } + else if (module->aux_heap_base_global_index != (uint32)-1 + && module->aux_heap_base + < (uint64)num_bytes_per_page * init_page_count) { + /* Insert app heap before __heap_base */ + aux_heap_base = module->aux_heap_base; + bytes_of_last_page = aux_heap_base % num_bytes_per_page; + if (bytes_of_last_page == 0) + bytes_of_last_page = num_bytes_per_page; + bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; + inc_page_count = + (heap_size - bytes_to_page_end + num_bytes_per_page - 1) + / num_bytes_per_page; + heap_offset = aux_heap_base; + aux_heap_base += heap_size; + + bytes_of_last_page = aux_heap_base % num_bytes_per_page; + if (bytes_of_last_page == 0) + bytes_of_last_page = num_bytes_per_page; + bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; + if (bytes_to_page_end < 1 * BH_KB) { + aux_heap_base += 1 * BH_KB; + inc_page_count++; + } + + /* Adjust __heap_base global value */ + global_idx = module->aux_heap_base_global_index; + bh_assert(module_inst->e->globals + && global_idx < module_inst->e->global_count); + global_addr = module_inst->global_data + + module_inst->e->globals[global_idx].data_offset; +#if WASM_ENABLE_MEMORY64 != 0 + if (memory->is_memory64) { + /* For memory64, the global value should be i64 */ + *(uint64 *)global_addr = aux_heap_base; + } + else +#endif + { + /* For memory32, the global value should be i32 */ + *(uint32 *)global_addr = (uint32)aux_heap_base; + } + LOG_VERBOSE("Reset __heap_base global to %" PRIu64, + aux_heap_base); + } + else { + /* Insert app heap before new page */ + inc_page_count = + (heap_size + num_bytes_per_page - 1) / num_bytes_per_page; + heap_offset = (uint64)num_bytes_per_page * init_page_count; + heap_size = (uint64)num_bytes_per_page * inc_page_count; + if (heap_size > 0) + heap_size -= 1 * BH_KB; + } + init_page_count += inc_page_count; + max_page_count += inc_page_count; + if (init_page_count > default_max_page) { + set_error_buf(error_buf, error_buf_size, + "failed to insert app heap into linear memory, " + "try using `--heap-size=0` option"); + return NULL; + } + + if (max_page_count > default_max_page) + max_page_count = default_max_page; + } + } + + LOG_VERBOSE("Memory instantiate:"); + LOG_VERBOSE(" page bytes: %u, init pages: %u, max pages: %u", + num_bytes_per_page, init_page_count, max_page_count); + if (memory_idx == 0) + LOG_VERBOSE(" heap offset: %" PRIu64 ", heap size: %u\n", heap_offset, + heap_size); + + max_memory_data_size = (uint64)num_bytes_per_page * max_page_count; + bh_assert(max_memory_data_size + <= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)); + (void)max_memory_data_size; + + bh_assert(memory != NULL); + + if (wasm_allocate_linear_memory(&memory->memory_data, is_shared_memory, + memory->is_memory64, num_bytes_per_page, + init_page_count, max_page_count, + &memory_data_size) + != BHT_OK) { + set_error_buf(error_buf, error_buf_size, + "allocate linear memory failed"); + return NULL; + } + + memory->module_type = Wasm_Module_Bytecode; + memory->num_bytes_per_page = num_bytes_per_page; + memory->cur_page_count = init_page_count; + memory->max_page_count = max_page_count; + memory->memory_data_size = memory_data_size; + + if (memory_idx == 0) { + memory->heap_data = memory->memory_data + heap_offset; + memory->heap_data_end = memory->heap_data + heap_size; + memory->memory_data_end = memory->memory_data + memory_data_size; + } + + /* Initialize heap */ + if (memory_idx == 0 && heap_size > 0) { + uint32 heap_struct_size = mem_allocator_get_heap_struct_size(); + + if (!(memory->heap_handle = runtime_malloc( + (uint64)heap_struct_size, error_buf, error_buf_size))) { + goto fail1; + } + if (!mem_allocator_create_with_struct_and_pool( + memory->heap_handle, heap_struct_size, memory->heap_data, + heap_size)) { + set_error_buf(error_buf, error_buf_size, "init app heap failed"); + goto fail2; + } + } + + if (memory_data_size > 0) { + wasm_runtime_set_mem_bound_check_bytes(memory, memory_data_size); + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (is_shared_memory) { + memory->is_shared_memory = 1; + memory->ref_count = 1; + } +#endif + + LOG_VERBOSE("Memory instantiate success."); + return memory; + +fail2: + if (memory_idx == 0 && heap_size > 0) + wasm_runtime_free(memory->heap_handle); +fail1: + if (memory->memory_data) + wasm_deallocate_linear_memory(memory); + + return NULL; +} + +/** + * Instantiate memories in a module. + */ +static WASMMemoryInstance ** +memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, + WASMModuleInstance *parent, uint32 heap_size, + uint32 max_memory_pages, char *error_buf, + uint32 error_buf_size) +{ + WASMImport *import; + uint32 mem_index = 0, i, + memory_count = module->import_memory_count + module->memory_count; + uint64 total_size; + WASMMemoryInstance **memories, *memory; + + total_size = sizeof(WASMMemoryInstance *) * (uint64)memory_count; + + if (!(memories = runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + memory = module_inst->global_table_data.memory_instances; + + /* instantiate memories from import section */ + import = module->import_memories; + for (i = 0; i < module->import_memory_count; i++, import++, memory++) { + uint32 num_bytes_per_page = + import->u.memory.mem_type.num_bytes_per_page; + uint32 init_page_count = import->u.memory.mem_type.init_page_count; + uint32 max_page_count = wasm_runtime_get_max_mem( + max_memory_pages, import->u.memory.mem_type.init_page_count, + import->u.memory.mem_type.max_page_count); + uint32 flags = import->u.memory.mem_type.flags; + uint32 actual_heap_size = heap_size; + +#if WASM_ENABLE_MULTI_MODULE != 0 + if (import->u.memory.import_module != NULL) { + WASMModuleInstance *module_inst_linked; + + if (!(module_inst_linked = get_sub_module_inst( + module_inst, import->u.memory.import_module))) { + set_error_buf(error_buf, error_buf_size, "unknown memory"); + memories_deinstantiate(module_inst, memories, memory_count); + return NULL; + } + + if (!(memories[mem_index++] = wasm_lookup_memory( + module_inst_linked, import->u.memory.field_name))) { + set_error_buf(error_buf, error_buf_size, "unknown memory"); + memories_deinstantiate(module_inst, memories, memory_count); + return NULL; + } + } + else +#endif + { + if (!(memories[mem_index] = memory_instantiate( + module_inst, parent, memory, mem_index, + num_bytes_per_page, init_page_count, max_page_count, + actual_heap_size, flags, error_buf, error_buf_size))) { + memories_deinstantiate(module_inst, memories, memory_count); + return NULL; + } + mem_index++; + } + } + + /* instantiate memories from memory section */ + for (i = 0; i < module->memory_count; i++, memory++) { + uint32 max_page_count = wasm_runtime_get_max_mem( + max_memory_pages, module->memories[i].init_page_count, + module->memories[i].max_page_count); + if (!(memories[mem_index] = memory_instantiate( + module_inst, parent, memory, mem_index, + module->memories[i].num_bytes_per_page, + module->memories[i].init_page_count, max_page_count, + heap_size, module->memories[i].flags, error_buf, + error_buf_size))) { + memories_deinstantiate(module_inst, memories, memory_count); + return NULL; + } + mem_index++; + } + + bh_assert(mem_index == memory_count); + (void)module_inst; + return memories; +} + +/** + * Destroy table instances. + */ +static void +tables_deinstantiate(WASMModuleInstance *module_inst) +{ + if (module_inst->tables) { + wasm_runtime_free(module_inst->tables); + } +#if WASM_ENABLE_MULTI_MODULE != 0 + if (module_inst->e->table_insts_linked) { + wasm_runtime_free(module_inst->e->table_insts_linked); + } +#endif +} + +/** + * Instantiate tables in a module. + */ +static WASMTableInstance ** +tables_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, + WASMTableInstance *first_table, char *error_buf, + uint32 error_buf_size) +{ + WASMImport *import; + uint32 table_index = 0, i; + uint32 table_count = module->import_table_count + module->table_count; + uint64 total_size = (uint64)sizeof(WASMTableInstance *) * table_count; + WASMTableInstance **tables, *table = first_table; +#if WASM_ENABLE_MULTI_MODULE != 0 + uint64 total_size_of_tables_linked = + (uint64)sizeof(WASMTableInstance *) * module->import_table_count; + WASMTableInstance **table_linked = NULL; +#endif + + if (!(tables = runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + +#if WASM_ENABLE_MULTI_MODULE != 0 + if (module->import_table_count > 0 + && !(module_inst->e->table_insts_linked = table_linked = runtime_malloc( + total_size_of_tables_linked, error_buf, error_buf_size))) { + goto fail; + } +#endif + + /* instantiate tables from import section */ + import = module->import_tables; + for (i = 0; i < module->import_table_count; i++, import++) { + uint32 max_size_fixed = 0; +#if WASM_ENABLE_MULTI_MODULE != 0 + WASMTableInstance *table_inst_linked = NULL; + WASMModuleInstance *module_inst_linked = NULL; + + if (import->u.table.import_module) { + if (!(module_inst_linked = get_sub_module_inst( + module_inst, import->u.table.import_module))) { + set_error_buf(error_buf, error_buf_size, "unknown table"); + goto fail; + } + + if (!(table_inst_linked = wasm_lookup_table( + module_inst_linked, import->u.table.field_name))) { + set_error_buf(error_buf, error_buf_size, "unknown table"); + goto fail; + } + + total_size = offsetof(WASMTableInstance, elems); + } + else +#endif + { + /* in order to save memory, alloc resource as few as possible */ + max_size_fixed = import->u.table.table_type.possible_grow + ? import->u.table.table_type.max_size + : import->u.table.table_type.init_size; + + /* it is a built-in table, every module has its own */ + total_size = offsetof(WASMTableInstance, elems); + /* store function indexes for non-gc, object pointers for gc */ + total_size += (uint64)sizeof(table_elem_type_t) * max_size_fixed; + } + + tables[table_index++] = table; + +#if WASM_ENABLE_GC == 0 + /* Set all elements to -1 to mark them as uninitialized elements */ + memset(table, -1, (uint32)total_size); +#else + /* For GC, all elements have already been set to NULL_REF (0) as + uninitialized elements */ +#endif + + table->is_table64 = import->u.table.table_type.flags & TABLE64_FLAG; + +#if WASM_ENABLE_MULTI_MODULE != 0 + *table_linked = table_inst_linked; + if (table_inst_linked != NULL) { + table->elem_type = table_inst_linked->elem_type; +#if WASM_ENABLE_GC != 0 + table->elem_ref_type = table_inst_linked->elem_ref_type; +#endif + table->cur_size = table_inst_linked->cur_size; + table->max_size = table_inst_linked->max_size; + } + else +#endif + { + table->elem_type = import->u.table.table_type.elem_type; +#if WASM_ENABLE_GC != 0 + table->elem_ref_type.elem_ref_type = + import->u.table.table_type.elem_ref_type; +#endif + table->cur_size = import->u.table.table_type.init_size; + table->max_size = max_size_fixed; + } + + table = (WASMTableInstance *)((uint8 *)table + (uint32)total_size); +#if WASM_ENABLE_MULTI_MODULE != 0 + table_linked++; +#endif + } + + /* instantiate tables from table section */ + for (i = 0; i < module->table_count; i++) { + uint32 max_size_fixed = 0; + + total_size = offsetof(WASMTableInstance, elems); +#if WASM_ENABLE_MULTI_MODULE != 0 + /* in case, a module which imports this table will grow it */ + max_size_fixed = module->tables[i].table_type.max_size; +#else + max_size_fixed = module->tables[i].table_type.possible_grow + ? module->tables[i].table_type.max_size + : module->tables[i].table_type.init_size; +#endif +#if WASM_ENABLE_GC == 0 + /* Store function indexes */ + total_size += sizeof(uintptr_t) * (uint64)max_size_fixed; +#else + /* Store object pointers */ + total_size += sizeof(uintptr_t) * (uint64)max_size_fixed; +#endif + + tables[table_index++] = table; + +#if WASM_ENABLE_GC == 0 + /* Set all elements to -1 to mark them as uninitialized elements */ + memset(table, -1, (uint32)total_size); +#else + /* For GC, all elements have already been set to NULL_REF (0) as + uninitialized elements */ +#endif + table->is_table64 = module->tables[i].table_type.flags & TABLE64_FLAG; + table->elem_type = module->tables[i].table_type.elem_type; +#if WASM_ENABLE_GC != 0 + table->elem_ref_type.elem_ref_type = + module->tables[i].table_type.elem_ref_type; +#endif + table->cur_size = module->tables[i].table_type.init_size; + table->max_size = max_size_fixed; + + table = (WASMTableInstance *)((uint8 *)table + (uint32)total_size); + } + + bh_assert(table_index == table_count); + (void)module_inst; + return tables; +#if WASM_ENABLE_MULTI_MODULE != 0 +fail: + wasm_runtime_free(tables); + return NULL; +#endif +} + +/** + * Destroy function instances. + */ +static void +functions_deinstantiate(WASMFunctionInstance *functions) +{ + if (functions) { + wasm_runtime_free(functions); + } +} + +/** + * Instantiate functions in a module. + */ +static WASMFunctionInstance * +functions_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, + char *error_buf, uint32 error_buf_size) +{ + WASMImport *import; + uint32 i, + function_count = module->import_function_count + module->function_count; + uint64 total_size = sizeof(WASMFunctionInstance) * (uint64)function_count; + WASMFunctionInstance *functions, *function; + + if (!(functions = runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + total_size = sizeof(void *) * (uint64)module->import_function_count; + if (total_size > 0 + && !(module_inst->import_func_ptrs = + runtime_malloc(total_size, error_buf, error_buf_size))) { + wasm_runtime_free(functions); + return NULL; + } + + /* instantiate functions from import section */ + function = functions; + import = module->import_functions; + for (i = 0; i < module->import_function_count; i++, import++) { + function->is_import_func = true; + +#if WASM_ENABLE_MULTI_MODULE != 0 + if (import->u.function.import_module) { + function->import_module_inst = get_sub_module_inst( + module_inst, import->u.function.import_module); + + if (function->import_module_inst) { + function->import_func_inst = + wasm_lookup_function(function->import_module_inst, + import->u.function.field_name); + } + } +#endif /* WASM_ENABLE_MULTI_MODULE */ + function->u.func_import = &import->u.function; + function->param_cell_num = import->u.function.func_type->param_cell_num; + function->ret_cell_num = import->u.function.func_type->ret_cell_num; + function->param_count = + (uint16)function->u.func_import->func_type->param_count; + function->param_types = function->u.func_import->func_type->types; + function->local_cell_num = 0; + function->local_count = 0; + function->local_types = NULL; + + /* Copy the function pointer to current instance */ + module_inst->import_func_ptrs[i] = + function->u.func_import->func_ptr_linked; + + function++; + } + + /* instantiate functions from function section */ + for (i = 0; i < module->function_count; i++) { + function->is_import_func = false; + function->u.func = module->functions[i]; + + function->param_cell_num = function->u.func->param_cell_num; + function->ret_cell_num = function->u.func->ret_cell_num; + function->local_cell_num = function->u.func->local_cell_num; + + function->param_count = + (uint16)function->u.func->func_type->param_count; + function->local_count = (uint16)function->u.func->local_count; + function->param_types = function->u.func->func_type->types; + function->local_types = function->u.func->local_types; + + function->local_offsets = function->u.func->local_offsets; + +#if WASM_ENABLE_FAST_INTERP != 0 + function->const_cell_num = function->u.func->const_cell_num; +#endif + + function++; + } + bh_assert((uint32)(function - functions) == function_count); + +#if WASM_ENABLE_FAST_JIT != 0 + module_inst->fast_jit_func_ptrs = module->fast_jit_func_ptrs; +#endif + + return functions; +} + +#if WASM_ENABLE_TAGS != 0 +/** + * Destroy tags instances. + */ +static void +tags_deinstantiate(WASMTagInstance *tags, void **import_tag_ptrs) +{ + if (tags) { + wasm_runtime_free(tags); + } + if (import_tag_ptrs) { + wasm_runtime_free(import_tag_ptrs); + } +} + +/** + * Instantiate tags in a module. + */ +static WASMTagInstance * +tags_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, + char *error_buf, uint32 error_buf_size) +{ + WASMImport *import; + uint32 i, tag_count = module->import_tag_count + module->tag_count; + uint64 total_size = sizeof(WASMTagInstance) * (uint64)tag_count; + WASMTagInstance *tags, *tag; + + if (!(tags = runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + total_size = sizeof(void *) * (uint64)module->import_tag_count; + if (total_size > 0 + && !(module_inst->e->import_tag_ptrs = + runtime_malloc(total_size, error_buf, error_buf_size))) { + wasm_runtime_free(tags); + return NULL; + } + + /* instantiate tags from import section */ + tag = tags; + import = module->import_tags; + for (i = 0; i < module->import_tag_count; i++, import++) { + tag->is_import_tag = true; + tag->u.tag_import = &import->u.tag; + tag->type = import->u.tag.type; + tag->attribute = import->u.tag.attribute; +#if WASM_ENABLE_MULTI_MODULE != 0 + if (import->u.tag.import_module) { + if (!(tag->import_module_inst = get_sub_module_inst( + module_inst, import->u.tag.import_module))) { + set_error_buf(error_buf, error_buf_size, "unknown tag"); + goto fail; + } + + if (!(tag->import_tag_inst = + wasm_lookup_tag(tag->import_module_inst, + import->u.tag.field_name, NULL))) { + set_error_buf(error_buf, error_buf_size, "unknown tag"); + goto fail; + } + + /* Copy the imported tag to current instance */ + module_inst->e->import_tag_ptrs[i] = + tag->u.tag_import->import_tag_linked; + } +#endif + tag++; + } + + /* instantiate tags from tag section */ + for (i = 0; i < module->tag_count; i++) { + tag->is_import_tag = false; + tag->type = module->tags[i]->type; + tag->u.tag = module->tags[i]; + +#if WASM_ENABLE_FAST_INTERP != 0 + /* tag->const_cell_num = function->u.func->const_cell_num; */ +#endif + tag++; + } + bh_assert((uint32)(tag - tags) == tag_count); + + return tags; + +#if WASM_ENABLE_MULTI_MODULE != 0 +fail: + tags_deinstantiate(tags, module_inst->e->import_tag_ptrs); + /* clean up */ + module_inst->e->import_tag_ptrs = NULL; + return NULL; +#endif +} +#endif /* end of WASM_ENABLE_TAGS != 0 */ + +/** + * Destroy global instances. + */ +static void +globals_deinstantiate(WASMGlobalInstance *globals) +{ + if (globals) + wasm_runtime_free(globals); +} + +static bool +check_global_init_expr(const WASMModule *module, uint32 global_index, + char *error_buf, uint32 error_buf_size) +{ + if (global_index >= module->import_global_count + module->global_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown global %d", + global_index); + return false; + } + +#if WASM_ENABLE_GC == 0 + /** + * Currently, constant expressions occurring as initializers of + * globals are further constrained in that contained global.get + * instructions are only allowed to refer to imported globals. + * + * And initializer expression cannot reference a mutable global. + */ + if (global_index >= module->import_global_count + || (module->import_globals + global_index)->u.global.type.is_mutable) { + set_error_buf(error_buf, error_buf_size, + "constant expression required"); + return false; + } +#endif + + return true; +} + +#if WASM_ENABLE_GC != 0 +/* Instantiate struct global variable recursively */ +static WASMStructObjectRef +instantiate_struct_global_recursive(WASMModule *module, + WASMModuleInstance *module_inst, + uint32 type_idx, uint8 flag, + WASMStructNewInitValues *init_values, + char *error_buf, uint32 error_buf_size) +{ + WASMRttType *rtt_type; + WASMStructObjectRef struct_obj; + WASMStructType *struct_type; + + struct_type = (WASMStructType *)module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new((WASMType *)struct_type, type_idx, + module->rtt_types, module->type_count, + &module->rtt_type_lock))) { + set_error_buf(error_buf, error_buf_size, "create rtt object failed"); + return NULL; + } + + if (!(struct_obj = wasm_struct_obj_new_internal( + module_inst->e->common.gc_heap_handle, rtt_type))) { + set_error_buf(error_buf, error_buf_size, "create struct object failed"); + return NULL; + } + + if (flag == INIT_EXPR_TYPE_STRUCT_NEW) { + uint32 field_idx; + WASMRefTypeMap *ref_type_map = struct_type->ref_type_maps; + + bh_assert(init_values->count == struct_type->field_count); + + for (field_idx = 0; field_idx < init_values->count; field_idx++) { + uint8 field_type = struct_type->fields[field_idx].field_type; + WASMRefType *field_ref_type = NULL; + if (wasm_is_type_multi_byte_type(field_type)) { + field_ref_type = ref_type_map->ref_type; + } + + if (wasm_reftype_is_subtype_of(field_type, field_ref_type, + REF_TYPE_STRUCTREF, NULL, + module->types, module->type_count) + || wasm_reftype_is_subtype_of(field_type, field_ref_type, + REF_TYPE_ARRAYREF, NULL, + module->types, module->type_count) + || wasm_reftype_is_subtype_of( + field_type, field_ref_type, REF_TYPE_FUNCREF, NULL, + module->types, module->type_count)) { + WASMType *wasm_type; + int32 heap_type = + ref_type_map->ref_type->ref_ht_common.heap_type; + WASMValue *wasm_value = &init_values->fields[field_idx]; + WASMValue field_value = { 0 }; + + bh_assert(heap_type >= 0); + wasm_type = module->types[heap_type]; + + bh_assert(wasm_type->type_flag == WASM_TYPE_STRUCT + || wasm_type->type_flag == WASM_TYPE_ARRAY + || wasm_type->type_flag == WASM_TYPE_FUNC); + + if (wasm_type->type_flag == WASM_TYPE_STRUCT) { + WASMStructNewInitValues *init_values1 = + (WASMStructNewInitValues *)wasm_value->data; + WASMStructObjectRef field = + instantiate_struct_global_recursive( + module, module_inst, heap_type, + init_values1 ? INIT_EXPR_TYPE_STRUCT_NEW + : INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT, + init_values1, error_buf, error_buf_size); + field_value.gc_obj = (WASMObjectRef)field; + wasm_struct_obj_set_field(struct_obj, field_idx, + &field_value); + } + else if (wasm_type->type_flag == WASM_TYPE_ARRAY) { + /* struct object's field is an array obj */ + set_error_buf(error_buf, error_buf_size, + "array as a field in struct object is " + "not supported in constant init expr"); + return NULL; + } + else if (wasm_type->type_flag == WASM_TYPE_FUNC) { + WASMFuncObjectRef func_obj = NULL; + /* UINT32_MAX indicates that it is a null reference */ + if (wasm_value->u32 != UINT32_MAX) { + if (!(func_obj = wasm_create_func_obj( + module_inst, wasm_value->u32, false, + error_buf, error_buf_size))) { + return NULL; + } + } + field_value.gc_obj = (WASMObjectRef)func_obj; + wasm_struct_obj_set_field(struct_obj, field_idx, + &field_value); + } + } + else { + wasm_struct_obj_set_field(struct_obj, field_idx, + &init_values->fields[field_idx]); + } + if (wasm_is_type_multi_byte_type(field_type)) { + ref_type_map++; + } + } + } + + return struct_obj; +} + +static WASMArrayObjectRef +instantiate_array_global_recursive(WASMModule *module, + WASMModuleInstance *module_inst, + uint32 type_idx, uint8 flag, uint32 len, + WASMValue *array_init_value, + WASMArrayNewInitValues *init_values, + char *error_buf, uint32 error_buf_size) +{ + WASMRttType *rtt_type; + WASMArrayObjectRef array_obj; + WASMArrayType *array_type; + + array_type = (WASMArrayType *)module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new((WASMType *)array_type, type_idx, + module->rtt_types, module->type_count, + &module->rtt_type_lock))) { + set_error_buf(error_buf, error_buf_size, "create rtt object failed"); + return NULL; + } + + if (!(array_obj = + wasm_array_obj_new_internal(module_inst->e->common.gc_heap_handle, + rtt_type, len, array_init_value))) { + set_error_buf(error_buf, error_buf_size, "create array object failed"); + return NULL; + } + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) { + uint32 elem_idx; + uint8 elem_type = array_type->elem_type; + WASMRefType *elem_ref_type = array_type->elem_ref_type; + + bh_assert(init_values); + + if (wasm_reftype_is_subtype_of(elem_type, elem_ref_type, + REF_TYPE_STRUCTREF, NULL, module->types, + module->type_count) + || wasm_reftype_is_subtype_of(elem_type, elem_ref_type, + REF_TYPE_ARRAYREF, NULL, + module->types, module->type_count) + || wasm_reftype_is_subtype_of(elem_type, elem_ref_type, + REF_TYPE_FUNCREF, NULL, module->types, + module->type_count)) { + /* TODO */ + } + + for (elem_idx = 0; elem_idx < len; elem_idx++) { + wasm_array_obj_set_elem(array_obj, elem_idx, + &init_values->elem_data[elem_idx]); + } + } + + return array_obj; +} +#endif + +static bool +get_init_value_recursive(WASMModule *module, InitializerExpression *expr, + WASMGlobalInstance *globals, WASMValue *value, + char *error_buf, uint32 error_buf_size) +{ + uint8 flag = expr->init_expr_type; + switch (flag) { + case INIT_EXPR_TYPE_GET_GLOBAL: + { + if (!check_global_init_expr(module, expr->u.unary.v.global_index, + error_buf, error_buf_size)) { + goto fail; + } + + *value = globals[expr->u.unary.v.global_index].initial_value; + break; + } + case INIT_EXPR_TYPE_I32_CONST: + case INIT_EXPR_TYPE_I64_CONST: + { + *value = expr->u.unary.v; + break; + } +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + case INIT_EXPR_TYPE_I32_ADD: + case INIT_EXPR_TYPE_I32_SUB: + case INIT_EXPR_TYPE_I32_MUL: + case INIT_EXPR_TYPE_I64_ADD: + case INIT_EXPR_TYPE_I64_SUB: + case INIT_EXPR_TYPE_I64_MUL: + { + WASMValue l_value, r_value; + if (!expr->u.binary.l_expr || !expr->u.binary.r_expr) { + goto fail; + } + if (!get_init_value_recursive(module, expr->u.binary.l_expr, + globals, &l_value, error_buf, + error_buf_size)) { + goto fail; + } + if (!get_init_value_recursive(module, expr->u.binary.r_expr, + globals, &r_value, error_buf, + error_buf_size)) { + goto fail; + } + + if (flag == INIT_EXPR_TYPE_I32_ADD) { + value->i32 = l_value.i32 + r_value.i32; + } + else if (flag == INIT_EXPR_TYPE_I32_SUB) { + value->i32 = l_value.i32 - r_value.i32; + } + else if (flag == INIT_EXPR_TYPE_I32_MUL) { + value->i32 = l_value.i32 * r_value.i32; + } + else if (flag == INIT_EXPR_TYPE_I64_ADD) { + value->i64 = l_value.i64 + r_value.i64; + } + else if (flag == INIT_EXPR_TYPE_I64_SUB) { + value->i64 = l_value.i64 - r_value.i64; + } + else if (flag == INIT_EXPR_TYPE_I64_MUL) { + value->i64 = l_value.i64 * r_value.i64; + } + break; + } +#endif /* end of WASM_ENABLE_EXTENDED_CONST_EXPR != 0 */ + default: + goto fail; + } + return true; +fail: + return false; +} + +/** + * Instantiate globals in a module. + */ +static WASMGlobalInstance * +globals_instantiate(WASMModule *module, WASMModuleInstance *module_inst, + char *error_buf, uint32 error_buf_size) +{ + WASMImport *import; + uint32 global_data_offset = 0; + uint32 i, global_count = module->import_global_count + module->global_count; + uint64 total_size = sizeof(WASMGlobalInstance) * (uint64)global_count; + WASMGlobalInstance *globals, *global; + + if (!(globals = runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + /* instantiate globals from import section */ + global = globals; + import = module->import_globals; + for (i = 0; i < module->import_global_count; i++, import++) { + WASMGlobalImport *global_import = &import->u.global; + global->type = global_import->type.val_type; + global->is_mutable = global_import->type.is_mutable; +#if WASM_ENABLE_GC != 0 + global->ref_type = global_import->ref_type; +#endif +#if WASM_ENABLE_MULTI_MODULE != 0 + if (global_import->import_module) { + if (!(global->import_module_inst = get_sub_module_inst( + module_inst, global_import->import_module))) { + set_error_buf(error_buf, error_buf_size, "unknown global"); + goto fail; + } + + if (!(global->import_global_inst = wasm_lookup_global( + global->import_module_inst, global_import->field_name))) { + set_error_buf(error_buf, error_buf_size, "unknown global"); + goto fail; + } + + /* The linked global instance has been initialized, we + just need to copy the value. */ + global->initial_value = + global_import->import_global_linked->init_expr.u.unary.v; + } + else +#endif + { + /* native globals share their initial_values in one module */ + bh_memcpy_s(&(global->initial_value), sizeof(WASMValue), + &(global_import->global_data_linked), + sizeof(WASMValue)); + } +#if WASM_ENABLE_FAST_JIT != 0 + bh_assert(global_data_offset == global_import->data_offset); +#endif + global->data_offset = global_data_offset; + global_data_offset += wasm_value_type_size(global->type); + + global++; + } + + /* instantiate globals from global section */ + for (i = 0; i < module->global_count; i++) { + InitializerExpression *init_expr = &(module->globals[i].init_expr); + uint8 flag = init_expr->init_expr_type; + + global->type = module->globals[i].type.val_type; + global->is_mutable = module->globals[i].type.is_mutable; +#if WASM_ENABLE_FAST_JIT != 0 + bh_assert(global_data_offset == module->globals[i].data_offset); +#endif + global->data_offset = global_data_offset; + global_data_offset += wasm_value_type_size(global->type); +#if WASM_ENABLE_GC != 0 + global->ref_type = module->globals[i].ref_type; +#endif + + switch (flag) { + case INIT_EXPR_TYPE_I32_CONST: + case INIT_EXPR_TYPE_I64_CONST: + case INIT_EXPR_TYPE_GET_GLOBAL: +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + case INIT_EXPR_TYPE_I32_ADD: + case INIT_EXPR_TYPE_I32_SUB: + case INIT_EXPR_TYPE_I32_MUL: + case INIT_EXPR_TYPE_I64_ADD: + case INIT_EXPR_TYPE_I64_SUB: + case INIT_EXPR_TYPE_I64_MUL: +#endif + { + if (!get_init_value_recursive(module, init_expr, globals, + &global->initial_value, error_buf, + error_buf_size)) { + goto fail; + } + break; + } +#if WASM_ENABLE_GC != 0 + case INIT_EXPR_TYPE_STRUCT_NEW: + case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT: + { + WASMStructObjectRef struct_obj; + WASMStructNewInitValues *init_values = NULL; + uint32 type_idx; + + if (flag == INIT_EXPR_TYPE_STRUCT_NEW) { + init_values = + (WASMStructNewInitValues *)init_expr->u.unary.v.data; + type_idx = init_values->type_idx; + } + else { + type_idx = init_expr->u.unary.v.type_index; + } + + struct_obj = instantiate_struct_global_recursive( + module, module_inst, type_idx, flag, init_values, error_buf, + error_buf_size); + if (!struct_obj) { + goto fail; + } + + global->initial_value.gc_obj = (void *)struct_obj; + break; + } + case INIT_EXPR_TYPE_ARRAY_NEW: + case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT: + case INIT_EXPR_TYPE_ARRAY_NEW_FIXED: + { + WASMArrayObjectRef array_obj; + WASMArrayNewInitValues *init_values = NULL; + WASMValue *array_init_value = NULL, empty_value = { 0 }; + uint32 type_idx, len; + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT) { + type_idx = + init_expr->u.unary.v.array_new_default.type_index; + len = init_expr->u.unary.v.array_new_default.length; + array_init_value = &empty_value; + } + else { + init_values = + (WASMArrayNewInitValues *)init_expr->u.unary.v.data; + type_idx = init_values->type_idx; + len = init_values->length; + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW) { + array_init_value = init_values->elem_data; + } + } + + array_obj = instantiate_array_global_recursive( + module, module_inst, type_idx, flag, len, array_init_value, + init_values, error_buf, error_buf_size); + + global->initial_value.gc_obj = (void *)array_obj; + break; + } + case INIT_EXPR_TYPE_I31_NEW: + { + global->initial_value.gc_obj = + (wasm_obj_t)wasm_i31_obj_new(init_expr->u.unary.v.i32); + break; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + default: + global->initial_value = init_expr->u.unary.v; + break; + } + + global++; + } + + bh_assert((uint32)(global - globals) == global_count); + bh_assert(global_data_offset == module->global_data_size); + (void)module_inst; + return globals; +fail: + wasm_runtime_free(globals); + return NULL; +} + +/** + * Return export function count in module export section. + */ +static uint32 +get_export_count(const WASMModule *module, uint8 kind) +{ + WASMExport *export = module->exports; + uint32 count = 0, i; + + for (i = 0; i < module->export_count; i++, export ++) + if (export->kind == kind) + count++; + + return count; +} + +/** + * Destroy export function instances. + */ +static void +export_functions_deinstantiate(WASMExportFuncInstance *functions) +{ + if (functions) + wasm_runtime_free(functions); +} + +static int +cmp_export_func_inst(const void *a, const void *b) +{ + const WASMExportFuncInstance *export_func1 = + (const WASMExportFuncInstance *)a; + const WASMExportFuncInstance *export_func2 = + (const WASMExportFuncInstance *)b; + + return strcmp(export_func1->name, export_func2->name); +} + +/** + * Instantiate export functions in a module. + */ +static WASMExportFuncInstance * +export_functions_instantiate(const WASMModule *module, + WASMModuleInstance *module_inst, + uint32 export_func_count, char *error_buf, + uint32 error_buf_size) +{ + WASMExportFuncInstance *export_funcs, *export_func; + WASMExport *export = module->exports; + uint32 i; + uint64 total_size = + sizeof(WASMExportFuncInstance) * (uint64)export_func_count; + + if (!(export_func = export_funcs = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + for (i = 0; i < module->export_count; i++, export ++) + if (export->kind == EXPORT_KIND_FUNC) { + export_func->name = export->name; + export_func->function = &module_inst->e->functions[export->index]; + export_func++; + } + + bh_assert((uint32)(export_func - export_funcs) == export_func_count); + + qsort(export_funcs, export_func_count, sizeof(WASMExportFuncInstance), + cmp_export_func_inst); + return export_funcs; +} + +#if WASM_ENABLE_TAGS != 0 +/** + * Destroy export function instances. + */ +static void +export_tags_deinstantiate(WASMExportTagInstance *tags) +{ + if (tags) + wasm_runtime_free(tags); +} + +/** + * Instantiate export functions in a module. + */ +static WASMExportTagInstance * +export_tags_instantiate(const WASMModule *module, + WASMModuleInstance *module_inst, + uint32 export_tag_count, char *error_buf, + uint32 error_buf_size) +{ + WASMExportTagInstance *export_tags, *export_tag; + WASMExport *export = module->exports; + uint32 i; + uint64 total_size = + sizeof(WASMExportTagInstance) * (uint64)export_tag_count; + + if (!(export_tag = export_tags = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + for (i = 0; i < module->export_count; i++, export ++) + if (export->kind == EXPORT_KIND_TAG) { + export_tag->name = export->name; + + bh_assert(module_inst->e->tags); + + export_tag->tag = &module_inst->e->tags[export->index]; + export_tag++; + } + + bh_assert((uint32)(export_tag - export_tags) == export_tag_count); + return export_tags; +} +#endif /* end of WASM_ENABLE_TAGS != 0 */ + +#if WASM_ENABLE_MULTI_MEMORY != 0 +static void +export_memories_deinstantiate(WASMExportMemInstance *memories) +{ + if (memories) + wasm_runtime_free(memories); +} + +static WASMExportMemInstance * +export_memories_instantiate(const WASMModule *module, + WASMModuleInstance *module_inst, + uint32 export_mem_count, char *error_buf, + uint32 error_buf_size) +{ + WASMExportMemInstance *export_memories, *export_memory; + WASMExport *export = module->exports; + uint32 i; + uint64 total_size = + sizeof(WASMExportMemInstance) * (uint64)export_mem_count; + + if (!(export_memory = export_memories = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + for (i = 0; i < module->export_count; i++, export ++) + if (export->kind == EXPORT_KIND_MEMORY) { + export_memory->name = export->name; + export_memory->memory = module_inst->memories[export->index]; + export_memory++; + } + + bh_assert((uint32)(export_memory - export_memories) == export_mem_count); + return export_memories; +} +#endif /* end of if WASM_ENABLE_MULTI_MEMORY != 0 */ + +#if WASM_ENABLE_MULTI_MODULE != 0 +static void +export_globals_deinstantiate(WASMExportGlobInstance *globals) +{ + if (globals) + wasm_runtime_free(globals); +} + +static WASMExportGlobInstance * +export_globals_instantiate(const WASMModule *module, + WASMModuleInstance *module_inst, + uint32 export_glob_count, char *error_buf, + uint32 error_buf_size) +{ + WASMExportGlobInstance *export_globals, *export_global; + WASMExport *export = module->exports; + uint32 i; + uint64 total_size = + sizeof(WASMExportGlobInstance) * (uint64)export_glob_count; + + if (!(export_global = export_globals = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + for (i = 0; i < module->export_count; i++, export ++) + if (export->kind == EXPORT_KIND_GLOBAL) { + export_global->name = export->name; + export_global->global = &module_inst->e->globals[export->index]; + export_global++; + } + + bh_assert((uint32)(export_global - export_globals) == export_glob_count); + return export_globals; +} + +#endif /* end of if WASM_ENABLE_MULTI_MODULE != 0 */ + +static WASMFunctionInstance * +lookup_post_instantiate_func(WASMModuleInstance *module_inst, + const char *func_name) +{ + WASMFunctionInstance *func; + WASMFuncType *func_type; + + if (!(func = wasm_lookup_function(module_inst, func_name))) + /* Not found */ + return NULL; + + func_type = func->u.func->func_type; + if (!(func_type->param_count == 0 && func_type->result_count == 0)) + /* Not a valid function type, ignore it */ + return NULL; + + return func; +} + +static bool +execute_post_instantiate_functions(WASMModuleInstance *module_inst, + bool is_sub_inst, WASMExecEnv *exec_env_main) +{ + WASMFunctionInstance *start_func = module_inst->e->start_function; + WASMFunctionInstance *initialize_func = NULL; + WASMFunctionInstance *post_inst_func = NULL; + WASMFunctionInstance *call_ctors_func = NULL; +#if WASM_ENABLE_LIBC_WASI != 0 + WASMModule *module = module_inst->module; +#endif + WASMModuleInstanceCommon *module_inst_main = NULL; +#ifdef OS_ENABLE_HW_BOUND_CHECK + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); +#endif + WASMExecEnv *exec_env = NULL, *exec_env_created = NULL; + bool ret = false; + +#if WASM_ENABLE_LIBC_WASI != 0 + /* + * WASI reactor instances may assume that _initialize will be called by + * the environment at most once, and that none of their other exports + * are accessed before that call. + */ + if (!is_sub_inst && module->import_wasi_api) { + initialize_func = + lookup_post_instantiate_func(module_inst, "_initialize"); + } +#endif + + /* Execute possible "__post_instantiate" function if wasm app is + compiled by emsdk's early version */ + if (!is_sub_inst) { + post_inst_func = + lookup_post_instantiate_func(module_inst, "__post_instantiate"); + } + +#if WASM_ENABLE_BULK_MEMORY != 0 + /* Only execute the memory init function for main instance since + the data segments will be dropped once initialized */ + if (!is_sub_inst +#if WASM_ENABLE_LIBC_WASI != 0 + && !module->import_wasi_api +#endif + ) { + call_ctors_func = + lookup_post_instantiate_func(module_inst, "__wasm_call_ctors"); + } +#endif + + if (!start_func && !initialize_func && !post_inst_func + && !call_ctors_func) { + /* No post instantiation functions to call */ + return true; + } + + if (is_sub_inst) { + bh_assert(exec_env_main); +#ifdef OS_ENABLE_HW_BOUND_CHECK + /* May come from pthread_create_wrapper, thread_spawn_wrapper and + wasm_cluster_spawn_exec_env. If it comes from the former two, + the exec_env_tls must be not NULL and equal to exec_env_main, + else if it comes from the last one, it may be NULL. */ + if (exec_env_tls) + bh_assert(exec_env_tls == exec_env_main); +#endif + exec_env = exec_env_main; + + /* Temporarily replace parent exec_env's module inst to current + module inst to avoid checking failure when calling the + wasm functions, and ensure that the exec_env's module inst + is the correct one. */ + module_inst_main = exec_env_main->module_inst; + wasm_exec_env_set_module_inst(exec_env, + (WASMModuleInstanceCommon *)module_inst); + } + else { + /* Try using the existing exec_env */ +#ifdef OS_ENABLE_HW_BOUND_CHECK + exec_env = exec_env_tls; +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + if (!exec_env) + exec_env = wasm_clusters_search_exec_env( + (WASMModuleInstanceCommon *)module_inst); +#endif + if (!exec_env) { + if (!(exec_env = exec_env_created = wasm_exec_env_create( + (WASMModuleInstanceCommon *)module_inst, + module_inst->default_wasm_stack_size))) { + wasm_set_exception(module_inst, "allocate memory failed"); + return false; + } + } + else { + /* Temporarily replace exec_env's module inst with current + module inst to ensure that the exec_env's module inst + is the correct one. */ + module_inst_main = exec_env->module_inst; + wasm_exec_env_set_module_inst( + exec_env, (WASMModuleInstanceCommon *)module_inst); + } + } + + /* Execute start function for both main instance and sub instance */ + if (start_func && !wasm_call_function(exec_env, start_func, 0, NULL)) { + goto fail; + } + +#if WASM_ENABLE_LIBC_WASI != 0 + if (initialize_func + && !wasm_call_function(exec_env, initialize_func, 0, NULL)) { + goto fail; + } +#else + (void)initialize_func; +#endif + + if (post_inst_func + && !wasm_call_function(exec_env, post_inst_func, 0, NULL)) { + goto fail; + } + + if (call_ctors_func + && !wasm_call_function(exec_env, call_ctors_func, 0, NULL)) { + goto fail; + } + + ret = true; + +fail: + if (is_sub_inst) { + /* Restore the parent exec_env's module inst */ + wasm_exec_env_restore_module_inst(exec_env_main, module_inst_main); + } + else { + if (module_inst_main) + /* Restore the existing exec_env's module inst */ + wasm_exec_env_restore_module_inst(exec_env, module_inst_main); + if (exec_env_created) + wasm_exec_env_destroy(exec_env_created); + } + + return ret; +} + +static bool +execute_malloc_function(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, + WASMFunctionInstance *malloc_func, + WASMFunctionInstance *retain_func, uint64 size, + uint64 *p_result) +{ +#ifdef OS_ENABLE_HW_BOUND_CHECK + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); +#endif + WASMExecEnv *exec_env_created = NULL; + WASMModuleInstanceCommon *module_inst_old = NULL; + union { + uint32 u32[3]; + uint64 u64; + } argv; + uint32 argc; + bool ret; +#if WASM_ENABLE_MEMORY64 != 0 + bool is_memory64 = module_inst->memories[0]->is_memory64; + if (is_memory64) { + argc = 2; + PUT_I64_TO_ADDR(&argv.u64, size); + } + else +#endif + { + argc = 1; + argv.u32[0] = (uint32)size; + } + + /* if __retain is exported, then this module is compiled by + assemblyscript, the memory should be managed by as's runtime, + in this case we need to call the retain function after malloc + the memory */ + if (retain_func) { + /* the malloc function from assemblyscript is: + function __new(size: usize, id: u32) + id = 0 means this is an ArrayBuffer object */ + argv.u32[argc] = 0; + argc++; + } + + if (exec_env) { +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (exec_env_tls) { + bh_assert(exec_env_tls == exec_env); + } +#endif + bh_assert(exec_env->module_inst + == (WASMModuleInstanceCommon *)module_inst); + } + else { + /* Try using the existing exec_env */ +#ifdef OS_ENABLE_HW_BOUND_CHECK + exec_env = exec_env_tls; +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + if (!exec_env) + exec_env = wasm_clusters_search_exec_env( + (WASMModuleInstanceCommon *)module_inst); +#endif + if (!exec_env) { + if (!(exec_env = exec_env_created = wasm_exec_env_create( + (WASMModuleInstanceCommon *)module_inst, + module_inst->default_wasm_stack_size))) { + wasm_set_exception(module_inst, "allocate memory failed"); + return false; + } + } + else { + /* Temporarily replace exec_env's module inst with current + module inst to ensure that the exec_env's module inst + is the correct one. */ + module_inst_old = exec_env->module_inst; + wasm_exec_env_set_module_inst( + exec_env, (WASMModuleInstanceCommon *)module_inst); + } + } + + ret = wasm_call_function(exec_env, malloc_func, argc, argv.u32); + + if (retain_func && ret) + ret = wasm_call_function(exec_env, retain_func, 1, argv.u32); + + if (module_inst_old) + /* Restore the existing exec_env's module inst */ + wasm_exec_env_restore_module_inst(exec_env, module_inst_old); + + if (exec_env_created) + wasm_exec_env_destroy(exec_env_created); + + if (ret) { +#if WASM_ENABLE_MEMORY64 != 0 + if (is_memory64) + *p_result = argv.u64; + else +#endif + { + *p_result = argv.u32[0]; + } + } + return ret; +} + +static bool +execute_free_function(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, + WASMFunctionInstance *free_func, uint64 offset) +{ +#ifdef OS_ENABLE_HW_BOUND_CHECK + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); +#endif + WASMExecEnv *exec_env_created = NULL; + WASMModuleInstanceCommon *module_inst_old = NULL; + union { + uint32 u32[2]; + uint64 u64; + } argv; + uint32 argc; + bool ret; + +#if WASM_ENABLE_MEMORY64 != 0 + if (module_inst->memories[0]->is_memory64) { + PUT_I64_TO_ADDR(&argv.u64, offset); + argc = 2; + } + else +#endif + { + argv.u32[0] = (uint32)offset; + argc = 1; + } + + if (exec_env) { +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (exec_env_tls) { + bh_assert(exec_env_tls == exec_env); + } +#endif + bh_assert(exec_env->module_inst + == (WASMModuleInstanceCommon *)module_inst); + } + else { + /* Try using the existing exec_env */ +#ifdef OS_ENABLE_HW_BOUND_CHECK + exec_env = exec_env_tls; +#endif +#if WASM_ENABLE_THREAD_MGR != 0 + if (!exec_env) + exec_env = wasm_clusters_search_exec_env( + (WASMModuleInstanceCommon *)module_inst); +#endif + if (!exec_env) { + if (!(exec_env = exec_env_created = wasm_exec_env_create( + (WASMModuleInstanceCommon *)module_inst, + module_inst->default_wasm_stack_size))) { + wasm_set_exception(module_inst, "allocate memory failed"); + return false; + } + } + else { + /* Temporarily replace exec_env's module inst with current + module inst to ensure that the exec_env's module inst + is the correct one. */ + module_inst_old = exec_env->module_inst; + wasm_exec_env_set_module_inst( + exec_env, (WASMModuleInstanceCommon *)module_inst); + } + } + + ret = wasm_call_function(exec_env, free_func, argc, argv.u32); + + if (module_inst_old) + /* Restore the existing exec_env's module inst */ + wasm_exec_env_restore_module_inst(exec_env, module_inst_old); + + if (exec_env_created) + wasm_exec_env_destroy(exec_env_created); + + return ret; +} + +static bool +check_linked_symbol(WASMModuleInstance *module_inst, char *error_buf, + uint32 error_buf_size) +{ + WASMModule *module = module_inst->module; + uint32 i; + + for (i = 0; i < module->import_function_count; i++) { + WASMFunctionImport *func = + &((module->import_functions + i)->u.function); + if (!func->func_ptr_linked +#if WASM_ENABLE_MULTI_MODULE != 0 + && !func->import_func_linked +#endif + ) { + LOG_WARNING("warning: failed to link import function (%s, %s)", + func->module_name, func->field_name); + } + } + + for (i = 0; i < module->import_global_count; i++) { + WASMGlobalImport *global = &((module->import_globals + i)->u.global); + + if (!global->is_linked) { +#if WASM_ENABLE_SPEC_TEST != 0 + set_error_buf(error_buf, error_buf_size, + "unknown import or incompatible import type"); + return false; +#else + set_error_buf_v(error_buf, error_buf_size, + "failed to link import global (%s, %s)", + global->module_name, global->field_name); + return false; +#endif /* WASM_ENABLE_SPEC_TEST != 0 */ + } + } + + for (i = 0; i < module->import_table_count; i++) { + WASMTableImport *table = &((module->import_tables + i)->u.table); + + if (!wasm_runtime_is_built_in_module(table->module_name) +#if WASM_ENABLE_MULTI_MODULE != 0 + && !table->import_table_linked +#endif + ) { + set_error_buf_v(error_buf, error_buf_size, + "failed to link import table (%s, %s)", + table->module_name, table->field_name); + return false; + } + } + + for (i = 0; i < module->import_memory_count; i++) { + WASMMemoryImport *memory = &((module->import_memories + i)->u.memory); + + if (!wasm_runtime_is_built_in_module(memory->module_name) +#if WASM_ENABLE_MULTI_MODULE != 0 + && !memory->import_memory_linked +#endif + ) { + set_error_buf_v(error_buf, error_buf_size, + "failed to link import memory (%s, %s)", + memory->module_name, memory->field_name); + return false; + } + } + +#if WASM_ENABLE_MULTI_MODULE != 0 +#if WASM_ENABLE_TAGS != 0 + for (i = 0; i < module->import_tag_count; i++) { + WASMTagImport *tag = &((module->import_tags + i)->u.tag); + + if (!tag->import_tag_linked) { + set_error_buf_v(error_buf, error_buf_size, + "failed to link import tag (%s, %s)", + tag->module_name, tag->field_name); + return false; + } + } +#endif /* WASM_ENABLE_TAGS != 0 */ +#endif + + return true; +} + +#if WASM_ENABLE_JIT != 0 +static bool +init_func_ptrs(WASMModuleInstance *module_inst, WASMModule *module, + char *error_buf, uint32 error_buf_size) +{ + uint32 i; + void **func_ptrs; + uint64 total_size = (uint64)sizeof(void *) * module_inst->e->function_count; + + /* Allocate memory */ + if (!(func_ptrs = module_inst->func_ptrs = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + /* Set import function pointers */ + for (i = 0; i < module->import_function_count; i++, func_ptrs++) { + WASMFunctionImport *import_func = + &module->import_functions[i].u.function; + /* TODO: handle multi module */ + *func_ptrs = import_func->func_ptr_linked; + } + + /* The defined function pointers will be set in + wasm_runtime_set_running_mode, no need to set them here */ + return true; +} +#endif /* end of WASM_ENABLE_JIT != 0 */ + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 +static uint32 +get_smallest_type_idx(WASMModule *module, WASMFuncType *func_type) +{ + uint32 i; + + for (i = 0; i < module->type_count; i++) { + if (func_type == (WASMFuncType *)module->types[i]) + return i; + } + + bh_assert(0); + return -1; +} + +static bool +init_func_type_indexes(WASMModuleInstance *module_inst, char *error_buf, + uint32 error_buf_size) +{ + uint32 i; + uint64 total_size = (uint64)sizeof(uint32) * module_inst->e->function_count; + + /* Allocate memory */ + if (!(module_inst->func_type_indexes = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < module_inst->e->function_count; i++) { + WASMFunctionInstance *func_inst = module_inst->e->functions + i; + WASMFuncType *func_type = func_inst->is_import_func + ? func_inst->u.func_import->func_type + : func_inst->u.func->func_type; + module_inst->func_type_indexes[i] = + get_smallest_type_idx(module_inst->module, func_type); + } + + return true; +} +#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 */ + +#if WASM_ENABLE_GC != 0 +void * +wasm_create_func_obj(WASMModuleInstance *module_inst, uint32 func_idx, + bool throw_exce, char *error_buf, uint32 error_buf_size) +{ + WASMModule *module = module_inst->module; + WASMRttTypeRef rtt_type; + WASMFuncObjectRef func_obj; + WASMFuncType *func_type; + uint32 type_idx; + + if (throw_exce) { + error_buf = module_inst->cur_exception; + error_buf_size = sizeof(module_inst->cur_exception); + } + + if (func_idx >= module->import_function_count + module->function_count) { + set_error_buf_v(error_buf, error_buf_size, "unknown function %d", + func_idx); + return NULL; + } + + if (func_idx < module->import_function_count) { + func_type = module->import_functions[func_idx].u.function.func_type; + type_idx = module->import_functions[func_idx].u.function.type_idx; + } + else { + func_type = module->functions[func_idx - module->import_function_count] + ->func_type; + type_idx = module->functions[func_idx - module->import_function_count] + ->type_idx; + } + + if (!(rtt_type = wasm_rtt_type_new((WASMType *)func_type, type_idx, + module->rtt_types, module->type_count, + &module->rtt_type_lock))) { + set_error_buf(error_buf, error_buf_size, "create rtt object failed"); + return NULL; + } + + if (!(func_obj = wasm_func_obj_new_internal( + module_inst->e->common.gc_heap_handle, rtt_type, func_idx))) { + set_error_buf(error_buf, error_buf_size, "create func object failed"); + return NULL; + } + + return func_obj; +} + +static bool +wasm_global_traverse_gc_rootset(WASMModuleInstance *module_inst, void *heap) +{ + WASMGlobalInstance *global = module_inst->e->globals; + WASMGlobalInstance *global_end = global + module_inst->e->global_count; + uint8 *global_data = module_inst->global_data; + WASMObjectRef gc_obj; + + while (global < global_end) { + if (wasm_is_type_reftype(global->type)) { + gc_obj = GET_REF_FROM_ADDR( + (uint32 *)(global_data + global->data_offset)); + if (wasm_obj_is_created_from_heap(gc_obj)) { + if (0 != mem_allocator_add_root((mem_allocator_t)heap, gc_obj)) + return false; + } + } + global++; + } + return true; +} + +static bool +wasm_table_traverse_gc_rootset(WASMModuleInstance *module_inst, void *heap) +{ + WASMTableInstance **tables = module_inst->tables, *table; + uint32 table_count = module_inst->table_count, i, j; + WASMObjectRef gc_obj, *table_elems; + + for (i = 0; i < table_count; i++) { + table = tables[i]; + table_elems = (WASMObjectRef *)table->elems; + for (j = 0; j < table->cur_size; j++) { + gc_obj = table_elems[j]; + if (wasm_obj_is_created_from_heap(gc_obj)) { + if (0 != mem_allocator_add_root((mem_allocator_t)heap, gc_obj)) + return false; + } + } + } + + return true; +} + +static bool +local_object_refs_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap) +{ + WASMLocalObjectRef *r; + WASMObjectRef gc_obj; + + for (r = exec_env->cur_local_object_ref; r; r = r->prev) { + gc_obj = r->val; + if (wasm_obj_is_created_from_heap(gc_obj)) { + if (0 != mem_allocator_add_root((mem_allocator_t)heap, gc_obj)) + return false; + } + } + return true; +} + +bool +wasm_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap) +{ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)exec_env->module_inst; + bool ret; + + ret = wasm_global_traverse_gc_rootset(module_inst, heap); + if (!ret) + return ret; + + ret = wasm_table_traverse_gc_rootset(module_inst, heap); + if (!ret) + return ret; + + ret = local_object_refs_traverse_gc_rootset(exec_env, heap); + if (!ret) + return ret; + + return wasm_interp_traverse_gc_rootset(exec_env, heap); +} +#endif /* end of WASM_ENABLE_GC != 0 */ + +static bool +set_running_mode(WASMModuleInstance *module_inst, RunningMode running_mode, + bool first_time_set) +{ + WASMModule *module = module_inst->module; + + if (running_mode == Mode_Default) { +#if WASM_ENABLE_FAST_JIT == 0 && WASM_ENABLE_JIT == 0 + running_mode = Mode_Interp; +#elif WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT == 0 + running_mode = Mode_Fast_JIT; +#elif WASM_ENABLE_FAST_JIT == 0 && WASM_ENABLE_JIT != 0 + running_mode = Mode_LLVM_JIT; +#else /* WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 */ +#if WASM_ENABLE_LAZY_JIT == 0 + running_mode = Mode_LLVM_JIT; +#else + running_mode = Mode_Multi_Tier_JIT; +#endif +#endif + } + + if (!wasm_runtime_is_running_mode_supported(running_mode)) + return false; + +#if !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0) /* No possible multi-tier JIT */ + (void)first_time_set; + module_inst->e->running_mode = running_mode; + + if (running_mode == Mode_Interp) { + /* Do nothing for Mode_Interp */ + } + else if (running_mode == Mode_Fast_JIT) { + /* Do nothing for Mode_Fast_JIT since + module_inst->fast_jit_func_ptrs is same as + module->fast_jit_func_ptrs */ + } +#if WASM_ENABLE_JIT != 0 + else if (running_mode == Mode_LLVM_JIT) { + /* Set defined function pointers */ + bh_memcpy_s(module_inst->func_ptrs + module->import_function_count, + sizeof(void *) * module->function_count, module->func_ptrs, + sizeof(void *) * module->function_count); + } +#endif + else { + bh_assert(0); + } +#else /* Possible multi-tier JIT */ + os_mutex_lock(&module->instance_list_lock); + + module_inst->e->running_mode = running_mode; + + if (running_mode == Mode_Interp) { + /* Do nothing for Mode_Interp */ + } +#if WASM_ENABLE_FAST_JIT != 0 + else if (running_mode == Mode_Fast_JIT) { + JitGlobals *jit_globals = jit_compiler_get_jit_globals(); + uint32 i; + + /* Allocate memory for fast_jit_func_ptrs if needed */ + if (!module_inst->fast_jit_func_ptrs + || module_inst->fast_jit_func_ptrs == module->fast_jit_func_ptrs) { + uint64 total_size = (uint64)sizeof(void *) * module->function_count; + if (!(module_inst->fast_jit_func_ptrs = + runtime_malloc(total_size, NULL, 0))) { + os_mutex_unlock(&module->instance_list_lock); + return false; + } + } + + for (i = 0; i < module->function_count; i++) { + if (module->functions[i]->fast_jit_jitted_code) { + /* current fast jit function has been compiled */ + module_inst->fast_jit_func_ptrs[i] = + module->functions[i]->fast_jit_jitted_code; + } + else { + module_inst->fast_jit_func_ptrs[i] = + jit_globals->compile_fast_jit_and_then_call; + } + } + } +#endif +#if WASM_ENABLE_JIT != 0 + else if (running_mode == Mode_LLVM_JIT) { + void **llvm_jit_func_ptrs; + uint32 i; + + /* Notify backend threads to start llvm jit compilation */ + module->enable_llvm_jit_compilation = true; + + /* Wait until llvm jit finishes initialization */ + os_mutex_lock(&module->tierup_wait_lock); + while (!module->llvm_jit_inited) { + os_cond_reltimedwait(&module->tierup_wait_cond, + &module->tierup_wait_lock, 10000); + if (module->orcjit_stop_compiling) { + /* init_llvm_jit_functions_stage2 failed */ + os_mutex_unlock(&module->tierup_wait_lock); + os_mutex_unlock(&module->instance_list_lock); + return false; + } + } + os_mutex_unlock(&module->tierup_wait_lock); + + llvm_jit_func_ptrs = + module_inst->func_ptrs + module->import_function_count; + for (i = 0; i < module->function_count; i++) { + llvm_jit_func_ptrs[i] = module->functions[i]->llvm_jit_func_ptr; + } + } +#endif + else if (running_mode == Mode_Multi_Tier_JIT) { + /* Notify backend threads to start llvm jit compilation */ + module->enable_llvm_jit_compilation = true; + + /* Free fast_jit_func_ptrs if it is allocated before */ + if (module_inst->fast_jit_func_ptrs + && module_inst->fast_jit_func_ptrs != module->fast_jit_func_ptrs) { + wasm_runtime_free(module_inst->fast_jit_func_ptrs); + } + module_inst->fast_jit_func_ptrs = module->fast_jit_func_ptrs; + + /* Copy all llvm jit func ptrs from the module */ + bh_memcpy_s(module_inst->func_ptrs + module->import_function_count, + sizeof(void *) * module->function_count, module->func_ptrs, + sizeof(void *) * module->function_count); + } + else { + bh_assert(0); + } + + /* Add module instance into module's instance list if not added */ + if (first_time_set) { + bool found = false; + WASMModuleInstance *node = module->instance_list; + + while (node) { + if (node == module_inst) { + found = true; + break; + } + node = node->e->next; + } + + if (!found) { + module_inst->e->next = module->instance_list; + module->instance_list = module_inst; + } + } + + os_mutex_unlock(&module->instance_list_lock); +#endif /* end of !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0) */ + + (void)module; + return true; +} + +bool +wasm_set_running_mode(WASMModuleInstance *module_inst, RunningMode running_mode) +{ + return set_running_mode(module_inst, running_mode, false); +} + +/** + * Instantiate module + */ +WASMModuleInstance * +wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, + WASMExecEnv *exec_env_main, uint32 stack_size, + uint32 heap_size, uint32 max_memory_pages, char *error_buf, + uint32 error_buf_size) +{ + WASMModuleInstance *module_inst; + WASMGlobalInstance *globals = NULL, *global; + WASMTableInstance *first_table; + uint32 global_count, i; + uint32 length, extra_info_offset; + mem_offset_t base_offset; + uint32 module_inst_struct_size = + offsetof(WASMModuleInstance, global_table_data.bytes); + uint64 module_inst_mem_inst_size; + uint64 total_size, table_size = 0; + uint8 *global_data, *global_data_end; +#if WASM_ENABLE_MULTI_MODULE != 0 + bool ret = false; +#endif + const bool is_sub_inst = parent != NULL; + + if (!module) + return NULL; + + /* Check the heap size */ + heap_size = align_uint(heap_size, 8); + if (heap_size > APP_HEAP_SIZE_MAX) + heap_size = APP_HEAP_SIZE_MAX; + + module_inst_mem_inst_size = + sizeof(WASMMemoryInstance) + * ((uint64)module->import_memory_count + module->memory_count); + +#if WASM_ENABLE_JIT != 0 + /* If the module doesn't have memory, reserve one mem_info space + with empty content to align with llvm jit compiler */ + if (module_inst_mem_inst_size == 0) + module_inst_mem_inst_size = (uint64)sizeof(WASMMemoryInstance); +#endif + + /* Size of module inst, memory instances and global data */ + total_size = (uint64)module_inst_struct_size + module_inst_mem_inst_size + + module->global_data_size; + + /* Calculate the size of table data */ + for (i = 0; i < module->import_table_count; i++) { + WASMTableImport *import_table = &module->import_tables[i].u.table; + table_size += offsetof(WASMTableInstance, elems); +#if WASM_ENABLE_MULTI_MODULE != 0 + table_size += (uint64)sizeof(table_elem_type_t) + * import_table->table_type.max_size; +#else + table_size += (uint64)sizeof(table_elem_type_t) + * (import_table->table_type.possible_grow + ? import_table->table_type.max_size + : import_table->table_type.init_size); +#endif + } + for (i = 0; i < module->table_count; i++) { + WASMTable *table = module->tables + i; + table_size += offsetof(WASMTableInstance, elems); +#if WASM_ENABLE_MULTI_MODULE != 0 + table_size += + (uint64)sizeof(table_elem_type_t) * table->table_type.max_size; +#else + table_size += + (uint64)sizeof(table_elem_type_t) + * (table->table_type.possible_grow ? table->table_type.max_size + : table->table_type.init_size); +#endif + } + total_size += table_size; + + /* The offset of WASMModuleInstanceExtra, make it 8-byte aligned */ + total_size = (total_size + 7LL) & ~7LL; + extra_info_offset = (uint32)total_size; + total_size += sizeof(WASMModuleInstanceExtra); + + /* Allocate the memory for module instance with memory instances, + global data, table data appended at the end */ + if (!(module_inst = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + module_inst->module_type = Wasm_Module_Bytecode; + module_inst->module = module; + module_inst->e = + (WASMModuleInstanceExtra *)((uint8 *)module_inst + extra_info_offset); + +#if WASM_ENABLE_MULTI_MODULE != 0 + module_inst->e->sub_module_inst_list = + &module_inst->e->sub_module_inst_list_head; + ret = wasm_runtime_sub_module_instantiate( + (WASMModuleCommon *)module, (WASMModuleInstanceCommon *)module_inst, + stack_size, heap_size, max_memory_pages, error_buf, error_buf_size); + if (!ret) { + LOG_DEBUG("build a sub module list failed"); + goto fail; + } +#endif + +#if WASM_ENABLE_BULK_MEMORY != 0 + if (module->data_seg_count > 0) { + module_inst->e->common.data_dropped = + bh_bitmap_new(0, module->data_seg_count); + if (module_inst->e->common.data_dropped == NULL) { + LOG_DEBUG("failed to allocate bitmaps"); + set_error_buf(error_buf, error_buf_size, + "failed to allocate bitmaps"); + goto fail; + } + for (i = 0; i < module->data_seg_count; i++) { + if (!module->data_segments[i]->is_passive) + bh_bitmap_set_bit(module_inst->e->common.data_dropped, i); + } + } +#endif +#if WASM_ENABLE_REF_TYPES != 0 + if (module->table_seg_count > 0) { + module_inst->e->common.elem_dropped = + bh_bitmap_new(0, module->table_seg_count); + if (module_inst->e->common.elem_dropped == NULL) { + LOG_DEBUG("failed to allocate bitmaps"); + set_error_buf(error_buf, error_buf_size, + "failed to allocate bitmaps"); + goto fail; + } + for (i = 0; i < module->table_seg_count; i++) { + if (wasm_elem_is_active(module->table_segments[i].mode) + || wasm_elem_is_declarative(module->table_segments[i].mode)) + bh_bitmap_set_bit(module_inst->e->common.elem_dropped, i); + } + } +#endif + +#if WASM_ENABLE_GC != 0 + if (!is_sub_inst) { + uint32 gc_heap_size = wasm_runtime_get_gc_heap_size_default(); + + if (gc_heap_size < GC_HEAP_SIZE_MIN) + gc_heap_size = GC_HEAP_SIZE_MIN; + if (gc_heap_size > GC_HEAP_SIZE_MAX) + gc_heap_size = GC_HEAP_SIZE_MAX; + + module_inst->e->common.gc_heap_pool = + runtime_malloc(gc_heap_size, error_buf, error_buf_size); + if (!module_inst->e->common.gc_heap_pool) + goto fail; + + module_inst->e->common.gc_heap_handle = mem_allocator_create( + module_inst->e->common.gc_heap_pool, gc_heap_size); + if (!module_inst->e->common.gc_heap_handle) + goto fail; + } +#endif + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + if (!(module_inst->frames = runtime_malloc((uint64)sizeof(Vector), + error_buf, error_buf_size))) { + goto fail; + } +#endif + + /* Instantiate global firstly to get the mutable data size */ + global_count = module->import_global_count + module->global_count; + if (global_count + && !(globals = globals_instantiate(module, module_inst, error_buf, + error_buf_size))) { + goto fail; + } + module_inst->e->global_count = global_count; + module_inst->e->globals = globals; + module_inst->global_data = (uint8 *)module_inst + module_inst_struct_size + + module_inst_mem_inst_size; + module_inst->global_data_size = module->global_data_size; + first_table = (WASMTableInstance *)(module_inst->global_data + + module->global_data_size); + + module_inst->memory_count = + module->import_memory_count + module->memory_count; + module_inst->table_count = module->import_table_count + module->table_count; + module_inst->e->function_count = + module->import_function_count + module->function_count; +#if WASM_ENABLE_TAGS != 0 + module_inst->e->tag_count = module->import_tag_count + module->tag_count; +#endif + + /* export */ + module_inst->export_func_count = get_export_count(module, EXPORT_KIND_FUNC); +#if WASM_ENABLE_MULTI_MEMORY != 0 + module_inst->export_memory_count = + get_export_count(module, EXPORT_KIND_MEMORY); +#endif +#if WASM_ENABLE_MULTI_MODULE != 0 + module_inst->export_table_count = + get_export_count(module, EXPORT_KIND_TABLE); +#if WASM_ENABLE_TAGS != 0 + module_inst->e->export_tag_count = + get_export_count(module, EXPORT_KIND_TAG); +#endif + module_inst->export_global_count = + get_export_count(module, EXPORT_KIND_GLOBAL); +#endif + + /* Instantiate memories/tables/functions/tags */ + if ((module_inst->memory_count > 0 + && !(module_inst->memories = memories_instantiate( + module, module_inst, parent, heap_size, max_memory_pages, + error_buf, error_buf_size))) + || (module_inst->table_count > 0 + && !(module_inst->tables = + tables_instantiate(module, module_inst, first_table, + error_buf, error_buf_size))) + || (module_inst->e->function_count > 0 + && !(module_inst->e->functions = functions_instantiate( + module, module_inst, error_buf, error_buf_size))) + || (module_inst->export_func_count > 0 + && !(module_inst->export_functions = export_functions_instantiate( + module, module_inst, module_inst->export_func_count, + error_buf, error_buf_size))) +#if WASM_ENABLE_TAGS != 0 + || (module_inst->e->tag_count > 0 + && !(module_inst->e->tags = tags_instantiate( + module, module_inst, error_buf, error_buf_size))) + || (module_inst->e->export_tag_count > 0 + && !(module_inst->e->export_tags = export_tags_instantiate( + module, module_inst, module_inst->e->export_tag_count, + error_buf, error_buf_size))) +#endif +#if WASM_ENABLE_MULTI_MODULE != 0 + || (module_inst->export_global_count > 0 + && !(module_inst->export_globals = export_globals_instantiate( + module, module_inst, module_inst->export_global_count, + error_buf, error_buf_size))) +#endif +#if WASM_ENABLE_MULTI_MEMORY != 0 + || (module_inst->export_memory_count > 0 + && !(module_inst->export_memories = export_memories_instantiate( + module, module_inst, module_inst->export_memory_count, + error_buf, error_buf_size))) +#endif +#if WASM_ENABLE_JIT != 0 + || (module_inst->e->function_count > 0 + && !init_func_ptrs(module_inst, module, error_buf, error_buf_size)) +#endif +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 + || (module_inst->e->function_count > 0 + && !init_func_type_indexes(module_inst, error_buf, error_buf_size)) +#endif + ) { + goto fail; + } + if (global_count > 0) { + /* Initialize the global data */ + global_data = module_inst->global_data; + global_data_end = global_data + module->global_data_size; + global = globals; + for (i = 0; i < global_count; i++, global++) { + switch (global->type) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + *(int32 *)global_data = global->initial_value.i32; + global_data += sizeof(int32); + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + bh_memcpy_s(global_data, + (uint32)(global_data_end - global_data), + &global->initial_value.i64, sizeof(int64)); + global_data += sizeof(int64); + break; +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + bh_memcpy_s(global_data, (uint32)sizeof(V128), + &global->initial_value.v128, sizeof(V128)); + global_data += sizeof(V128); + break; +#endif +#if WASM_ENABLE_GC != 0 + case VALUE_TYPE_EXTERNREF: + /* the initial value should be a null reference */ + bh_assert(global->initial_value.gc_obj == NULL_REF); + STORE_PTR((void **)global_data, NULL_REF); + global_data += sizeof(void *); + break; +#endif + default: + { +#if WASM_ENABLE_GC != 0 + InitializerExpression *global_init = NULL; + bh_assert(wasm_is_type_reftype(global->type)); + + if (i >= module->import_global_count) { + global_init = + &module->globals[i - module->import_global_count] + .init_expr; + } + + if (global->type == REF_TYPE_NULLFUNCREF + || global->type == REF_TYPE_NULLEXTERNREF + || global->type == REF_TYPE_NULLREF) { + STORE_PTR((void **)global_data, NULL_REF); + global_data += sizeof(void *); + break; + } + + /* We can't create funcref obj during global instantiation + * since the functions are not instantiated yet, so we need + * to defer the initialization here */ + if (global_init + && (global_init->init_expr_type + == INIT_EXPR_TYPE_FUNCREF_CONST) + && wasm_reftype_is_subtype_of( + global->type, global->ref_type, REF_TYPE_FUNCREF, + NULL, module_inst->module->types, + module_inst->module->type_count)) { + WASMFuncObjectRef func_obj = NULL; + /* UINT32_MAX indicates that it is a null reference */ + if ((uint32)global->initial_value.i32 != UINT32_MAX) { + if (!(func_obj = wasm_create_func_obj( + module_inst, global->initial_value.i32, + false, error_buf, error_buf_size))) + goto fail; + } + STORE_PTR((void **)global_data, func_obj); + global_data += sizeof(void *); + /* Also update the initial_value since other globals may + * refer to this */ + global->initial_value.gc_obj = (wasm_obj_t)func_obj; + break; + } + else { + STORE_PTR((void **)global_data, + global->initial_value.gc_obj); + global_data += sizeof(void *); + break; + } +#endif + bh_assert(0); + break; + } + } + } + bh_assert(global_data == global_data_end); + } + + if (!check_linked_symbol(module_inst, error_buf, error_buf_size)) { + goto fail; + } + + /* Initialize the memory data with data segment section */ + for (i = 0; i < module->data_seg_count; i++) { + WASMMemoryInstance *memory = NULL; + uint8 *memory_data = NULL; + uint64 memory_size = 0; + WASMDataSeg *data_seg = module->data_segments[i]; + WASMValue offset_value; + +#if WASM_ENABLE_BULK_MEMORY != 0 + if (data_seg->is_passive) + continue; +#endif + if (is_sub_inst) + /* Ignore setting memory init data if the memory has been + initialized */ + continue; + + /* has check it in loader */ + memory = module_inst->memories[data_seg->memory_index]; + bh_assert(memory); + + memory_data = memory->memory_data; + memory_size = + (uint64)memory->num_bytes_per_page * memory->cur_page_count; + bh_assert(memory_data || memory_size == 0); + + uint8 offset_flag = data_seg->base_offset.init_expr_type; + bh_assert(offset_flag == INIT_EXPR_TYPE_GET_GLOBAL + || (memory->is_memory64 ? is_valid_i64_offset(offset_flag) + : is_valid_i32_offset(offset_flag))); + + if (!get_init_value_recursive(module, &data_seg->base_offset, globals, + &offset_value, error_buf, + error_buf_size)) { + goto fail; + } + + if (offset_flag == INIT_EXPR_TYPE_GET_GLOBAL) { + if (!globals + || globals[data_seg->base_offset.u.unary.v.global_index].type + != (memory->is_memory64 ? VALUE_TYPE_I64 + : VALUE_TYPE_I32)) { + set_error_buf(error_buf, error_buf_size, + "data segment does not fit"); + goto fail; + } + } + +#if WASM_ENABLE_MEMORY64 != 0 + if (memory->is_memory64) { + base_offset = (uint64)offset_value.i64; + } + else +#endif + { + base_offset = (uint32)offset_value.i32; + } + /* check offset */ + if (base_offset > memory_size) { +#if WASM_ENABLE_MEMORY64 != 0 + LOG_DEBUG("base_offset(%" PRIu64 ") > memory_size(%" PRIu64 ")", + base_offset, memory_size); +#else + LOG_DEBUG("base_offset(%u) > memory_size(%" PRIu64 ")", base_offset, + memory_size); +#endif +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + set_error_buf(error_buf, error_buf_size, + "out of bounds memory access"); +#else + set_error_buf(error_buf, error_buf_size, + "data segment does not fit"); +#endif + goto fail; + } + + /* check offset + length(could be zero) */ + length = data_seg->data_length; + if ((uint64)base_offset + length > memory_size) { +#if WASM_ENABLE_MEMORY64 != 0 + LOG_DEBUG("base_offset(%" PRIu64 + ") + length(%d) > memory_size(%" PRIu64 ")", + base_offset, length, memory_size); +#else + LOG_DEBUG("base_offset(%u) + length(%d) > memory_size(%" PRIu64 ")", + base_offset, length, memory_size); +#endif +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + set_error_buf(error_buf, error_buf_size, + "out of bounds memory access"); +#else + set_error_buf(error_buf, error_buf_size, + "data segment does not fit"); +#endif + goto fail; + } + + if (memory_data) { + bh_memcpy_s(memory_data + base_offset, + (uint32)(memory_size - base_offset), data_seg->data, + length); + } + } + +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 +#if UINTPTR_MAX == UINT64_MAX + module_inst->e->shared_heap_start_off.u64 = UINT64_MAX; +#else + module_inst->e->shared_heap_start_off.u32[0] = UINT32_MAX; +#endif + module_inst->e->shared_heap = NULL; +#endif + +#if WASM_ENABLE_GC != 0 + /* Initialize the table data with init expr */ + for (i = 0; i < module->table_count; i++) { + WASMTable *table = module->tables + i; + WASMTableInstance *table_inst = module_inst->tables[i]; + table_elem_type_t *table_data; + uint32 j; + + if (table->init_expr.init_expr_type == 0) { + /* No table initializer */ + continue; + } + + table_data = table_inst->elems; + + bh_assert( + table->init_expr.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL + || table->init_expr.init_expr_type == INIT_EXPR_TYPE_FUNCREF_CONST + || table->init_expr.init_expr_type == INIT_EXPR_TYPE_REFNULL_CONST); + + if (table->init_expr.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { + if (!check_global_init_expr(module, + table->init_expr.u.unary.v.global_index, + error_buf, error_buf_size)) { + goto fail; + } + + table->init_expr.u.unary.v.gc_obj = + globals[table->init_expr.u.unary.v.global_index] + .initial_value.gc_obj; + } + else if (table->init_expr.init_expr_type + == INIT_EXPR_TYPE_FUNCREF_CONST) { + uint32 func_idx = table->init_expr.u.unary.v.ref_index; + if (func_idx != UINT32_MAX) { + if (!(table->init_expr.u.unary.v.gc_obj = + wasm_create_func_obj(module_inst, func_idx, false, + error_buf, error_buf_size))) + goto fail; + } + else { + table->init_expr.u.unary.v.gc_obj = NULL_REF; + } + } + else if (table->init_expr.init_expr_type + == INIT_EXPR_TYPE_REFNULL_CONST) { + table->init_expr.u.unary.v.gc_obj = NULL_REF; + } + + LOG_DEBUG("Init table [%d] elements from [%d] to [%d] as: %p", i, 0, + table_inst->cur_size, + (void *)table->init_expr.u.unary.v.gc_obj); + for (j = 0; j < table_inst->cur_size; j++) { + *(table_data + j) = table->init_expr.u.unary.v.gc_obj; + } + } +#endif /* end of WASM_ENABLE_GC != 0 */ + + /* Initialize the table data with table segment section */ + for (i = 0; module_inst->table_count > 0 && i < module->table_seg_count; + i++) { + WASMTableSeg *table_seg = module->table_segments + i; + /* has check it in loader */ + WASMTableInstance *table = module_inst->tables[table_seg->table_index]; + table_elem_type_t *table_data; + WASMValue offset_value; + uint32 j; +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + uint8 tbl_elem_type; + uint32 tbl_init_size, tbl_max_size; +#endif +#if WASM_ENABLE_GC != 0 + WASMRefType *tbl_elem_ref_type; +#endif + + bh_assert(table); + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + (void)wasm_runtime_get_table_inst_elem_type( + (WASMModuleInstanceCommon *)module_inst, table_seg->table_index, + &tbl_elem_type, +#if WASM_ENABLE_GC != 0 + &tbl_elem_ref_type, +#endif + &tbl_init_size, &tbl_max_size); + +#if WASM_ENABLE_GC == 0 + if (tbl_elem_type != VALUE_TYPE_FUNCREF + && tbl_elem_type != VALUE_TYPE_EXTERNREF) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: elements segment does not fit"); + goto fail; + } +#elif WASM_ENABLE_GC != 0 + if (!wasm_elem_is_declarative(table_seg->mode) + && !wasm_reftype_is_subtype_of( + table_seg->elem_type, table_seg->elem_ref_type, + table->elem_type, table->elem_ref_type.elem_ref_type, + module->types, module->type_count)) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: elements segment does not fit"); + goto fail; + } +#endif + (void)tbl_init_size; + (void)tbl_max_size; +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + + table_data = table->elems; +#if WASM_ENABLE_MULTI_MODULE != 0 + if (table_seg->table_index < module->import_table_count + && module_inst->e->table_insts_linked[table_seg->table_index]) { + table_data = + module_inst->e->table_insts_linked[table_seg->table_index] + ->elems; + } +#endif + bh_assert(table_data); + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + if (!wasm_elem_is_active(table_seg->mode)) + continue; +#endif + + uint8 offset_flag = table_seg->base_offset.init_expr_type; +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + bh_assert(offset_flag == INIT_EXPR_TYPE_GET_GLOBAL + || offset_flag == INIT_EXPR_TYPE_FUNCREF_CONST + || offset_flag == INIT_EXPR_TYPE_REFNULL_CONST + || is_valid_i32_offset(offset_flag)); +#else + bh_assert(offset_flag == INIT_EXPR_TYPE_GET_GLOBAL + || is_valid_i32_offset(offset_flag)); +#endif + + if (!get_init_value_recursive(module, &table_seg->base_offset, globals, + &offset_value, error_buf, + error_buf_size)) { + goto fail; + } + + if (offset_flag == INIT_EXPR_TYPE_GET_GLOBAL) { + if (!globals + || globals[table_seg->base_offset.u.unary.v.global_index].type + != VALUE_TYPE_I32) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: elements segment does not fit"); + goto fail; + } + } + + /* check offset since length might negative */ + if ((uint32)offset_value.i32 > table->cur_size) { + LOG_DEBUG("base_offset(%d) > table->cur_size(%d)", offset_value.i32, + table->cur_size); +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + set_error_buf(error_buf, error_buf_size, + "out of bounds table access"); +#else + set_error_buf(error_buf, error_buf_size, + "type mismatch: elements segment does not fit"); +#endif + goto fail; + } + + /* check offset + length(could be zero) */ + length = table_seg->value_count; + if ((uint32)offset_value.i32 + length > table->cur_size) { + LOG_DEBUG("base_offset(%d) + length(%d)> table->cur_size(%d)", + offset_value.i32, length, table->cur_size); +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 + set_error_buf(error_buf, error_buf_size, + "out of bounds table access"); +#else + set_error_buf(error_buf, error_buf_size, + "type mismatch: elements segment does not fit"); +#endif + goto fail; + } + + for (j = 0; j < length; j++) { + InitializerExpression *init_expr = &table_seg->init_values[j]; + uint8 flag = init_expr->init_expr_type; + void *ref = NULL; + + /* const and get global init values should be resolved during + * loading */ + bh_assert((flag == INIT_EXPR_TYPE_GET_GLOBAL) + || (flag == INIT_EXPR_TYPE_REFNULL_CONST) + || ((flag >= INIT_EXPR_TYPE_FUNCREF_CONST) + && (flag <= INIT_EXPR_TYPE_EXTERN_CONVERT_ANY))); + + switch (flag) { + case INIT_EXPR_TYPE_REFNULL_CONST: + ref = NULL; + break; + case INIT_EXPR_TYPE_FUNCREF_CONST: + { +#if WASM_ENABLE_GC == 0 + ref = (void *)(uintptr_t)init_expr->u.unary.v.ref_index; +#else + WASMFuncObjectRef func_obj; + uint32 func_idx = init_expr->u.unary.v.ref_index; + /* UINT32_MAX indicates that it is a null reference */ + if (func_idx != UINT32_MAX) { + if (!(func_obj = wasm_create_func_obj( + module_inst, func_idx, false, error_buf, + error_buf_size))) { + goto fail; + } + ref = func_obj; + } + else { + ref = NULL_REF; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + break; + } +#if WASM_ENABLE_GC != 0 + case INIT_EXPR_TYPE_GET_GLOBAL: + { + if (!check_global_init_expr( + module, init_expr->u.unary.v.global_index, + error_buf, error_buf_size)) { + goto fail; + } + + ref = globals[init_expr->u.unary.v.global_index] + .initial_value.gc_obj; + break; + } + case INIT_EXPR_TYPE_STRUCT_NEW: + case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT: + { + WASMRttType *rtt_type; + WASMStructObjectRef struct_obj; + WASMStructType *struct_type; + WASMStructNewInitValues *init_values = NULL; + uint32 type_idx; + + if (flag == INIT_EXPR_TYPE_STRUCT_NEW) { + init_values = (WASMStructNewInitValues *) + init_expr->u.unary.v.data; + type_idx = init_values->type_idx; + } + else { + type_idx = init_expr->u.unary.v.type_index; + } + + struct_type = (WASMStructType *)module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)struct_type, type_idx, + module->rtt_types, module->type_count, + &module->rtt_type_lock))) { + set_error_buf(error_buf, error_buf_size, + "create rtt object failed"); + goto fail; + } + + if (!(struct_obj = wasm_struct_obj_new_internal( + module_inst->e->common.gc_heap_handle, + rtt_type))) { + set_error_buf(error_buf, error_buf_size, + "create struct object failed"); + goto fail; + } + + if (flag == INIT_EXPR_TYPE_STRUCT_NEW) { + uint32 field_idx; + + bh_assert(init_values->count + == struct_type->field_count); + + for (field_idx = 0; field_idx < init_values->count; + field_idx++) { + wasm_struct_obj_set_field( + struct_obj, field_idx, + &init_values->fields[field_idx]); + } + } + + ref = struct_obj; + break; + } + case INIT_EXPR_TYPE_ARRAY_NEW: + case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT: + case INIT_EXPR_TYPE_ARRAY_NEW_FIXED: + { + WASMRttType *rtt_type; + WASMArrayObjectRef array_obj; + WASMArrayType *array_type; + WASMArrayNewInitValues *init_values = NULL; + WASMValue *arr_init_val = NULL, empty_val = { 0 }; + uint32 type_idx, len; + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT) { + type_idx = + init_expr->u.unary.v.array_new_default.type_index; + len = init_expr->u.unary.v.array_new_default.length; + arr_init_val = &empty_val; + } + else { + init_values = + (WASMArrayNewInitValues *)init_expr->u.unary.v.data; + type_idx = init_values->type_idx; + len = init_values->length; + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) { + arr_init_val = init_values->elem_data; + } + } + + array_type = (WASMArrayType *)module->types[type_idx]; + + if (!(rtt_type = wasm_rtt_type_new( + (WASMType *)array_type, type_idx, + module->rtt_types, module->type_count, + &module->rtt_type_lock))) { + set_error_buf(error_buf, error_buf_size, + "create rtt object failed"); + goto fail; + } + + if (!(array_obj = wasm_array_obj_new_internal( + module_inst->e->common.gc_heap_handle, rtt_type, + len, arr_init_val))) { + set_error_buf(error_buf, error_buf_size, + "create array object failed"); + goto fail; + } + + if (flag == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) { + uint32 elem_idx; + + bh_assert(init_values); + + for (elem_idx = 0; elem_idx < len; elem_idx++) { + wasm_array_obj_set_elem( + array_obj, elem_idx, + &init_values->elem_data[elem_idx]); + } + } + + ref = array_obj; + + break; + } + case INIT_EXPR_TYPE_I31_NEW: + { + ref = + (wasm_obj_t)wasm_i31_obj_new(init_expr->u.unary.v.i32); + break; + } +#endif /* end of WASM_ENABLE_GC != 0 */ + } + + *(table_data + offset_value.i32 + j) = (table_elem_type_t)ref; + } + } + + /* Initialize the thread related data */ + if (stack_size == 0) + stack_size = DEFAULT_WASM_STACK_SIZE; + + module_inst->default_wasm_stack_size = stack_size; + + if (module->malloc_function != (uint32)-1) { + module_inst->e->malloc_function = + &module_inst->e->functions[module->malloc_function]; + } + + if (module->free_function != (uint32)-1) { + module_inst->e->free_function = + &module_inst->e->functions[module->free_function]; + } + + if (module->retain_function != (uint32)-1) { + module_inst->e->retain_function = + &module_inst->e->functions[module->retain_function]; + } + +#if WASM_ENABLE_LIBC_WASI != 0 + /* The sub-instance will get the wasi_ctx from main-instance */ + if (!is_sub_inst) { + if (!wasm_runtime_init_wasi( + (WASMModuleInstanceCommon *)module_inst, + module->wasi_args.dir_list, module->wasi_args.dir_count, + module->wasi_args.map_dir_list, module->wasi_args.map_dir_count, + module->wasi_args.env, module->wasi_args.env_count, + module->wasi_args.addr_pool, module->wasi_args.addr_count, + module->wasi_args.ns_lookup_pool, + module->wasi_args.ns_lookup_count, module->wasi_args.argv, + module->wasi_args.argc, module->wasi_args.stdio[0], + module->wasi_args.stdio[1], module->wasi_args.stdio[2], + error_buf, error_buf_size)) { + goto fail; + } + } +#endif + +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (!is_sub_inst) { + /* Add module instance into module's instance list */ + os_mutex_lock(&module->instance_list_lock); + if (module->instance_list) { + LOG_WARNING( + "warning: multiple instances referencing to the same module " + "may cause unexpected behaviour during debugging"); + } + module_inst->e->next = module->instance_list; + module->instance_list = module_inst; + os_mutex_unlock(&module->instance_list_lock); + } +#endif + + /* Set running mode before executing wasm functions */ + if (!set_running_mode(module_inst, wasm_runtime_get_default_running_mode(), + true)) { + set_error_buf(error_buf, error_buf_size, + "set instance running mode failed"); + goto fail; + } + + if (module->start_function != (uint32)-1) { + /* TODO: fix start function can be import function issue */ + if (module->start_function >= module->import_function_count) + module_inst->e->start_function = + &module_inst->e->functions[module->start_function]; + } + + if (!execute_post_instantiate_functions(module_inst, is_sub_inst, + exec_env_main)) { + set_error_buf(error_buf, error_buf_size, module_inst->cur_exception); + goto fail; + } + +#if WASM_ENABLE_MEMORY_TRACING != 0 + wasm_runtime_dump_module_inst_mem_consumption( + (WASMModuleInstanceCommon *)module_inst); +#endif + + (void)global_data_end; + return module_inst; + +fail: + wasm_deinstantiate(module_inst, false); + return NULL; +} + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 +static void +destroy_c_api_frames(Vector *frames) +{ + WASMCApiFrame frame = { 0 }; + uint32 i, total_frames, ret; + + total_frames = (uint32)bh_vector_size(frames); + + for (i = 0; i < total_frames; i++) { + ret = bh_vector_get(frames, i, &frame); + bh_assert(ret); + + if (frame.lp) + wasm_runtime_free(frame.lp); + } + + ret = bh_vector_destroy(frames); + bh_assert(ret); + (void)ret; +} +#endif + +void +wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst) +{ + if (!module_inst) + return; + + if (module_inst->exec_env_singleton) { + /* wasm_exec_env_destroy will call + wasm_cluster_wait_for_all_except_self to wait for other + threads, so as to destroy their exec_envs and module + instances first, and avoid accessing the shared resources + of current module instance after it is deinstantiated. */ + wasm_exec_env_destroy(module_inst->exec_env_singleton); + } + +#if WASM_ENABLE_DEBUG_INTERP != 0 \ + || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0) + /* Remove instance from module's instance list before freeing + func_ptrs and fast_jit_func_ptrs of the instance, to avoid + accessing the freed memory in the jit backend compilation + threads */ + { + WASMModule *module = module_inst->module; + WASMModuleInstance *instance_prev = NULL, *instance; + os_mutex_lock(&module->instance_list_lock); + + instance = module->instance_list; + while (instance) { + if (instance == module_inst) { + if (!instance_prev) + module->instance_list = instance->e->next; + else + instance_prev->e->next = instance->e->next; + break; + } + instance_prev = instance; + instance = instance->e->next; + } + + os_mutex_unlock(&module->instance_list_lock); + } +#endif + +#if WASM_ENABLE_JIT != 0 + if (module_inst->func_ptrs) + wasm_runtime_free(module_inst->func_ptrs); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + if (module_inst->fast_jit_func_ptrs + && module_inst->fast_jit_func_ptrs + != module_inst->module->fast_jit_func_ptrs) + wasm_runtime_free(module_inst->fast_jit_func_ptrs); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 + if (module_inst->func_type_indexes) + wasm_runtime_free(module_inst->func_type_indexes); +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + wasm_runtime_sub_module_deinstantiate( + (WASMModuleInstanceCommon *)module_inst); +#endif + + if (module_inst->memory_count > 0) + memories_deinstantiate(module_inst, module_inst->memories, + module_inst->memory_count); + + if (module_inst->import_func_ptrs) { + wasm_runtime_free(module_inst->import_func_ptrs); + } + + tables_deinstantiate(module_inst); + functions_deinstantiate(module_inst->e->functions); +#if WASM_ENABLE_TAGS != 0 + tags_deinstantiate(module_inst->e->tags, module_inst->e->import_tag_ptrs); +#endif + globals_deinstantiate(module_inst->e->globals); + export_functions_deinstantiate(module_inst->export_functions); +#if WASM_ENABLE_TAGS != 0 + export_tags_deinstantiate(module_inst->e->export_tags); +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 + export_globals_deinstantiate(module_inst->export_globals); +#endif + +#if WASM_ENABLE_MULTI_MEMORY != 0 + export_memories_deinstantiate(module_inst->export_memories); +#endif + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + wasm_externref_cleanup((WASMModuleInstanceCommon *)module_inst); +#endif + +#if WASM_ENABLE_GC != 0 + if (!is_sub_inst) { + if (module_inst->e->common.gc_heap_handle) + mem_allocator_destroy(module_inst->e->common.gc_heap_handle); + if (module_inst->e->common.gc_heap_pool) + wasm_runtime_free(module_inst->e->common.gc_heap_pool); + } +#endif + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + if (module_inst->frames) { + destroy_c_api_frames(module_inst->frames); + wasm_runtime_free(module_inst->frames); + module_inst->frames = NULL; + } +#endif + + if (module_inst->c_api_func_imports) + wasm_runtime_free(module_inst->c_api_func_imports); + + if (!is_sub_inst) { + wasm_native_call_context_dtors((WASMModuleInstanceCommon *)module_inst); + } + +#if WASM_ENABLE_BULK_MEMORY != 0 + bh_bitmap_delete(module_inst->e->common.data_dropped); +#endif +#if WASM_ENABLE_REF_TYPES != 0 + bh_bitmap_delete(module_inst->e->common.elem_dropped); +#endif + + wasm_runtime_free(module_inst); +} + +WASMFunctionInstance * +wasm_lookup_function(const WASMModuleInstance *module_inst, const char *name) +{ + WASMExportFuncInstance key = { .name = (char *)name }; + WASMExportFuncInstance *export_func_inst; + + if (!module_inst->export_functions) + return NULL; + + export_func_inst = bsearch( + &key, module_inst->export_functions, module_inst->export_func_count, + sizeof(WASMExportFuncInstance), cmp_export_func_inst); + + if (!export_func_inst) + return NULL; + + return export_func_inst->function; +} + +WASMMemoryInstance * +wasm_lookup_memory(const WASMModuleInstance *module_inst, const char *name) +{ +#if WASM_ENABLE_MULTI_MEMORY != 0 + uint32 i; + for (i = 0; i < module_inst->export_memory_count; i++) + if (!strcmp(module_inst->export_memories[i].name, name)) + return module_inst->export_memories[i].memory; + return NULL; +#else + (void)module_inst->export_memories; + if (!module_inst->memories) + return NULL; + return module_inst->memories[0]; +#endif +} + +#if WASM_ENABLE_MULTI_MODULE != 0 +WASMGlobalInstance * +wasm_lookup_global(const WASMModuleInstance *module_inst, const char *name) +{ + uint32 i; + for (i = 0; i < module_inst->export_global_count; i++) + if (!strcmp(module_inst->export_globals[i].name, name)) + return module_inst->export_globals[i].global; + return NULL; +} + +WASMTableInstance * +wasm_lookup_table(const WASMModuleInstance *module_inst, const char *name) +{ + /** + * using a strong assumption that one module instance only has + * one table instance + */ + (void)module_inst->export_tables; + return module_inst->tables[0]; +} + +#if WASM_ENABLE_TAGS != 0 +WASMTagInstance * +wasm_lookup_tag(const WASMModuleInstance *module_inst, const char *name, + const char *signature) +{ + uint32 i; + for (i = 0; i < module_inst->e->export_tag_count; i++) + if (!strcmp(module_inst->e->export_tags[i].name, name)) + return module_inst->e->export_tags[i].tag; + (void)signature; + return NULL; +} +#endif + +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK +static void +call_wasm_with_hw_bound_check(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, + WASMFunctionInstance *function, unsigned argc, + uint32 argv[]) +{ + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); + WASMJmpBuf jmpbuf_node = { 0 }, *jmpbuf_node_pop; + WASMRuntimeFrame *prev_frame = wasm_exec_env_get_cur_frame(exec_env); + uint8 *prev_top = exec_env->wasm_stack.top; +#ifdef BH_PLATFORM_WINDOWS + int result; + bool has_exception; + char exception[EXCEPTION_BUF_LEN]; +#endif + bool ret = true; + + if (!exec_env_tls) { + if (!os_thread_signal_inited()) { + wasm_set_exception(module_inst, "thread signal env not inited"); + return; + } + + /* Set thread handle and stack boundary if they haven't been set */ + wasm_exec_env_set_thread_info(exec_env); + + wasm_runtime_set_exec_env_tls(exec_env); + } + else { + if (exec_env_tls != exec_env) { + wasm_set_exception(module_inst, "invalid exec env"); + return; + } + } + + /* Check native stack overflow firstly to ensure we have enough + native stack to run the following codes before actually calling + the aot function in invokeNative function. */ + if (!wasm_runtime_detect_native_stack_overflow(exec_env)) { + return; + } + + wasm_exec_env_push_jmpbuf(exec_env, &jmpbuf_node); + + if (os_setjmp(jmpbuf_node.jmpbuf) == 0) { +#ifndef BH_PLATFORM_WINDOWS + wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv); +#else + __try { + wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv); + } __except (wasm_copy_exception(module_inst, NULL) + ? EXCEPTION_EXECUTE_HANDLER + : EXCEPTION_CONTINUE_SEARCH) { + /* Exception was thrown in wasm_exception_handler */ + ret = false; + } + has_exception = wasm_copy_exception(module_inst, exception); + if (has_exception && strstr(exception, "native stack overflow")) { + /* After a stack overflow, the stack was left + in a damaged state, let the CRT repair it */ + result = _resetstkoflw(); + bh_assert(result != 0); + } +#endif + } + else { + /* Exception has been set in signal handler before calling longjmp */ + ret = false; + } + + /* Note: can't check wasm_get_exception(module_inst) here, there may be + * exception which is not caught by hardware (e.g. uninitialized elements), + * then the stack-frame is already freed inside wasm_interp_call_wasm */ + if (!ret) { +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + if (wasm_interp_create_call_stack(exec_env)) { + wasm_interp_dump_call_stack(exec_env, true, NULL, 0); + } +#endif + /* Restore operand frames */ + wasm_exec_env_set_cur_frame(exec_env, prev_frame); + exec_env->wasm_stack.top = prev_top; + } + + jmpbuf_node_pop = wasm_exec_env_pop_jmpbuf(exec_env); + bh_assert(&jmpbuf_node == jmpbuf_node_pop); + if (!exec_env->jmpbuf_stack_top) { + wasm_runtime_set_exec_env_tls(NULL); + } + if (!ret) { + os_sigreturn(); + os_signal_unmask(); + } + (void)jmpbuf_node_pop; +} +#define interp_call_wasm call_wasm_with_hw_bound_check +#else +#define interp_call_wasm wasm_interp_call_wasm +#endif + +bool +wasm_call_function(WASMExecEnv *exec_env, WASMFunctionInstance *function, + unsigned argc, uint32 argv[]) +{ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)exec_env->module_inst; + +#ifndef OS_ENABLE_HW_BOUND_CHECK + /* Set thread handle and stack boundary */ + wasm_exec_env_set_thread_info(exec_env); +#else + /* Set thread info in call_wasm_with_hw_bound_check when + hw bound check is enabled */ +#endif + + /* Set exec env, so it can be later retrieved from instance */ + module_inst->cur_exec_env = exec_env; + + interp_call_wasm(module_inst, exec_env, function, argc, argv); + return !wasm_copy_exception(module_inst, NULL); +} + +#if WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0 +/* look for the function name */ +static char * +get_func_name_from_index(const WASMModuleInstance *inst, uint32 func_index) +{ + char *func_name = NULL; + WASMFunctionInstance *func_inst = inst->e->functions + func_index; + + if (func_inst->is_import_func) { + func_name = func_inst->u.func_import->field_name; + } + else { +#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0 + func_name = func_inst->u.func->field_name; +#endif + /* if custom name section is not generated, + search symbols from export table */ + if (!func_name) { + unsigned j; + for (j = 0; j < inst->export_func_count; j++) { + WASMExportFuncInstance *export_func = + inst->export_functions + j; + if (export_func->function == func_inst) { + func_name = export_func->name; + break; + } + } + } + } + + return func_name; +} +#endif /*WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0*/ + +#if WASM_ENABLE_PERF_PROFILING != 0 +void +wasm_dump_perf_profiling(const WASMModuleInstance *module_inst) +{ + WASMFunctionInstance *func_inst; + char *func_name; + uint32 i; + + os_printf("Performance profiler data:\n"); + for (i = 0; i < module_inst->e->function_count; i++) { + func_inst = module_inst->e->functions + i; + + if (func_inst->total_exec_cnt == 0) + continue; + + func_name = get_func_name_from_index(module_inst, i); + if (func_name) + os_printf( + " func %s, execution time: %.3f ms, execution count: %" PRIu32 + " times, children execution time: %.3f ms\n", + func_name, func_inst->total_exec_time / 1000.0, + func_inst->total_exec_cnt, + func_inst->children_exec_time / 1000.0); + else + os_printf(" func %" PRIu32 + ", execution time: %.3f ms, execution count: %" PRIu32 + " times, children execution time: %.3f ms\n", + i, func_inst->total_exec_time / 1000.0, + func_inst->total_exec_cnt, + func_inst->children_exec_time / 1000.0); + } +} + +double +wasm_summarize_wasm_execute_time(const WASMModuleInstance *inst) +{ + double ret = 0; + + unsigned i; + for (i = 0; i < inst->e->function_count; i++) { + WASMFunctionInstance *func = inst->e->functions + i; + ret += (func->total_exec_time - func->children_exec_time) / 1000.0; + } + + return ret; +} + +double +wasm_get_wasm_func_exec_time(const WASMModuleInstance *inst, + const char *func_name) +{ + unsigned i; + for (i = 0; i < inst->e->function_count; i++) { + char *name_in_wasm = get_func_name_from_index(inst, i); + if (name_in_wasm && strcmp(name_in_wasm, func_name) == 0) { + WASMFunctionInstance *func = inst->e->functions + i; + return (func->total_exec_time - func->children_exec_time) / 1000.0; + } + } + + return -1.0; +} +#endif /*WASM_ENABLE_PERF_PROFILING != 0*/ + +uint64 +wasm_module_malloc_internal(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, uint64 size, + void **p_native_addr) +{ + WASMMemoryInstance *memory = wasm_get_default_memory(module_inst); + uint8 *addr = NULL; + uint64 offset = 0; + + /* TODO: Memory64 size check based on memory idx type */ + bh_assert(size <= UINT32_MAX); + + if (!memory) { + wasm_set_exception(module_inst, "uninitialized memory"); + return 0; + } + + if (memory->heap_handle) { + addr = mem_allocator_malloc(memory->heap_handle, (uint32)size); + } + else if (module_inst->e->malloc_function && module_inst->e->free_function) { + if (!execute_malloc_function( + module_inst, exec_env, module_inst->e->malloc_function, + module_inst->e->retain_function, size, &offset)) { + return 0; + } + /* If we use app's malloc function, + the default memory may be changed while memory growing */ + memory = wasm_get_default_memory(module_inst); + addr = offset ? memory->memory_data + offset : NULL; + } + + if (!addr) { + if (memory->heap_handle + && mem_allocator_is_heap_corrupted(memory->heap_handle)) { + wasm_runtime_show_app_heap_corrupted_prompt(); + wasm_set_exception(module_inst, "app heap corrupted"); + } + else { + LOG_WARNING("warning: allocate %" PRIu64 " bytes memory failed", + size); + } + return 0; + } + if (p_native_addr) + *p_native_addr = addr; + + return (uint64)(addr - memory->memory_data); +} + +uint64 +wasm_module_realloc_internal(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, uint64 ptr, uint64 size, + void **p_native_addr) +{ + WASMMemoryInstance *memory = wasm_get_default_memory(module_inst); + uint8 *addr = NULL; + + /* TODO: Memory64 ptr and size check based on memory idx type */ + bh_assert(ptr <= UINT32_MAX); + bh_assert(size <= UINT32_MAX); + + if (!memory) { + wasm_set_exception(module_inst, "uninitialized memory"); + return 0; + } + + if (memory->heap_handle) { + addr = mem_allocator_realloc( + memory->heap_handle, + (uint32)ptr ? memory->memory_data + (uint32)ptr : NULL, + (uint32)size); + } + + /* Only support realloc in WAMR's app heap */ + (void)exec_env; + + if (!addr) { + if (memory->heap_handle + && mem_allocator_is_heap_corrupted(memory->heap_handle)) { + wasm_set_exception(module_inst, "app heap corrupted"); + } + else { + wasm_set_exception(module_inst, "out of memory"); + } + return 0; + } + if (p_native_addr) + *p_native_addr = addr; + + return (uint64)(addr - memory->memory_data); +} + +void +wasm_module_free_internal(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, uint64 ptr) +{ + WASMMemoryInstance *memory = wasm_get_default_memory(module_inst); + + /* TODO: Memory64 ptr and size check based on memory idx type */ + bh_assert(ptr <= UINT32_MAX); + + if (!memory) { + return; + } + + if (ptr) { + uint8 *addr = memory->memory_data + (uint32)ptr; + uint8 *memory_data_end; + + /* memory->memory_data_end may be changed in memory grow */ + SHARED_MEMORY_LOCK(memory); + memory_data_end = memory->memory_data_end; + SHARED_MEMORY_UNLOCK(memory); + + if (memory->heap_handle && memory->heap_data <= addr + && addr < memory->heap_data_end) { + mem_allocator_free(memory->heap_handle, addr); + } + else if (module_inst->e->malloc_function + && module_inst->e->free_function && memory->memory_data <= addr + && addr < memory_data_end) { + execute_free_function(module_inst, exec_env, + module_inst->e->free_function, ptr); + } + } +} + +uint64 +wasm_module_malloc(WASMModuleInstance *module_inst, uint64 size, + void **p_native_addr) +{ + return wasm_module_malloc_internal(module_inst, NULL, size, p_native_addr); +} + +uint64 +wasm_module_realloc(WASMModuleInstance *module_inst, uint64 ptr, uint64 size, + void **p_native_addr) +{ + return wasm_module_realloc_internal(module_inst, NULL, ptr, size, + p_native_addr); +} + +void +wasm_module_free(WASMModuleInstance *module_inst, uint64 ptr) +{ + wasm_module_free_internal(module_inst, NULL, ptr); +} + +uint64 +wasm_module_dup_data(WASMModuleInstance *module_inst, const char *src, + uint64 size) +{ + char *buffer; + uint64 buffer_offset; + + /* TODO: Memory64 size check based on memory idx type */ + bh_assert(size <= UINT32_MAX); + + buffer_offset = wasm_module_malloc(module_inst, size, (void **)&buffer); + + if (buffer_offset != 0) { + buffer = wasm_runtime_addr_app_to_native( + (WASMModuleInstanceCommon *)module_inst, buffer_offset); + bh_memcpy_s(buffer, (uint32)size, src, (uint32)size); + } + return buffer_offset; +} + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 +bool +wasm_enlarge_table(WASMModuleInstance *module_inst, uint32 table_idx, + uint32 inc_size, table_elem_type_t init_val) +{ + uint32 total_size, i; + table_elem_type_t *new_table_data_start; + WASMTableInstance *table_inst; + + if (!inc_size) { + return true; + } + + bh_assert(table_idx < module_inst->table_count); + table_inst = wasm_get_table_inst(module_inst, table_idx); + if (!table_inst) { + return false; + } + + if (inc_size > UINT32_MAX - table_inst->cur_size) { + return false; + } + + total_size = table_inst->cur_size + inc_size; + if (total_size > table_inst->max_size) { + return false; + } + + /* fill in */ + new_table_data_start = table_inst->elems + table_inst->cur_size; + for (i = 0; i < inc_size; ++i) { + new_table_data_start[i] = init_val; + } + + table_inst->cur_size = total_size; + return true; +} +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +static bool +call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 tbl_elem_idx, + uint32 argc, uint32 argv[], bool check_type_idx, uint32 type_idx) +{ + WASMModuleInstance *module_inst = NULL; + WASMTableInstance *table_inst = NULL; + table_elem_type_t tbl_elem_val = NULL_REF; + uint32 func_idx = 0; + WASMFunctionInstance *func_inst = NULL; + + module_inst = (WASMModuleInstance *)exec_env->module_inst; + bh_assert(module_inst); + + table_inst = module_inst->tables[tbl_idx]; + if (!table_inst) { + wasm_set_exception(module_inst, "unknown table"); + goto got_exception; + } + + if (tbl_elem_idx >= table_inst->cur_size) { + wasm_set_exception(module_inst, "undefined element"); + goto got_exception; + } + + tbl_elem_val = ((table_elem_type_t *)table_inst->elems)[tbl_elem_idx]; + if (tbl_elem_val == NULL_REF) { + wasm_set_exception(module_inst, "uninitialized element"); + goto got_exception; + } + +#if WASM_ENABLE_GC == 0 + func_idx = (uint32)tbl_elem_val; +#else + func_idx = + wasm_func_obj_get_func_idx_bound((WASMFuncObjectRef)tbl_elem_val); +#endif + + /** + * we insist to call functions owned by the module itself + **/ + if (func_idx >= module_inst->e->function_count) { + wasm_set_exception(module_inst, "unknown function"); + goto got_exception; + } + + func_inst = module_inst->e->functions + func_idx; + + if (check_type_idx) { + WASMType *cur_type = module_inst->module->types[type_idx]; + WASMType *cur_func_type; + + if (func_inst->is_import_func) + cur_func_type = (WASMType *)func_inst->u.func_import->func_type; + else + cur_func_type = (WASMType *)func_inst->u.func->func_type; + + if (cur_type != cur_func_type) { + wasm_set_exception(module_inst, "indirect call type mismatch"); + goto got_exception; + } + } + + interp_call_wasm(module_inst, exec_env, func_inst, argc, argv); + + return !wasm_copy_exception(module_inst, NULL); + +got_exception: + return false; +} + +bool +wasm_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 argc, uint32 argv[]) +{ + return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0); +} + +#if WASM_ENABLE_THREAD_MGR != 0 +bool +wasm_set_aux_stack(WASMExecEnv *exec_env, uint64 start_offset, uint32 size) +{ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)exec_env->module_inst; + uint32 stack_top_idx = module_inst->module->aux_stack_top_global_index; + +#if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION == 0 + /* Check the aux stack space */ + uint64 data_end = module_inst->module->aux_data_end; + uint64 stack_bottom = module_inst->module->aux_stack_bottom; + bool is_stack_before_data = stack_bottom < data_end ? true : false; + if ((is_stack_before_data && (size > start_offset)) + || ((!is_stack_before_data) && (start_offset - data_end < size))) + return false; +#endif + + if (stack_top_idx != (uint32)-1) { + /* The aux stack top is a wasm global, + set the initial value for the global */ + uint8 *global_addr = + module_inst->global_data + + module_inst->e->globals[stack_top_idx].data_offset; + *(int32 *)global_addr = (uint32)start_offset; + /* The aux stack boundary is a constant value, + set the value to exec_env */ + exec_env->aux_stack_boundary = (uintptr_t)start_offset - size; + exec_env->aux_stack_bottom = (uintptr_t)start_offset; + return true; + } + + return false; +} + +bool +wasm_get_aux_stack(WASMExecEnv *exec_env, uint64 *start_offset, uint32 *size) +{ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)exec_env->module_inst; + + /* The aux stack information is resolved in loader + and store in module */ + uint64 stack_bottom = module_inst->module->aux_stack_bottom; + uint32 total_aux_stack_size = module_inst->module->aux_stack_size; + + if (stack_bottom != 0 && total_aux_stack_size != 0) { + if (start_offset) + *start_offset = stack_bottom; + if (size) + *size = total_aux_stack_size; + return true; + } + return false; +} +#endif + +#if (WASM_ENABLE_MEMORY_PROFILING != 0) || (WASM_ENABLE_MEMORY_TRACING != 0) +void +wasm_get_module_mem_consumption(const WASMModule *module, + WASMModuleMemConsumption *mem_conspn) +{ + uint32 i, size; + + memset(mem_conspn, 0, sizeof(*mem_conspn)); + + mem_conspn->module_struct_size = sizeof(WASMModule); + + mem_conspn->types_size = sizeof(WASMFuncType *) * module->type_count; + for (i = 0; i < module->type_count; i++) { + WASMFuncType *type = module->types[i]; + size = offsetof(WASMFuncType, types) + + sizeof(uint8) * (type->param_count + type->result_count); + mem_conspn->types_size += size; + } + + mem_conspn->imports_size = sizeof(WASMImport) * module->import_count; + + mem_conspn->functions_size = + sizeof(WASMFunction *) * module->function_count; + for (i = 0; i < module->function_count; i++) { + WASMFunction *func = module->functions[i]; + WASMFuncType *type = func->func_type; + size = sizeof(WASMFunction) + func->local_count + + sizeof(uint16) * (type->param_count + func->local_count); +#if WASM_ENABLE_FAST_INTERP != 0 + size += + func->code_compiled_size + sizeof(uint32) * func->const_cell_num; +#endif + mem_conspn->functions_size += size; + } + + mem_conspn->tables_size = sizeof(WASMTable) * module->table_count; + mem_conspn->memories_size = sizeof(WASMMemory) * module->memory_count; + mem_conspn->globals_size = sizeof(WASMGlobal) * module->global_count; + mem_conspn->exports_size = sizeof(WASMExport) * module->export_count; + + mem_conspn->table_segs_size = + sizeof(WASMTableSeg) * module->table_seg_count; + for (i = 0; i < module->table_seg_count; i++) { + WASMTableSeg *table_seg = &module->table_segments[i]; + mem_conspn->tables_size += + sizeof(InitializerExpression *) * table_seg->value_count; + } + + mem_conspn->data_segs_size = sizeof(WASMDataSeg *) * module->data_seg_count; + for (i = 0; i < module->data_seg_count; i++) { + mem_conspn->data_segs_size += sizeof(WASMDataSeg); + } + + if (module->const_str_list) { + StringNode *node = module->const_str_list, *node_next; + while (node) { + node_next = node->next; + mem_conspn->const_strs_size += + sizeof(StringNode) + strlen(node->str) + 1; + node = node_next; + } + } + + mem_conspn->total_size += mem_conspn->module_struct_size; + mem_conspn->total_size += mem_conspn->types_size; + mem_conspn->total_size += mem_conspn->imports_size; + mem_conspn->total_size += mem_conspn->functions_size; + mem_conspn->total_size += mem_conspn->tables_size; + mem_conspn->total_size += mem_conspn->memories_size; + mem_conspn->total_size += mem_conspn->globals_size; + mem_conspn->total_size += mem_conspn->exports_size; + mem_conspn->total_size += mem_conspn->table_segs_size; + mem_conspn->total_size += mem_conspn->data_segs_size; + mem_conspn->total_size += mem_conspn->const_strs_size; +} + +void +wasm_get_module_inst_mem_consumption(const WASMModuleInstance *module_inst, + WASMModuleInstMemConsumption *mem_conspn) +{ + uint32 i; + uint64 size; + + memset(mem_conspn, 0, sizeof(*mem_conspn)); + + mem_conspn->module_inst_struct_size = (uint8 *)module_inst->e + - (uint8 *)module_inst + + sizeof(WASMModuleInstanceExtra); + + mem_conspn->memories_size = + sizeof(WASMMemoryInstance *) * module_inst->memory_count; + for (i = 0; i < module_inst->memory_count; i++) { + WASMMemoryInstance *memory = module_inst->memories[i]; + size = (uint64)memory->num_bytes_per_page * memory->cur_page_count; + mem_conspn->memories_size += size; + mem_conspn->app_heap_size += memory->heap_data_end - memory->heap_data; + /* size of app heap structure */ + mem_conspn->memories_size += mem_allocator_get_heap_struct_size(); + /* Module instance structures have been appended into the end of + module instance */ + } + + mem_conspn->tables_size = + sizeof(WASMTableInstance *) * module_inst->table_count; + /* Table instance structures and table elements have been appended into + the end of module instance */ + + mem_conspn->functions_size = + sizeof(WASMFunctionInstance) * module_inst->e->function_count; + + mem_conspn->globals_size = + sizeof(WASMGlobalInstance) * module_inst->e->global_count; + /* Global data has been appended into the end of module instance */ + + mem_conspn->exports_size = + sizeof(WASMExportFuncInstance) * module_inst->export_func_count; + + mem_conspn->total_size += mem_conspn->module_inst_struct_size; + mem_conspn->total_size += mem_conspn->memories_size; + mem_conspn->total_size += mem_conspn->functions_size; + mem_conspn->total_size += mem_conspn->tables_size; + mem_conspn->total_size += mem_conspn->globals_size; + mem_conspn->total_size += mem_conspn->exports_size; +} +#endif /* end of (WASM_ENABLE_MEMORY_PROFILING != 0) \ + || (WASM_ENABLE_MEMORY_TRACING != 0) */ + +#if WASM_ENABLE_COPY_CALL_STACK != 0 +uint32 +wasm_interp_copy_callstack(WASMExecEnv *exec_env, WASMCApiFrame *buffer, + uint32 length, uint32 skip_n, char *error_buf, + uint32_t error_buf_size) +{ + /* + * Note for devs: please refrain from such modifications inside of + * wasm_interp_copy_callstack + * - any allocations/freeing memory + * - dereferencing any pointers other than: exec_env, exec_env->module_inst, + * exec_env->module_inst->module, pointers between stack's bottom and + * top_boundary For more details check wasm_copy_callstack in + * wasm_export.h + */ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)wasm_exec_env_get_module_inst(exec_env); + WASMInterpFrame *cur_frame = wasm_exec_env_get_cur_frame(exec_env); + uint8 *top_boundary = exec_env->wasm_stack.top_boundary; + uint8 *bottom = exec_env->wasm_stack.bottom; + uint32 count = 0; + + WASMCApiFrame record_frame; + while (cur_frame && (uint8_t *)cur_frame >= bottom + && (uint8_t *)cur_frame + sizeof(WASMInterpFrame) <= top_boundary + && count < (skip_n + length)) { + if (!cur_frame->function) { + cur_frame = cur_frame->prev_frame; + continue; + } + if (count < skip_n) { + ++count; + cur_frame = cur_frame->prev_frame; + continue; + } + record_frame.instance = module_inst; + record_frame.module_offset = 0; + // It's safe to dereference module_inst->e because "e" is asigned only + // once in wasm_instantiate + record_frame.func_index = + (uint32)(cur_frame->function - module_inst->e->functions); + buffer[count - skip_n] = record_frame; + cur_frame = cur_frame->prev_frame; + ++count; + } + return count >= skip_n ? count - skip_n : 0; +} +#endif // WASM_ENABLE_COPY_CALL_STACK + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 +bool +wasm_interp_create_call_stack(struct WASMExecEnv *exec_env) +{ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)wasm_exec_env_get_module_inst(exec_env); + WASMModule *module = module_inst->module; + WASMInterpFrame *first_frame, + *cur_frame = wasm_exec_env_get_cur_frame(exec_env); + uint32 n = 0; + + /* count frames includes a function */ + first_frame = cur_frame; + while (cur_frame) { + if (cur_frame->function) { + n++; + } + cur_frame = cur_frame->prev_frame; + } + + /* release previous stack frames and create new ones */ + destroy_c_api_frames(module_inst->frames); + if (!bh_vector_init(module_inst->frames, n, sizeof(WASMCApiFrame), false)) { + return false; + } + + cur_frame = first_frame; + n = 0; + + while (cur_frame) { + WASMCApiFrame frame = { 0 }; + WASMFunctionInstance *func_inst = cur_frame->function; + const char *func_name = NULL; + const uint8 *func_code_base = NULL; + uint32 max_local_cell_num, max_stack_cell_num; + uint32 all_cell_num, lp_size; + + if (!func_inst) { + cur_frame = cur_frame->prev_frame; + continue; + } + + /* place holder, will overwrite it in wasm_c_api */ + frame.instance = module_inst; + frame.module_offset = 0; + frame.func_index = (uint32)(func_inst - module_inst->e->functions); + + func_code_base = wasm_get_func_code(func_inst); + if (!cur_frame->ip || !func_code_base) { + frame.func_offset = 0; + } + else { +#if WASM_ENABLE_FAST_INTERP == 0 + frame.func_offset = (uint32)(cur_frame->ip - module->load_addr); +#else + frame.func_offset = (uint32)(cur_frame->ip - func_code_base); +#endif + } + + func_name = get_func_name_from_index(module_inst, frame.func_index); + frame.func_name_wp = func_name; + + if (frame.func_index >= module->import_function_count) { + uint32 wasm_func_idx = + frame.func_index - module->import_function_count; + max_local_cell_num = + module->functions[wasm_func_idx]->param_cell_num + + module->functions[wasm_func_idx]->local_cell_num; + max_stack_cell_num = + module->functions[wasm_func_idx]->max_stack_cell_num; + all_cell_num = max_local_cell_num + max_stack_cell_num; +#if WASM_ENABLE_FAST_INTERP != 0 + all_cell_num += module->functions[wasm_func_idx]->const_cell_num; +#endif + } + else { + WASMFuncType *func_type = + module->import_functions[frame.func_index].u.function.func_type; + max_local_cell_num = + func_type->param_cell_num > 2 ? func_type->param_cell_num : 2; + max_stack_cell_num = 0; + all_cell_num = max_local_cell_num + max_stack_cell_num; + } + +#if WASM_ENABLE_GC == 0 + lp_size = all_cell_num * 4; +#else + lp_size = align_uint(all_cell_num * 5, 4); +#endif + if (lp_size > 0) { + if (!(frame.lp = wasm_runtime_malloc(lp_size))) { + destroy_c_api_frames(module_inst->frames); + return false; + } + bh_memcpy_s(frame.lp, lp_size, cur_frame->lp, lp_size); + +#if WASM_ENABLE_GC != 0 +#if WASM_ENABLE_FAST_INTERP == 0 + frame.sp = frame.lp + (cur_frame->sp - cur_frame->lp); +#else + /* for fast-interp, let frame sp point to the end of the frame */ + frame.sp = frame.lp + all_cell_num; +#endif + frame.frame_ref = (uint8 *)frame.lp + + (wasm_interp_get_frame_ref(cur_frame) + - (uint8 *)cur_frame->lp); +#endif + } + + if (!bh_vector_append(module_inst->frames, &frame)) { + if (frame.lp) + wasm_runtime_free(frame.lp); + destroy_c_api_frames(module_inst->frames); + return false; + } + + cur_frame = cur_frame->prev_frame; + n++; + } + + return true; +} + +#define PRINT_OR_DUMP() \ + do { \ + total_len += \ + wasm_runtime_dump_line_buf_impl(line_buf, print, &buf, &len); \ + if ((!print) && buf && (len == 0)) { \ + exception_unlock(module_inst); \ + return total_len; \ + } \ + } while (0) + +uint32 +wasm_interp_dump_call_stack(struct WASMExecEnv *exec_env, bool print, char *buf, + uint32 len) +{ + WASMModuleInstance *module_inst = + (WASMModuleInstance *)wasm_exec_env_get_module_inst(exec_env); + uint32 n = 0, total_len = 0, total_frames; + /* reserve 256 bytes for line buffer, any line longer than 256 bytes + * will be truncated */ + char line_buf[256]; + + if (!module_inst->frames) { + return 0; + } + + total_frames = (uint32)bh_vector_size(module_inst->frames); + if (total_frames == 0) { + return 0; + } + + exception_lock(module_inst); + snprintf(line_buf, sizeof(line_buf), "\n"); + PRINT_OR_DUMP(); + + while (n < total_frames) { + WASMCApiFrame frame = { 0 }; + uint32 line_length, i; + + if (!bh_vector_get(module_inst->frames, n, &frame)) { + exception_unlock(module_inst); + return 0; + } + +#if WASM_ENABLE_FAST_JIT != 0 + /* Fast JIT doesn't support committing ip (instruction pointer) yet */ + if (module_inst->e->running_mode == Mode_Fast_JIT + || module_inst->e->running_mode == Mode_Multi_Tier_JIT) { + /* function name not exported, print number instead */ + if (frame.func_name_wp == NULL) { + line_length = snprintf(line_buf, sizeof(line_buf), + "#%02" PRIu32 " $f%" PRIu32 "\n", n, + frame.func_index); + } + else { + line_length = + snprintf(line_buf, sizeof(line_buf), "#%02" PRIu32 " %s\n", + n, frame.func_name_wp); + } + } + else +#endif + { + /* function name not exported, print number instead */ + if (frame.func_name_wp == NULL) { + line_length = + snprintf(line_buf, sizeof(line_buf), + "#%02" PRIu32 ": 0x%04x - $f%" PRIu32 "\n", n, + frame.func_offset, frame.func_index); + } + else { + line_length = snprintf(line_buf, sizeof(line_buf), + "#%02" PRIu32 ": 0x%04x - %s\n", n, + frame.func_offset, frame.func_name_wp); + } + } + + if (line_length >= sizeof(line_buf)) { + uint32 line_buffer_len = sizeof(line_buf); + /* If line too long, ensure the last character is '\n' */ + for (i = line_buffer_len - 5; i < line_buffer_len - 2; i++) { + line_buf[i] = '.'; + } + line_buf[line_buffer_len - 2] = '\n'; + } + + PRINT_OR_DUMP(); + + n++; + } + snprintf(line_buf, sizeof(line_buf), "\n"); + PRINT_OR_DUMP(); + exception_unlock(module_inst); + + return total_len + 1; +} +#endif /* end of WASM_ENABLE_DUMP_CALL_STACK */ + +#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 +void +jit_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id) +{ + if (id != EXCE_ALREADY_THROWN) + wasm_set_exception_with_id(module_inst, id); +#ifdef OS_ENABLE_HW_BOUND_CHECK + wasm_runtime_access_exce_check_guard_page(); +#endif +} + +bool +jit_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, + uint64 app_buf_addr, uint64 app_buf_size, + void **p_native_addr) +{ + bool ret = wasm_check_app_addr_and_convert( + module_inst, is_str, app_buf_addr, app_buf_size, p_native_addr); + +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!ret) + wasm_runtime_access_exce_check_guard_page(); +#endif + + return ret; +} +#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ + || WASM_ENABLE_WAMR_COMPILER != 0 */ + +#if WASM_ENABLE_FAST_JIT != 0 +bool +fast_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 type_idx, uint32 argc, uint32 *argv) +{ + return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, true, + type_idx); +} +#endif /* end of WASM_ENABLE_FAST_JIT != 0 */ + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + +bool +llvm_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 argc, uint32 *argv) +{ + bool ret; + + bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode); + + ret = call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0); +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!ret) + wasm_runtime_access_exce_check_guard_page(); +#endif + return ret; +} + +bool +llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, + uint32 *argv) +{ + WASMModuleInstance *module_inst; + WASMModule *module; + uint32 *func_type_indexes; + uint32 func_type_idx; + WASMFuncType *func_type; + void *func_ptr; + WASMFunctionImport *import_func; + CApiFuncImport *c_api_func_import = NULL; + const char *signature; + void *attachment; + char buf[96]; + bool ret = false; + + bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode); + + module_inst = (WASMModuleInstance *)wasm_runtime_get_module_inst(exec_env); + module = module_inst->module; + func_type_indexes = module_inst->func_type_indexes; + func_type_idx = func_type_indexes[func_idx]; + func_type = (WASMFuncType *)module->types[func_type_idx]; + func_ptr = module_inst->func_ptrs[func_idx]; + + bh_assert(func_idx < module->import_function_count); + + import_func = &module->import_functions[func_idx].u.function; + if (import_func->call_conv_wasm_c_api) { + if (module_inst->c_api_func_imports) { + c_api_func_import = module_inst->c_api_func_imports + func_idx; + func_ptr = c_api_func_import->func_ptr_linked; + } + else { + c_api_func_import = NULL; + func_ptr = NULL; + } + } + + if (!func_ptr) { + snprintf(buf, sizeof(buf), + "failed to call unlinked import function (%s, %s)", + import_func->module_name, import_func->field_name); + wasm_set_exception(module_inst, buf); + goto fail; + } + + attachment = import_func->attachment; + if (import_func->call_conv_wasm_c_api) { + ret = wasm_runtime_invoke_c_api_native( + (WASMModuleInstanceCommon *)module_inst, func_ptr, func_type, argc, + argv, c_api_func_import->with_env_arg, c_api_func_import->env_arg); + } + else if (!import_func->call_conv_raw) { + signature = import_func->signature; + ret = + wasm_runtime_invoke_native(exec_env, func_ptr, func_type, signature, + attachment, argv, argc, argv); + } + else { + signature = import_func->signature; + ret = wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type, + signature, attachment, argv, argc, + argv); + } + +fail: +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!ret) + wasm_runtime_access_exce_check_guard_page(); +#endif + return ret; +} + +#if WASM_ENABLE_BULK_MEMORY != 0 +bool +llvm_jit_memory_init(WASMModuleInstance *module_inst, uint32 seg_index, + uint32 offset, uint32 len, size_t dst) +{ + WASMMemoryInstance *memory_inst; + WASMModule *module; + uint8 *data; + uint8 *maddr; + uint64 seg_len; + + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + + memory_inst = wasm_get_default_memory(module_inst); + + if (bh_bitmap_get_bit(module_inst->e->common.data_dropped, seg_index)) { + seg_len = 0; + data = NULL; + } + else { + module = module_inst->module; + seg_len = module->data_segments[seg_index]->data_length; + data = module->data_segments[seg_index]->data; + } + + if (!wasm_runtime_validate_app_addr((WASMModuleInstanceCommon *)module_inst, + (uint64)dst, (uint64)len)) + return false; + + if ((uint64)offset + (uint64)len > seg_len) { + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; + } + + maddr = wasm_runtime_addr_app_to_native( + (WASMModuleInstanceCommon *)module_inst, (uint64)dst); + + SHARED_MEMORY_LOCK(memory_inst); + bh_memcpy_s(maddr, CLAMP_U64_TO_U32(memory_inst->memory_data_size - dst), + data + offset, len); + SHARED_MEMORY_UNLOCK(memory_inst); + return true; +} + +bool +llvm_jit_data_drop(WASMModuleInstance *module_inst, uint32 seg_index) +{ + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + + bh_bitmap_set_bit(module_inst->e->common.data_dropped, seg_index); + /* Currently we can't free the dropped data segment + as they are stored in wasm bytecode */ + return true; +} +#endif /* end of WASM_ENABLE_BULK_MEMORY != 0 */ + +#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 +void +llvm_jit_drop_table_seg(WASMModuleInstance *module_inst, uint32 tbl_seg_idx) +{ + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + bh_bitmap_set_bit(module_inst->e->common.elem_dropped, tbl_seg_idx); +} + +void +llvm_jit_table_init(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 tbl_seg_idx, uint32 length, uint32 src_offset, + uint32 dst_offset) +{ + WASMTableInstance *tbl_inst; + WASMTableSeg *tbl_seg; + table_elem_type_t *table_elems; + InitializerExpression *tbl_seg_init_values = NULL, *init_values; + uint32 i, tbl_seg_len = 0; +#if WASM_ENABLE_GC != 0 + void *func_obj; +#endif + + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + + tbl_inst = wasm_get_table_inst(module_inst, tbl_idx); + tbl_seg = module_inst->module->table_segments + tbl_seg_idx; + + bh_assert(tbl_inst); + bh_assert(tbl_seg); + + if (!bh_bitmap_get_bit(module_inst->e->common.elem_dropped, tbl_seg_idx)) { + /* table segment isn't dropped */ + tbl_seg_init_values = tbl_seg->init_values; + tbl_seg_len = tbl_seg->value_count; + } + + if (offset_len_out_of_bounds(src_offset, length, tbl_seg_len) + || offset_len_out_of_bounds(dst_offset, length, tbl_inst->cur_size)) { + jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + if (!length) { + return; + } + + table_elems = tbl_inst->elems + dst_offset; + init_values = tbl_seg_init_values + src_offset; + + for (i = 0; i < length; i++) { +#if WASM_ENABLE_GC != 0 + /* UINT32_MAX indicates that it is a null ref */ + if (init_values[i].u.unary.v.ref_index != UINT32_MAX) { + if (!(func_obj = wasm_create_func_obj( + module_inst, init_values[i].u.unary.v.ref_index, true, + NULL, 0))) { + wasm_set_exception(module_inst, "null function reference"); + return; + } + table_elems[i] = func_obj; + } + else { + table_elems[i] = NULL_REF; + } +#else + table_elems[i] = init_values[i].u.unary.v.ref_index; +#endif + } +} + +void +llvm_jit_table_copy(WASMModuleInstance *module_inst, uint32 src_tbl_idx, + uint32 dst_tbl_idx, uint32 length, uint32 src_offset, + uint32 dst_offset) +{ + WASMTableInstance *src_tbl_inst; + WASMTableInstance *dst_tbl_inst; + + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + + src_tbl_inst = wasm_get_table_inst(module_inst, src_tbl_idx); + dst_tbl_inst = wasm_get_table_inst(module_inst, dst_tbl_idx); + bh_assert(src_tbl_inst); + bh_assert(dst_tbl_inst); + + if (offset_len_out_of_bounds(dst_offset, length, dst_tbl_inst->cur_size) + || offset_len_out_of_bounds(src_offset, length, + src_tbl_inst->cur_size)) { + jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + /* if src_offset >= dst_offset, copy from front to back */ + /* if src_offset < dst_offset, copy from back to front */ + /* merge all together */ + bh_memmove_s((uint8 *)dst_tbl_inst + offsetof(WASMTableInstance, elems) + + sizeof(table_elem_type_t) * dst_offset, + (uint32)sizeof(table_elem_type_t) + * (dst_tbl_inst->cur_size - dst_offset), + (uint8 *)src_tbl_inst + offsetof(WASMTableInstance, elems) + + sizeof(table_elem_type_t) * src_offset, + (uint32)sizeof(table_elem_type_t) * length); +} + +void +llvm_jit_table_fill(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 length, uintptr_t val, uint32 data_offset) +{ + WASMTableInstance *tbl_inst; + + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + + tbl_inst = wasm_get_table_inst(module_inst, tbl_idx); + bh_assert(tbl_inst); + + if (offset_len_out_of_bounds(data_offset, length, tbl_inst->cur_size)) { + jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); + return; + } + + for (; length != 0; data_offset++, length--) { + tbl_inst->elems[data_offset] = (table_elem_type_t)val; + } +} + +uint32 +llvm_jit_table_grow(WASMModuleInstance *module_inst, uint32 tbl_idx, + uint32 inc_size, uintptr_t init_val) +{ + WASMTableInstance *tbl_inst; + uint32 i, orig_size, total_size; + + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + + tbl_inst = wasm_get_table_inst(module_inst, tbl_idx); + if (!tbl_inst) { + return (uint32)-1; + } + + orig_size = tbl_inst->cur_size; + + if (!inc_size) { + return orig_size; + } + + if (tbl_inst->cur_size > UINT32_MAX - inc_size) { /* integer overflow */ +#if WASM_ENABLE_SPEC_TEST == 0 + LOG_WARNING("table grow (%" PRIu32 "-> %" PRIu32 + ") failed because of integer overflow", + tbl_inst->cur_size, inc_size); +#endif + return (uint32)-1; + } + + total_size = tbl_inst->cur_size + inc_size; + if (total_size > tbl_inst->max_size) { +#if WASM_ENABLE_SPEC_TEST == 0 + LOG_WARNING("table grow (%" PRIu32 "-> %" PRIu32 + ") failed because of over max size", + tbl_inst->cur_size, inc_size); +#endif + return (uint32)-1; + } + + /* fill in */ + for (i = 0; i < inc_size; ++i) { + tbl_inst->elems[tbl_inst->cur_size + i] = (table_elem_type_t)init_val; + } + + tbl_inst->cur_size = total_size; + return orig_size; +} +#endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ + +#if WASM_ENABLE_GC != 0 +void * +llvm_jit_create_func_obj(WASMModuleInstance *module_inst, uint32 func_idx, + bool throw_exce, char *error_buf, + uint32 error_buf_size) +{ + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); + + return wasm_create_func_obj(module_inst, func_idx, throw_exce, error_buf, + error_buf_size); +} + +bool +llvm_jit_obj_is_instance_of(WASMModuleInstance *module_inst, + WASMObjectRef gc_obj, uint32 type_index) +{ + WASMModule *module = module_inst->module; + WASMType **types = module->types; + uint32 type_count = module->type_count; + + return wasm_obj_is_instance_of(gc_obj, type_index, types, type_count); +} + +bool +llvm_jit_func_type_is_super_of(WASMModuleInstance *module_inst, + uint32 type_idx1, uint32 type_idx2) +{ + WASMModule *module = module_inst->module; + WASMType **types = module->types; + + if (type_idx1 == type_idx2) + return true; + + bh_assert(types[type_idx1]->type_flag == WASM_TYPE_FUNC); + bh_assert(types[type_idx2]->type_flag == WASM_TYPE_FUNC); + return wasm_func_type_is_super_of((WASMFuncType *)types[type_idx1], + (WASMFuncType *)types[type_idx2]); +} + +WASMRttTypeRef +llvm_jit_rtt_type_new(WASMModuleInstance *module_inst, uint32 type_index) +{ + WASMModule *module = module_inst->module; + WASMType *defined_type = module->types[type_index]; + WASMRttType **rtt_types = module->rtt_types; + uint32 rtt_type_count = module->type_count; + korp_mutex *rtt_type_lock = &module->rtt_type_lock; + + return wasm_rtt_type_new(defined_type, type_index, rtt_types, + rtt_type_count, rtt_type_lock); +} + +bool +llvm_array_init_with_data(WASMModuleInstance *module_inst, uint32 seg_index, + uint32 data_seg_offset, WASMArrayObjectRef array_obj, + uint32 elem_size, uint32 array_len) +{ + WASMModule *wasm_module = module_inst->module; + WASMDataSeg *data_seg; + uint8 *array_elem_base; + uint64 total_size; + + data_seg = wasm_module->data_segments[seg_index]; + total_size = (int64)elem_size * array_len; + + if (data_seg_offset >= data_seg->data_length + || total_size > data_seg->data_length - data_seg_offset) { + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; + } + + array_elem_base = (uint8 *)wasm_array_obj_first_elem_addr(array_obj); + bh_memcpy_s(array_elem_base, (uint32)total_size, + data_seg->data + data_seg_offset, (uint32)total_size); + + return true; +} +#endif /* end of WASM_ENABLE_GC != 0 */ + +#endif /* end of WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 */ + +#if WASM_ENABLE_LIBC_WASI != 0 && WASM_ENABLE_MULTI_MODULE != 0 +void +wasm_propagate_wasi_args(WASMModule *module) +{ + if (!module->import_count) + return; + + bh_assert(&module->import_module_list_head); + + WASMRegisteredModule *node = + bh_list_first_elem(&module->import_module_list_head); + while (node) { + WASIArguments *wasi_args_impt_mod = + &((WASMModule *)(node->module))->wasi_args; + bh_assert(wasi_args_impt_mod); + + bh_memcpy_s(wasi_args_impt_mod, sizeof(WASIArguments), + &module->wasi_args, sizeof(WASIArguments)); + node = bh_list_elem_next(node); + } +} +#endif + +bool +wasm_check_utf8_str(const uint8 *str, uint32 len) +{ + /* The valid ranges are taken from page 125, below link + https://www.unicode.org/versions/Unicode9.0.0/ch03.pdf */ + const uint8 *p = str, *p_end = str + len; + uint8 chr; + + while (p < p_end) { + chr = *p; + + if (chr == 0) { + LOG_WARNING( + "LIMITATION: a string which contains '\\00' is unsupported"); + return false; + } + else if (chr < 0x80) { + p++; + } + else if (chr >= 0xC2 && chr <= 0xDF && p + 1 < p_end) { + if (p[1] < 0x80 || p[1] > 0xBF) { + return false; + } + p += 2; + } + else if (chr >= 0xE0 && chr <= 0xEF && p + 2 < p_end) { + if (chr == 0xE0) { + if (p[1] < 0xA0 || p[1] > 0xBF || p[2] < 0x80 || p[2] > 0xBF) { + return false; + } + } + else if (chr == 0xED) { + if (p[1] < 0x80 || p[1] > 0x9F || p[2] < 0x80 || p[2] > 0xBF) { + return false; + } + } + else { /* chr >= 0xE1 && chr <= 0xEF */ + if (p[1] < 0x80 || p[1] > 0xBF || p[2] < 0x80 || p[2] > 0xBF) { + return false; + } + } + p += 3; + } + else if (chr >= 0xF0 && chr <= 0xF4 && p + 3 < p_end) { + if (chr == 0xF0) { + if (p[1] < 0x90 || p[1] > 0xBF || p[2] < 0x80 || p[2] > 0xBF + || p[3] < 0x80 || p[3] > 0xBF) { + return false; + } + } + else if (chr <= 0xF3) { /* and also chr >= 0xF1 */ + if (p[1] < 0x80 || p[1] > 0xBF || p[2] < 0x80 || p[2] > 0xBF + || p[3] < 0x80 || p[3] > 0xBF) { + return false; + } + } + else { /* chr == 0xF4 */ + if (p[1] < 0x80 || p[1] > 0x8F || p[2] < 0x80 || p[2] > 0xBF + || p[3] < 0x80 || p[3] > 0xBF) { + return false; + } + } + p += 4; + } + else { + return false; + } + } + return (p == p_end); +} + +char * +wasm_const_str_list_insert(const uint8 *str, uint32 len, WASMModule *module, + bool is_load_from_file_buf, char *error_buf, + uint32 error_buf_size) +{ + StringNode *node, *node_next; + + if (!wasm_check_utf8_str(str, len)) { + set_error_buf(error_buf, error_buf_size, "invalid UTF-8 encoding"); + return NULL; + } + + if (len == 0) { + return ""; + } + else if (is_load_from_file_buf) { + /* As the file buffer can be referred to after loading, we use + the previous byte of leb encoded size to adjust the string: + move string 1 byte backward and then append '\0' */ + char *c_str = (char *)str - 1; + bh_memmove_s(c_str, len + 1, c_str + 1, len); + c_str[len] = '\0'; + return c_str; + } + + /* Search const str list */ + node = module->const_str_list; + while (node) { + node_next = node->next; + if (strlen(node->str) == len && !memcmp(node->str, str, len)) + break; + node = node_next; + } + + if (node) { + return node->str; + } + + if (!(node = runtime_malloc(sizeof(StringNode) + len + 1, error_buf, + error_buf_size))) { + return NULL; + } + + node->str = ((char *)node) + sizeof(StringNode); + bh_memcpy_s(node->str, len + 1, str, len); + node->str[len] = '\0'; + + if (!module->const_str_list) { + /* set as head */ + module->const_str_list = node; + node->next = NULL; + } + else { + /* insert it */ + node->next = module->const_str_list; + module->const_str_list = node; + } + + return node->str; +} + +bool +wasm_set_module_name(WASMModule *module, const char *name, char *error_buf, + uint32_t error_buf_size) +{ + if (!name) + return false; + + module->name = + wasm_const_str_list_insert((const uint8 *)name, (uint32)strlen(name), + module, false, error_buf, error_buf_size); + return module->name != NULL; +} + +const char * +wasm_get_module_name(WASMModule *module) +{ + return module->name; +} diff --git a/wamr/src/wasm_runtime_common.c b/wamr/src/wasm_runtime_common.c new file mode 100644 index 0000000..69d083d --- /dev/null +++ b/wamr/src/wasm_runtime_common.c @@ -0,0 +1,8007 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_platform.h" // bh_platform.h +#include "bh_common.h" // bh_common.h +#include "bh_assert.h" // bh_assert.h +#include "bh_log.h" // bh_log.h +#include "wasm_native.h" // wasm_native.h +#include "wasm_runtime_common.h" // wasm_runtime_common.h +#include "wasm_memory.h" // wasm_memory.h +#if WASM_ENABLE_INTERP != 0 +#include "wasm_runtime.h" // ../interpreter/wasm_runtime.h +#endif +#if WASM_ENABLE_AOT != 0 + + +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 +#include "thread_manager.h" // ../libraries/thread-mgr/thread_manager.h +#if WASM_ENABLE_DEBUG_INTERP != 0 +#include "debug_engine.h" // ../libraries/debug-engine/debug_engine.h +#endif +#endif + +#if WASM_ENABLE_FAST_JIT != 0 + +#endif +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + +#endif +#include "wasm_c_api_internal.h" // ../common/wasm_c_api_internal.h +#include "version.h" // ../../version.h + +/** + * For runtime build, BH_MALLOC/BH_FREE should be defined as + * wasm_runtime_malloc/wasm_runtime_free. + */ +#define CHECK(a) CHECK1(a) +#define CHECK1(a) SHOULD_BE_##a + +#define SHOULD_BE_wasm_runtime_malloc 1 +#if !CHECK(BH_MALLOC) +#error unexpected BH_MALLOC +#endif +#undef SHOULD_BE_wasm_runtime_malloc + +#define SHOULD_BE_wasm_runtime_free 1 +#if !CHECK(BH_FREE) +#error unexpected BH_FREE +#endif +#undef SHOULD_BE_wasm_runtime_free + +#undef CHECK +#undef CHECK1 + +#if WASM_ENABLE_MULTI_MODULE != 0 +/** + * A safety insurance to prevent + * circular dependencies which leads stack overflow + * try to break early + */ +typedef struct LoadingModule { + bh_list_link l; + /* point to a string pool */ + const char *module_name; +} LoadingModule; + +static bh_list loading_module_list_head; +static bh_list *const loading_module_list = &loading_module_list_head; +static korp_mutex loading_module_list_lock; + +/** + * A list to store all exported functions/globals/memories/tables + * of every fully loaded module + */ +static bh_list registered_module_list_head; +static bh_list *const registered_module_list = ®istered_module_list_head; +static korp_mutex registered_module_list_lock; +static void +wasm_runtime_destroy_registered_module_list(void); +#endif /* WASM_ENABLE_MULTI_MODULE */ + +#define E_TYPE_XIP 4 + +static uint8 +val_type_to_val_kind(uint8 value_type); + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 +/* Initialize externref hashmap */ +static bool +wasm_externref_map_init(void); + +/* Destroy externref hashmap */ +static void +wasm_externref_map_destroy(void); +#endif /* end of WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 */ + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) + snprintf(error_buf, error_buf_size, "%s", string); +} + +static void * +runtime_malloc(uint64 size, WASMModuleInstanceCommon *module_inst, + char *error_buf, uint32 error_buf_size) +{ + void *mem; + + if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) { + if (module_inst != NULL) { + wasm_runtime_set_exception(module_inst, "allocate memory failed"); + } + else if (error_buf != NULL) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + } + return NULL; + } + + memset(mem, 0, (uint32)size); + return mem; +} + +#if WASM_ENABLE_MULTI_MODULE != 0 +/* TODO: Let loader_malloc be a general API both for AOT and WASM. */ + +#define loader_malloc(size, error_buf, error_buf_size) \ + runtime_malloc(size, NULL, error_buf, error_buf_size) + +static void +set_error_buf_v(const WASMModuleCommon *module, char *error_buf, + uint32 error_buf_size, const char *format, ...) +{ + va_list args; + char buf[128]; + if (error_buf != NULL) { + va_start(args, format); + vsnprintf(buf, sizeof(buf), format, args); + va_end(args); + if (module->module_type == Wasm_Module_AoT) { + snprintf(error_buf, error_buf_size, "AOT module load failed: %s", + buf); + } + else if (module->module_type == Wasm_Module_Bytecode) { + snprintf(error_buf, error_buf_size, "WASM module load failed: %s", + buf); + } + } +} +#endif + +#if WASM_ENABLE_FAST_JIT != 0 +static JitCompOptions jit_options = { 0 }; +#endif + +#if WASM_ENABLE_JIT != 0 +/* opt_level: 3, size_level: 3, segue-flags: 0, + quick_invoke_c_api_import: false */ +static LLVMJITOptions llvm_jit_options = { 3, 3, 0, false }; +#endif + +#if WASM_ENABLE_GC != 0 +static uint32 gc_heap_size_default = GC_HEAP_SIZE_DEFAULT; +#endif + +static RunningMode runtime_running_mode = Mode_Default; + +#ifdef OS_ENABLE_HW_BOUND_CHECK +/* The exec_env of thread local storage, set before calling function + and used in signal handler, as we cannot get it from the argument + of signal handler */ +static os_thread_local_attribute WASMExecEnv *exec_env_tls = NULL; + +static bool +is_sig_addr_in_guard_pages(void *sig_addr, WASMModuleInstance *module_inst) +{ + WASMMemoryInstance *memory_inst; +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; +#endif + uint8 *mapped_mem_start_addr = NULL; + uint8 *mapped_mem_end_addr = NULL; + uint32 i; + + for (i = 0; i < module_inst->memory_count; ++i) { + /* To be compatible with multi memory, get the ith memory instance */ + memory_inst = wasm_get_memory_with_idx(module_inst, i); + mapped_mem_start_addr = memory_inst->memory_data; + mapped_mem_end_addr = memory_inst->memory_data + 8 * (uint64)BH_GB; + if (mapped_mem_start_addr <= (uint8 *)sig_addr + && (uint8 *)sig_addr < mapped_mem_end_addr) { + /* The address which causes segmentation fault is inside + the memory instance's guard regions */ + return true; + } + } + +#if WASM_ENABLE_SHARED_HEAP != 0 + shared_heap = + wasm_runtime_get_shared_heap((WASMModuleInstanceCommon *)module_inst); + if (shared_heap) { + mapped_mem_start_addr = shared_heap->base_addr; + mapped_mem_end_addr = shared_heap->base_addr + 8 * (uint64)BH_GB; + if (mapped_mem_start_addr <= (uint8 *)sig_addr + && (uint8 *)sig_addr < mapped_mem_end_addr) { + /* The address which causes segmentation fault is inside + the shared heap's guard regions */ + return true; + } + } +#endif + + return false; +} + +#ifndef BH_PLATFORM_WINDOWS +static void +runtime_signal_handler(void *sig_addr) +{ + WASMModuleInstance *module_inst; + WASMJmpBuf *jmpbuf_node; + uint32 page_size = os_getpagesize(); +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + uint8 *stack_min_addr; + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; +#endif + + /* Check whether current thread is running wasm function */ + if (exec_env_tls && exec_env_tls->handle == os_self_thread() + && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { + /* Get mapped mem info of current instance */ + module_inst = (WASMModuleInstance *)exec_env_tls->module_inst; + +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + /* Get stack info of current thread */ + stack_min_addr = os_thread_get_stack_boundary(); +#endif + + if (is_sig_addr_in_guard_pages(sig_addr, module_inst)) { + wasm_set_exception(module_inst, "out of bounds memory access"); + os_longjmp(jmpbuf_node->jmpbuf, 1); + } +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + else if (stack_min_addr <= (uint8 *)sig_addr + && (uint8 *)sig_addr + < stack_min_addr + page_size * guard_page_count) { + /* The address which causes segmentation fault is inside + native thread's guard page */ + wasm_set_exception(module_inst, "native stack overflow"); + os_longjmp(jmpbuf_node->jmpbuf, 1); + } +#endif + else if (exec_env_tls->exce_check_guard_page <= (uint8 *)sig_addr + && (uint8 *)sig_addr + < exec_env_tls->exce_check_guard_page + page_size) { + bh_assert(wasm_copy_exception(module_inst, NULL)); + os_longjmp(jmpbuf_node->jmpbuf, 1); + } + } +} +#else /* else of BH_PLATFORM_WINDOWS */ + +#if WASM_ENABLE_AOT != 0 +#include + +static uint32 +decode_insn(uint8 *insn) +{ + uint8 *data = (uint8 *)insn; + uint32 length = 32; /* reserve enough size */ + + /* Initialize decoder context */ + ZydisDecoder decoder; + ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, + ZYDIS_STACK_WIDTH_64); + + /* Initialize formatter */ + ZydisFormatter formatter; + ZydisFormatterInit(&formatter, ZYDIS_FORMATTER_STYLE_INTEL); + + /* Loop over the instructions in our buffer */ + ZyanU64 runtime_address = (ZyanU64)(uintptr_t)data; + ZyanUSize offset = 0; + ZydisDecodedInstruction instruction; + ZydisDecodedOperand operands[ZYDIS_MAX_OPERAND_COUNT_VISIBLE]; + char buffer[256]; + + if (ZYAN_SUCCESS(ZydisDecoderDecodeFull( + &decoder, data + offset, length - offset, &instruction, operands, + ZYDIS_MAX_OPERAND_COUNT_VISIBLE, + ZYDIS_DFLAG_VISIBLE_OPERANDS_ONLY))) { + + /* Format & print the binary instruction structure to + human readable format */ + ZydisFormatterFormatInstruction(&formatter, &instruction, operands, + instruction.operand_count_visible, + buffer, sizeof(buffer), + runtime_address); + +#if 0 + /* Print current instruction */ + os_printf("%012" PRIX64 " ", runtime_address); + puts(buffer); +#endif + + return instruction.length; + } + + /* Decode failed */ + return 0; +} +#endif /* end of WASM_ENABLE_AOT != 0 */ + +static LONG +next_action(WASMModuleInstance *module_inst, EXCEPTION_POINTERS *exce_info) +{ +#if WASM_ENABLE_AOT != 0 + uint32 insn_size; +#endif + + if (module_inst->module_type == Wasm_Module_Bytecode + && module_inst->e->running_mode == Mode_Interp) { + /* Continue to search next exception handler for + interpreter mode as it can be caught by + `__try { .. } __except { .. }` sentences in + wasm_runtime.c */ + return EXCEPTION_CONTINUE_SEARCH; + } + +#if WASM_ENABLE_AOT != 0 + /* Skip current instruction and continue to run for AOT/JIT mode. + TODO: implement unwind support for AOT/JIT code in Windows platform */ + insn_size = decode_insn((uint8 *)exce_info->ContextRecord->Rip); + if (insn_size > 0) { + exce_info->ContextRecord->Rip += insn_size; + return EXCEPTION_CONTINUE_EXECUTION; + } +#endif + + /* return different value from EXCEPTION_CONTINUE_SEARCH (= 0) + and EXCEPTION_CONTINUE_EXECUTION (= -1) */ + return -2; +} + +static LONG +runtime_exception_handler(EXCEPTION_POINTERS *exce_info) +{ + PEXCEPTION_RECORD ExceptionRecord = exce_info->ExceptionRecord; + uint8 *sig_addr = (uint8 *)ExceptionRecord->ExceptionInformation[1]; + WASMModuleInstance *module_inst; + WASMJmpBuf *jmpbuf_node; + uint8 *mapped_mem_start_addr = NULL; + uint8 *mapped_mem_end_addr = NULL; + uint32 page_size = os_getpagesize(); + LONG ret; + + if (exec_env_tls && exec_env_tls->handle == os_self_thread() + && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { + module_inst = (WASMModuleInstance *)exec_env_tls->module_inst; + if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { + if (is_sig_addr_in_guard_pages(sig_addr, module_inst)) { + /* The address which causes segmentation fault is inside + the memory instance's guard regions. + Set exception and let the wasm func continue to run, when + the wasm func returns, the caller will check whether the + exception is thrown and return to runtime. */ + wasm_set_exception(module_inst, "out of bounds memory access"); + ret = next_action(module_inst, exce_info); + if (ret == EXCEPTION_CONTINUE_SEARCH + || ret == EXCEPTION_CONTINUE_EXECUTION) + return ret; + } + else if (exec_env_tls->exce_check_guard_page <= (uint8 *)sig_addr + && (uint8 *)sig_addr + < exec_env_tls->exce_check_guard_page + page_size) { + bh_assert(wasm_copy_exception(module_inst, NULL)); + ret = next_action(module_inst, exce_info); + if (ret == EXCEPTION_CONTINUE_SEARCH + || ret == EXCEPTION_CONTINUE_EXECUTION) + return ret; + } + } +#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + else if (ExceptionRecord->ExceptionCode == EXCEPTION_STACK_OVERFLOW) { + /* Set stack overflow exception and let the wasm func continue + to run, when the wasm func returns, the caller will check + whether the exception is thrown and return to runtime, and + the damaged stack will be recovered by _resetstkoflw(). */ + wasm_set_exception(module_inst, "native stack overflow"); + ret = next_action(module_inst, exce_info); + if (ret == EXCEPTION_CONTINUE_SEARCH + || ret == EXCEPTION_CONTINUE_EXECUTION) + return ret; + } +#endif + else { + LOG_WARNING("Unhandled exception thrown: exception code: 0x%lx, " + "exception address: %p, exception information: %p\n", + ExceptionRecord->ExceptionCode, + ExceptionRecord->ExceptionAddress, sig_addr); + } + } + + return EXCEPTION_CONTINUE_SEARCH; +} +#endif /* end of BH_PLATFORM_WINDOWS */ + +static bool +runtime_signal_init() +{ +#ifndef BH_PLATFORM_WINDOWS + return os_thread_signal_init(runtime_signal_handler) == 0 ? true : false; +#else + if (os_thread_signal_init() != 0) + return false; + + if (!AddVectoredExceptionHandler(1, runtime_exception_handler)) { + os_thread_signal_destroy(); + return false; + } +#endif + return true; +} + +static void +runtime_signal_destroy() +{ +#ifdef BH_PLATFORM_WINDOWS + RemoveVectoredExceptionHandler(runtime_exception_handler); +#endif + os_thread_signal_destroy(); +} + +void +wasm_runtime_set_exec_env_tls(WASMExecEnv *exec_env) +{ + exec_env_tls = exec_env; +} + +WASMExecEnv * +wasm_runtime_get_exec_env_tls() +{ + return exec_env_tls; +} +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ + +static bool +wasm_runtime_env_init(void) +{ + if (bh_platform_init() != 0) + return false; + +#if WASM_ENABLE_INVOKE_NATIVE != 0 + if (wasm_native_init() == false) { + goto fail1; + } +#endif + +#if WASM_ENABLE_MULTI_MODULE + if (BHT_OK != os_mutex_init(®istered_module_list_lock)) { + goto fail2; + } + + if (BHT_OK != os_mutex_init(&loading_module_list_lock)) { + goto fail3; + } +#endif + +#if WASM_ENABLE_SHARED_MEMORY + if (!wasm_shared_memory_init()) { + goto fail4; + } +#endif + +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_THREAD_MGR != 0) + if (!thread_manager_init()) { + goto fail5; + } +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!runtime_signal_init()) { + goto fail6; + } +#endif + +#if WASM_ENABLE_AOT != 0 +#if WASM_ENABLE_DEBUG_AOT != 0 + if (!jit_debug_engine_init()) { + goto fail7; + } +#endif +#endif + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + if (!wasm_externref_map_init()) { + goto fail8; + } +#endif + +#if WASM_ENABLE_FAST_JIT != 0 + if (!jit_compiler_init(&jit_options)) { + goto fail9; + } +#endif + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + if (!aot_compiler_init()) { + goto fail10; + } +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 && defined(OS_ENABLE_WAKEUP_BLOCKING_OP) + if (os_blocking_op_init() != BHT_OK) { + goto fail11; + } + os_end_blocking_op(); +#endif + + return true; + +#if WASM_ENABLE_THREAD_MGR != 0 && defined(OS_ENABLE_WAKEUP_BLOCKING_OP) +fail11: +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + aot_compiler_destroy(); +#endif +#endif +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 +fail10: +#if WASM_ENABLE_FAST_JIT != 0 + jit_compiler_destroy(); +#endif +#endif +#if WASM_ENABLE_FAST_JIT != 0 +fail9: +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + wasm_externref_map_destroy(); +#endif +#endif +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 +fail8: +#endif +#if WASM_ENABLE_AOT != 0 +#if WASM_ENABLE_DEBUG_AOT != 0 + jit_debug_engine_destroy(); +fail7: +#endif +#endif +#ifdef OS_ENABLE_HW_BOUND_CHECK + runtime_signal_destroy(); +fail6: +#endif +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_THREAD_MGR != 0) + thread_manager_destroy(); +fail5: +#endif +#if WASM_ENABLE_SHARED_MEMORY + wasm_shared_memory_destroy(); +fail4: +#endif +#if WASM_ENABLE_MULTI_MODULE + os_mutex_destroy(&loading_module_list_lock); +fail3: + os_mutex_destroy(®istered_module_list_lock); +fail2: +#endif +#if WASM_ENABLE_INVOKE_NATIVE != 0 + wasm_native_destroy(); +#endif +fail1: + bh_platform_destroy(); + + return false; +} + +static bool +wasm_runtime_exec_env_check(WASMExecEnv *exec_env) +{ + return exec_env && exec_env->module_inst && exec_env->wasm_stack_size > 0 + && exec_env->wasm_stack.top_boundary + == exec_env->wasm_stack.bottom + exec_env->wasm_stack_size + && exec_env->wasm_stack.top <= exec_env->wasm_stack.top_boundary; +} + +#if defined(OS_THREAD_MUTEX_INITIALIZER) +/** + * lock for wasm_runtime_init/wasm_runtime_full_init and runtime_ref_count + * Note: if the platform has mutex initializer, we use a global lock to + * lock the operations of runtime init/full_init, otherwise when there are + * operations happening simultaneously in multiple threads, developer + * must create the lock by himself, and use it to lock the operations + */ +static korp_mutex runtime_lock = OS_THREAD_MUTEX_INITIALIZER; +#endif +static int32 runtime_ref_count = 0; + +static bool +wasm_runtime_init_internal(void) +{ + if (!wasm_runtime_memory_init(Alloc_With_System_Allocator, NULL)) + return false; + + if (!wasm_runtime_env_init()) { + wasm_runtime_memory_destroy(); + return false; + } + + return true; +} + +bool +wasm_runtime_init() +{ + bool ret = true; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&runtime_lock); +#endif + + bh_assert(runtime_ref_count >= 0); + if (runtime_ref_count == 0) { + ret = wasm_runtime_init_internal(); + } + if (ret) { + runtime_ref_count++; + } + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&runtime_lock); +#endif + + return ret; +} + +static void +wasm_runtime_destroy_internal(void) +{ +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + wasm_externref_map_destroy(); +#endif + +#if WASM_ENABLE_AOT != 0 +#if WASM_ENABLE_DEBUG_AOT != 0 + jit_debug_engine_destroy(); +#endif +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK + runtime_signal_destroy(); +#endif + + /* runtime env destroy */ +#if WASM_ENABLE_MULTI_MODULE + wasm_runtime_destroy_loading_module_list(); + os_mutex_destroy(&loading_module_list_lock); + + wasm_runtime_destroy_registered_module_list(); + os_mutex_destroy(®istered_module_list_lock); +#endif + +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 + /* Destroy LLVM-JIT compiler after destroying the modules + * loaded by multi-module feature, since these modules may + * create backend threads to compile the wasm functions, + * which may access the LLVM resources. We wait until they + * finish the compilation to avoid accessing the destroyed + * resources in the compilation threads. + */ + aot_compiler_destroy(); +#endif + +#if WASM_ENABLE_FAST_JIT != 0 + /* Destroy Fast-JIT compiler after destroying the modules + * loaded by multi-module feature, since the Fast JIT's + * code cache allocator may be used by these modules. + */ + jit_compiler_destroy(); +#endif + +#if WASM_ENABLE_SHARED_MEMORY + wasm_shared_memory_destroy(); +#endif + +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_THREAD_MGR != 0) +#if WASM_ENABLE_DEBUG_INTERP != 0 + wasm_debug_engine_destroy(); +#endif + thread_manager_destroy(); +#endif + +#if WASM_ENABLE_INVOKE_NATIVE != 0 + wasm_native_destroy(); +#endif + bh_platform_destroy(); + + wasm_runtime_memory_destroy(); +} + +void +wasm_runtime_destroy() +{ +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&runtime_lock); +#endif + + bh_assert(runtime_ref_count > 0); + runtime_ref_count--; + if (runtime_ref_count == 0) { + wasm_runtime_destroy_internal(); + } + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&runtime_lock); +#endif +} + +RunningMode +wasm_runtime_get_default_running_mode(void) +{ + return runtime_running_mode; +} + +#if WASM_ENABLE_JIT != 0 +LLVMJITOptions * +wasm_runtime_get_llvm_jit_options(void) +{ + return &llvm_jit_options; +} +#endif + +#if WASM_ENABLE_GC != 0 +uint32 +wasm_runtime_get_gc_heap_size_default(void) +{ + return gc_heap_size_default; +} +#endif + +static bool +wasm_runtime_full_init_internal(RuntimeInitArgs *init_args) +{ + if (!wasm_runtime_memory_init(init_args->mem_alloc_type, + &init_args->mem_alloc_option)) + return false; + + if (!wasm_runtime_set_default_running_mode(init_args->running_mode)) { + wasm_runtime_memory_destroy(); + return false; + } + +#if WASM_ENABLE_FAST_JIT != 0 + jit_options.code_cache_size = init_args->fast_jit_code_cache_size; +#endif + +#if WASM_ENABLE_GC != 0 + uint32 gc_heap_size = init_args->gc_heap_size; + if (gc_heap_size > 0) { + gc_heap_size_default = gc_heap_size; + } +#endif + +#if WASM_ENABLE_JIT != 0 + llvm_jit_options.size_level = init_args->llvm_jit_size_level; + llvm_jit_options.opt_level = init_args->llvm_jit_opt_level; + llvm_jit_options.segue_flags = init_args->segue_flags; +#endif + +#if WASM_ENABLE_LINUX_PERF != 0 + wasm_runtime_set_linux_perf(init_args->enable_linux_perf); +#else + if (init_args->enable_linux_perf) + LOG_WARNING("warning: to enable linux perf support, please recompile " + "with -DWAMR_BUILD_LINUX_PERF=1"); +#endif + + if (!wasm_runtime_env_init()) { + wasm_runtime_memory_destroy(); + return false; + } + +#if WASM_ENABLE_DEBUG_INTERP != 0 + if (strlen(init_args->ip_addr)) + if (!wasm_debug_engine_init(init_args->ip_addr, + init_args->instance_port)) { + wasm_runtime_destroy(); + return false; + } +#endif + +#if WASM_ENABLE_INVOKE_NATIVE != 0 + if (init_args->n_native_symbols > 0 + && !wasm_runtime_register_natives(init_args->native_module_name, + init_args->native_symbols, + init_args->n_native_symbols)) { + wasm_runtime_destroy(); + return false; + } +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_set_max_thread_num(init_args->max_thread_num); +#endif + + return true; +} + +bool +wasm_runtime_full_init(RuntimeInitArgs *init_args) +{ + bool ret = true; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&runtime_lock); +#endif + + bh_assert(runtime_ref_count >= 0); + if (runtime_ref_count == 0) { + ret = wasm_runtime_full_init_internal(init_args); + } + if (ret) { + runtime_ref_count++; + } + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&runtime_lock); +#endif + + return ret; +} + +void +wasm_runtime_set_log_level(log_level_t level) +{ + bh_log_set_verbose_level(level); +} + +bool +wasm_runtime_is_running_mode_supported(RunningMode running_mode) +{ + if (running_mode == Mode_Default) { + return true; + } + else if (running_mode == Mode_Interp) { +#if WASM_ENABLE_INTERP != 0 + return true; +#endif + } + else if (running_mode == Mode_Fast_JIT) { +#if WASM_ENABLE_FAST_JIT != 0 + return true; +#endif + } + else if (running_mode == Mode_LLVM_JIT) { +#if WASM_ENABLE_JIT != 0 + return true; +#endif + } + else if (running_mode == Mode_Multi_Tier_JIT) { +#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ + && WASM_ENABLE_LAZY_JIT != 0 + return true; +#endif + } + + return false; +} + +bool +wasm_runtime_set_default_running_mode(RunningMode running_mode) +{ + if (wasm_runtime_is_running_mode_supported(running_mode)) { + runtime_running_mode = running_mode; + return true; + } + return false; +} + +PackageType +get_package_type(const uint8 *buf, uint32 size) +{ + if (buf && size >= 4) { +#if (WASM_ENABLE_WORD_ALIGN_READ != 0) + uint32 buf32 = *(uint32 *)buf; + buf = (const uint8 *)&buf32; +#endif + if (buf[0] == '\0' && buf[1] == 'a' && buf[2] == 's' && buf[3] == 'm') + return Wasm_Module_Bytecode; + if (buf[0] == '\0' && buf[1] == 'a' && buf[2] == 'o' && buf[3] == 't') + return Wasm_Module_AoT; + } + return Package_Type_Unknown; +} + +PackageType +wasm_runtime_get_file_package_type(const uint8 *buf, uint32 size) +{ + return get_package_type(buf, size); +} + +PackageType +wasm_runtime_get_module_package_type(WASMModuleCommon *const module) +{ + if (!module) { + return Package_Type_Unknown; + } + + return module->module_type; +} + +uint32 +wasm_runtime_get_file_package_version(const uint8 *buf, uint32 size) +{ + if (buf && size >= 8) { + uint32 version; +#if (WASM_ENABLE_WORD_ALIGN_READ != 0) + uint32 buf32 = *(uint32 *)(buf + sizeof(uint32)); + buf = (const uint8 *)&buf32; + version = buf[0] | buf[1] << 8 | buf[2] << 16 | buf[3] << 24; +#else + version = buf[4] | buf[5] << 8 | buf[6] << 16 | buf[7] << 24; +#endif + return version; + } + + return 0; +} + +uint32 +wasm_runtime_get_module_package_version(WASMModuleCommon *const module) +{ + if (!module) { + return 0; + } + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + WASMModule *wasm_module = (WASMModule *)module; + return wasm_module->package_version; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + AOTModule *aot_module = (AOTModule *)module; + return aot_module->package_version; + } +#endif + + return 0; +} + +uint32 +wasm_runtime_get_current_package_version(package_type_t package_type) +{ + switch (package_type) { + case Wasm_Module_Bytecode: + return WASM_CURRENT_VERSION; + case Wasm_Module_AoT: + return AOT_CURRENT_VERSION; + case Package_Type_Unknown: + default: + return 0; + } +} + +#if WASM_ENABLE_AOT != 0 +static uint8 * +align_ptr(const uint8 *p, uint32 b) +{ + uintptr_t v = (uintptr_t)p; + uintptr_t m = b - 1; + return (uint8 *)((v + m) & ~m); +} + +#define CHECK_BUF(buf, buf_end, length) \ + do { \ + if ((uintptr_t)buf + length < (uintptr_t)buf \ + || (uintptr_t)buf + length > (uintptr_t)buf_end) \ + return false; \ + } while (0) + +/* NOLINTNEXTLINE */ +#define read_uint16(p, p_end, res) \ + do { \ + p = (uint8 *)align_ptr(p, sizeof(uint16)); \ + CHECK_BUF(p, p_end, sizeof(uint16)); \ + res = *(uint16 *)p; \ + p += sizeof(uint16); \ + } while (0) + +/* NOLINTNEXTLINE */ +#define read_uint32(p, p_end, res) \ + do { \ + p = (uint8 *)align_ptr(p, sizeof(uint32)); \ + CHECK_BUF(p, p_end, sizeof(uint32)); \ + res = *(uint32 *)p; \ + p += sizeof(uint32); \ + } while (0) + +bool +wasm_runtime_is_xip_file(const uint8 *buf, uint32 size) +{ + const uint8 *p = buf, *p_end = buf + size; + uint32 section_type, section_size; + uint16 e_type; + + if (get_package_type(buf, size) != Wasm_Module_AoT) + return false; + + CHECK_BUF(p, p_end, 8); + p += 8; + while (p < p_end) { + read_uint32(p, p_end, section_type); + read_uint32(p, p_end, section_size); + CHECK_BUF(p, p_end, section_size); + + if (section_type == AOT_SECTION_TYPE_TARGET_INFO) { + p += 4; + read_uint16(p, p_end, e_type); + return (e_type == E_TYPE_XIP) ? true : false; + } + else if (section_type >= AOT_SECTION_TYPE_SIGNATURE) { + return false; + } + p += section_size; + } + + return false; +} +#endif /* end of WASM_ENABLE_AOT */ + +#if (WASM_ENABLE_THREAD_MGR != 0) && (WASM_ENABLE_DEBUG_INTERP != 0) +uint32 +wasm_runtime_start_debug_instance_with_port(WASMExecEnv *exec_env, int32_t port) +{ + WASMModuleInstanceCommon *module_inst = + wasm_runtime_get_module_inst(exec_env); + WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(module_inst); + bh_assert(cluster); + + if (module_inst->module_type != Wasm_Module_Bytecode) { + LOG_WARNING("Attempt to create a debug instance for an AOT module"); + return 0; + } + + if (cluster->debug_inst) { + LOG_WARNING("Cluster already bind to a debug instance"); + return cluster->debug_inst->control_thread->port; + } + + if (wasm_debug_instance_create(cluster, port)) { + return cluster->debug_inst->control_thread->port; + } + + return 0; +} + +uint32 +wasm_runtime_start_debug_instance(WASMExecEnv *exec_env) +{ + return wasm_runtime_start_debug_instance_with_port(exec_env, -1); +} +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 +static module_reader reader; +static module_destroyer destroyer; +void +wasm_runtime_set_module_reader(const module_reader reader_cb, + const module_destroyer destroyer_cb) +{ + reader = reader_cb; + destroyer = destroyer_cb; +} + +module_reader +wasm_runtime_get_module_reader() +{ + return reader; +} + +module_destroyer +wasm_runtime_get_module_destroyer() +{ + return destroyer; +} + +static WASMRegisteredModule * +wasm_runtime_find_module_registered_by_reference(WASMModuleCommon *module) +{ + WASMRegisteredModule *reg_module = NULL; + + os_mutex_lock(®istered_module_list_lock); + reg_module = bh_list_first_elem(registered_module_list); + while (reg_module && module != reg_module->module) { + reg_module = bh_list_elem_next(reg_module); + } + os_mutex_unlock(®istered_module_list_lock); + + return reg_module; +} + +bool +wasm_runtime_register_module_internal(const char *module_name, + WASMModuleCommon *module, + uint8 *orig_file_buf, + uint32 orig_file_buf_size, + char *error_buf, uint32 error_buf_size) +{ + WASMRegisteredModule *node = NULL; + + node = wasm_runtime_find_module_registered_by_reference(module); + if (node) { /* module has been registered */ + if (node->module_name) { /* module has name */ + if (!module_name || strcmp(node->module_name, module_name)) { + /* module has different name */ + LOG_DEBUG("module(%p) has been registered with name %s", module, + node->module_name); + set_error_buf(error_buf, error_buf_size, + "Register module failed: " + "failed to rename the module"); + return false; + } + else { + /* module has the same name */ + LOG_DEBUG( + "module(%p) has been registered with the same name %s", + module, node->module_name); + return true; + } + } + else { + /* module has empty name, reset it */ + node->module_name = module_name; + return true; + } + } + + /* module hasn't been registered */ + node = runtime_malloc(sizeof(WASMRegisteredModule), NULL, NULL, 0); + if (!node) { + LOG_DEBUG("malloc WASMRegisteredModule failed. SZ=%zu", + sizeof(WASMRegisteredModule)); + return false; + } + + /* share the string and the module */ + node->module_name = module_name; + node->module = module; + node->orig_file_buf = orig_file_buf; + node->orig_file_buf_size = orig_file_buf_size; + + os_mutex_lock(®istered_module_list_lock); + bh_list_status ret = bh_list_insert(registered_module_list, node); + bh_assert(BH_LIST_SUCCESS == ret); + (void)ret; + os_mutex_unlock(®istered_module_list_lock); + return true; +} + +bool +wasm_runtime_register_module(const char *module_name, WASMModuleCommon *module, + char *error_buf, uint32 error_buf_size) +{ + if (!error_buf || !error_buf_size) { + LOG_ERROR("error buffer is required"); + return false; + } + + if (!module_name || !module) { + LOG_DEBUG("module_name and module are required"); + set_error_buf(error_buf, error_buf_size, + "Register module failed: " + "module_name and module are required"); + return false; + } + + if (wasm_runtime_is_built_in_module(module_name)) { + LOG_DEBUG("%s is a built-in module name", module_name); + set_error_buf(error_buf, error_buf_size, + "Register module failed: " + "can not register as a built-in module"); + return false; + } + + return wasm_runtime_register_module_internal(module_name, module, NULL, 0, + error_buf, error_buf_size); +} + +void +wasm_runtime_unregister_module(const WASMModuleCommon *module) +{ + WASMRegisteredModule *registered_module = NULL; + + os_mutex_lock(®istered_module_list_lock); + registered_module = bh_list_first_elem(registered_module_list); + while (registered_module && module != registered_module->module) { + registered_module = bh_list_elem_next(registered_module); + } + + /* it does not matter if it is not exist. after all, it is gone */ + if (registered_module) { + bh_list_remove(registered_module_list, registered_module); + wasm_runtime_free(registered_module); + } + os_mutex_unlock(®istered_module_list_lock); +} + +WASMModuleCommon * +wasm_runtime_find_module_registered(const char *module_name) +{ + WASMRegisteredModule *module = NULL, *module_next; + + os_mutex_lock(®istered_module_list_lock); + module = bh_list_first_elem(registered_module_list); + while (module) { + module_next = bh_list_elem_next(module); + if (module->module_name && !strcmp(module_name, module->module_name)) { + break; + } + module = module_next; + } + os_mutex_unlock(®istered_module_list_lock); + + return module ? module->module : NULL; +} + +/* + * simply destroy all + */ +static void +wasm_runtime_destroy_registered_module_list() +{ + WASMRegisteredModule *reg_module = NULL; + + os_mutex_lock(®istered_module_list_lock); + reg_module = bh_list_first_elem(registered_module_list); + while (reg_module) { + WASMRegisteredModule *next_reg_module = bh_list_elem_next(reg_module); + + bh_list_remove(registered_module_list, reg_module); + + /* now, it is time to release every module in the runtime */ + if (reg_module->module->module_type == Wasm_Module_Bytecode) { +#if WASM_ENABLE_INTERP != 0 + wasm_unload((WASMModule *)reg_module->module); +#endif + } + else { +#if WASM_ENABLE_AOT != 0 + aot_unload((AOTModule *)reg_module->module); +#endif + } + + /* destroy the file buffer */ + if (destroyer && reg_module->orig_file_buf) { + destroyer(reg_module->orig_file_buf, + reg_module->orig_file_buf_size); + reg_module->orig_file_buf = NULL; + reg_module->orig_file_buf_size = 0; + } + + wasm_runtime_free(reg_module); + reg_module = next_reg_module; + } + os_mutex_unlock(®istered_module_list_lock); +} + +bool +wasm_runtime_add_loading_module(const char *module_name, char *error_buf, + uint32 error_buf_size) +{ + LOG_DEBUG("add %s into a loading list", module_name); + LoadingModule *loadingModule = + runtime_malloc(sizeof(LoadingModule), NULL, error_buf, error_buf_size); + + if (!loadingModule) { + return false; + } + + /* share the incoming string */ + loadingModule->module_name = module_name; + + os_mutex_lock(&loading_module_list_lock); + bh_list_status ret = bh_list_insert(loading_module_list, loadingModule); + bh_assert(BH_LIST_SUCCESS == ret); + (void)ret; + os_mutex_unlock(&loading_module_list_lock); + return true; +} + +void +wasm_runtime_delete_loading_module(const char *module_name) +{ + LOG_DEBUG("delete %s from a loading list", module_name); + + LoadingModule *module = NULL; + + os_mutex_lock(&loading_module_list_lock); + module = bh_list_first_elem(loading_module_list); + while (module && strcmp(module->module_name, module_name)) { + module = bh_list_elem_next(module); + } + + /* it does not matter if it is not exist. after all, it is gone */ + if (module) { + bh_list_remove(loading_module_list, module); + wasm_runtime_free(module); + } + os_mutex_unlock(&loading_module_list_lock); +} + +bool +wasm_runtime_is_loading_module(const char *module_name) +{ + LOG_DEBUG("find %s in a loading list", module_name); + + LoadingModule *module = NULL; + + os_mutex_lock(&loading_module_list_lock); + module = bh_list_first_elem(loading_module_list); + while (module && strcmp(module_name, module->module_name)) { + module = bh_list_elem_next(module); + } + os_mutex_unlock(&loading_module_list_lock); + + return module != NULL; +} + +void +wasm_runtime_destroy_loading_module_list() +{ + LoadingModule *module = NULL; + + os_mutex_lock(&loading_module_list_lock); + module = bh_list_first_elem(loading_module_list); + while (module) { + LoadingModule *next_module = bh_list_elem_next(module); + + bh_list_remove(loading_module_list, module); + /* + * will not free the module_name since it is + * shared one of the const string pool + */ + wasm_runtime_free(module); + + module = next_module; + } + + os_mutex_unlock(&loading_module_list_lock); +} +#endif /* WASM_ENABLE_MULTI_MODULE */ + +bool +wasm_runtime_is_built_in_module(const char *module_name) +{ + return (!strcmp("env", module_name) || !strcmp("wasi_unstable", module_name) + || !strcmp("wasi_snapshot_preview1", module_name) +#if WASM_ENABLE_SPEC_TEST != 0 + || !strcmp("spectest", module_name) +#endif +#if WASM_ENABLE_WASI_TEST != 0 + || !strcmp("foo", module_name) +#endif + || !strcmp("", module_name)); +} + +#if WASM_ENABLE_THREAD_MGR != 0 +bool +wasm_exec_env_set_aux_stack(WASMExecEnv *exec_env, uint64 start_offset, + uint32 size) +{ + WASMModuleInstanceCommon *module_inst = + wasm_exec_env_get_module_inst(exec_env); +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + return wasm_set_aux_stack(exec_env, start_offset, size); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + return aot_set_aux_stack(exec_env, start_offset, size); + } +#endif + return false; +} + +bool +wasm_exec_env_get_aux_stack(WASMExecEnv *exec_env, uint64 *start_offset, + uint32 *size) +{ + WASMModuleInstanceCommon *module_inst = + wasm_exec_env_get_module_inst(exec_env); +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + return wasm_get_aux_stack(exec_env, start_offset, size); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + return aot_get_aux_stack(exec_env, start_offset, size); + } +#endif + return false; +} + +void +wasm_runtime_set_max_thread_num(uint32 num) +{ + wasm_cluster_set_max_thread_num(num); +} +#endif /* end of WASM_ENABLE_THREAD_MGR */ + +static WASMModuleCommon * +register_module_with_null_name(WASMModuleCommon *module_common, char *error_buf, + uint32 error_buf_size) +{ +#if WASM_ENABLE_MULTI_MODULE != 0 + if (module_common) { + if (!wasm_runtime_register_module_internal(NULL, module_common, NULL, 0, + error_buf, error_buf_size)) { + wasm_runtime_unload(module_common); + return NULL; + } + return module_common; + } + else + return NULL; +#else + return module_common; +#endif +} + +WASMModuleCommon * +wasm_runtime_load_ex(uint8 *buf, uint32 size, const LoadArgs *args, + char *error_buf, uint32 error_buf_size) +{ + WASMModuleCommon *module_common = NULL; + uint32 package_type; + bool magic_header_detected = false; + + if (!args) { + set_error_buf(error_buf, error_buf_size, + "WASM module load failed: null load arguments"); + return NULL; + } + + if (size < 4) { + set_error_buf(error_buf, error_buf_size, + "WASM module load failed: unexpected end"); + return NULL; + } + + package_type = get_package_type(buf, size); + if (package_type == Wasm_Module_Bytecode) { +#if WASM_ENABLE_INTERP != 0 + magic_header_detected = true; +#endif + } + else if (package_type == Wasm_Module_AoT) { +#if WASM_ENABLE_AOT != 0 + magic_header_detected = true; +#endif + } + if (!magic_header_detected) { + set_error_buf(error_buf, error_buf_size, + "WASM module load failed: magic header not detected"); + return NULL; + } + + if (package_type == Wasm_Module_Bytecode) { +#if WASM_ENABLE_INTERP != 0 + module_common = + (WASMModuleCommon *)wasm_load(buf, size, +#if WASM_ENABLE_MULTI_MODULE != 0 + true, +#endif + args, error_buf, error_buf_size); + if (module_common) + ((WASMModule *)module_common)->is_binary_freeable = + args->wasm_binary_freeable; +#endif + } + else if (package_type == Wasm_Module_AoT) { +#if WASM_ENABLE_AOT != 0 + module_common = (WASMModuleCommon *)aot_load_from_aot_file( + buf, size, args, error_buf, error_buf_size); + if (module_common) + ((AOTModule *)module_common)->is_binary_freeable = + args->wasm_binary_freeable; +#endif + } + + if (!module_common) { + LOG_DEBUG("WASM module load failed"); + return NULL; + } + + /*TODO: use file name as name and register with name? */ + return register_module_with_null_name(module_common, error_buf, + error_buf_size); +} + +bool +wasm_runtime_resolve_symbols(WASMModuleCommon *module) +{ +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + return wasm_resolve_symbols((WASMModule *)module); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + return aot_resolve_symbols((AOTModule *)module); + } +#endif + return false; +} + +WASMModuleCommon * +wasm_runtime_load(uint8 *buf, uint32 size, char *error_buf, + uint32 error_buf_size) +{ + LoadArgs args = { 0 }; + args.name = ""; + args.wasm_binary_freeable = false; + return wasm_runtime_load_ex(buf, size, &args, error_buf, error_buf_size); +} + +WASMModuleCommon * +wasm_runtime_load_from_sections(WASMSection *section_list, bool is_aot, + char *error_buf, uint32 error_buf_size) +{ + WASMModuleCommon *module_common; + + if (!is_aot) { +#if WASM_ENABLE_INTERP != 0 + module_common = (WASMModuleCommon *)wasm_load_from_sections( + section_list, error_buf, error_buf_size); + if (!module_common) { + LOG_DEBUG("WASM module load failed from sections"); + return NULL; + } + ((WASMModule *)module_common)->is_binary_freeable = true; + return register_module_with_null_name(module_common, error_buf, + error_buf_size); +#endif + } + else { +#if WASM_ENABLE_AOT != 0 + module_common = (WASMModuleCommon *)aot_load_from_sections( + section_list, error_buf, error_buf_size); + if (!module_common) { + LOG_DEBUG("WASM module load failed from sections"); + return NULL; + } + ((AOTModule *)module_common)->is_binary_freeable = true; + return register_module_with_null_name(module_common, error_buf, + error_buf_size); +#endif + } + +#if WASM_ENABLE_INTERP == 0 || WASM_ENABLE_AOT == 0 + set_error_buf(error_buf, error_buf_size, + "WASM module load failed: invalid section list type"); + return NULL; +#endif +} + +void +wasm_runtime_unload(WASMModuleCommon *module) +{ +#if WASM_ENABLE_MULTI_MODULE != 0 + /** + * since we will unload and free all module when runtime_destroy() + * we don't want users to unwillingly disrupt it + */ + return; +#endif + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + wasm_unload((WASMModule *)module); + return; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + aot_unload((AOTModule *)module); + return; + } +#endif +} + +uint32 +wasm_runtime_get_max_mem(uint32 max_memory_pages, uint32 module_init_page_count, + uint32 module_max_page_count) +{ + if (max_memory_pages == 0) { + /* Max memory not overwritten by runtime, use value from wasm module */ + return module_max_page_count; + } + + if (max_memory_pages < module_init_page_count) { + LOG_WARNING("Cannot override max memory with value lower than module " + "initial memory"); + return module_init_page_count; + } + + if (max_memory_pages > module_max_page_count) { + LOG_WARNING("Cannot override max memory with value greater than module " + "max memory"); + return module_max_page_count; + } + + return max_memory_pages; +} + +WASMModuleInstanceCommon * +wasm_runtime_instantiate_internal(WASMModuleCommon *module, + WASMModuleInstanceCommon *parent, + WASMExecEnv *exec_env_main, uint32 stack_size, + uint32 heap_size, uint32 max_memory_pages, + char *error_buf, uint32 error_buf_size) +{ +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) + return (WASMModuleInstanceCommon *)wasm_instantiate( + (WASMModule *)module, (WASMModuleInstance *)parent, exec_env_main, + stack_size, heap_size, max_memory_pages, error_buf, error_buf_size); +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) + return (WASMModuleInstanceCommon *)aot_instantiate( + (AOTModule *)module, (AOTModuleInstance *)parent, exec_env_main, + stack_size, heap_size, max_memory_pages, error_buf, error_buf_size); +#endif + set_error_buf(error_buf, error_buf_size, + "Instantiate module failed, invalid module type"); + return NULL; +} + +WASMModuleInstanceCommon * +wasm_runtime_instantiate(WASMModuleCommon *module, uint32 stack_size, + uint32 heap_size, char *error_buf, + uint32 error_buf_size) +{ + return wasm_runtime_instantiate_internal(module, NULL, NULL, stack_size, + heap_size, 0, error_buf, + error_buf_size); +} + +static void +instantiation_args_set_defaults(struct InstantiationArgs2 *args) +{ + memset(args, 0, sizeof(*args)); +} + +WASMModuleInstanceCommon * +wasm_runtime_instantiate_ex(WASMModuleCommon *module, + const InstantiationArgs *args, char *error_buf, + uint32 error_buf_size) +{ + struct InstantiationArgs2 v2; + instantiation_args_set_defaults(&v2); + v2.v1 = *args; + return wasm_runtime_instantiate_ex2(module, &v2, error_buf, error_buf_size); +} + +bool +wasm_runtime_instantiation_args_create(struct InstantiationArgs2 **p) +{ + struct InstantiationArgs2 *args = wasm_runtime_malloc(sizeof(*args)); + if (args == NULL) { + return false; + } + instantiation_args_set_defaults(args); + *p = args; + return true; +} + +void +wasm_runtime_instantiation_args_destroy(struct InstantiationArgs2 *p) +{ + wasm_runtime_free(p); +} + +void +wasm_runtime_instantiation_args_set_default_stack_size( + struct InstantiationArgs2 *p, uint32 v) +{ + p->v1.default_stack_size = v; +} + +void +wasm_runtime_instantiation_args_set_host_managed_heap_size( + struct InstantiationArgs2 *p, uint32 v) +{ + p->v1.host_managed_heap_size = v; +} + +void +wasm_runtime_instantiation_args_set_max_memory_pages( + struct InstantiationArgs2 *p, uint32 v) +{ + p->v1.max_memory_pages = v; +} + +WASMModuleInstanceCommon * +wasm_runtime_instantiate_ex2(WASMModuleCommon *module, + const struct InstantiationArgs2 *args, + char *error_buf, uint32 error_buf_size) +{ + return wasm_runtime_instantiate_internal( + module, NULL, NULL, args->v1.default_stack_size, + args->v1.host_managed_heap_size, args->v1.max_memory_pages, error_buf, + error_buf_size); +} + +void +wasm_runtime_deinstantiate_internal(WASMModuleInstanceCommon *module_inst, + bool is_sub_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + wasm_deinstantiate((WASMModuleInstance *)module_inst, is_sub_inst); + return; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + aot_deinstantiate((AOTModuleInstance *)module_inst, is_sub_inst); + return; + } +#endif +} + +bool +wasm_runtime_set_running_mode(wasm_module_inst_t module_inst, + RunningMode running_mode) +{ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return true; +#endif + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *module_inst_interp = + (WASMModuleInstance *)module_inst; + + return wasm_set_running_mode(module_inst_interp, running_mode); + } +#endif + + return false; +} + +RunningMode +wasm_runtime_get_running_mode(wasm_module_inst_t module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *module_inst_interp = + (WASMModuleInstance *)module_inst; + return module_inst_interp->e->running_mode; + } +#endif + + return Mode_Default; +} + +void +wasm_runtime_deinstantiate(WASMModuleInstanceCommon *module_inst) +{ + wasm_runtime_deinstantiate_internal(module_inst, false); +} + +WASMModuleCommon * +wasm_runtime_get_module(WASMModuleInstanceCommon *module_inst) +{ + return (WASMModuleCommon *)((WASMModuleInstance *)module_inst)->module; +} + +WASMExecEnv * +wasm_runtime_create_exec_env(WASMModuleInstanceCommon *module_inst, + uint32 stack_size) +{ + return wasm_exec_env_create(module_inst, stack_size); +} + +void +wasm_runtime_destroy_exec_env(WASMExecEnv *exec_env) +{ + wasm_exec_env_destroy(exec_env); +} + +#if WASM_ENABLE_COPY_CALL_STACK != 0 +uint32 +wasm_copy_callstack(const wasm_exec_env_t exec_env, WASMCApiFrame *buffer, + const uint32 length, const uint32 skip_n, char *error_buf, + uint32_t error_buf_size) +{ + /* + * Note for devs: please refrain from such modifications inside of + * wasm_copy_callstack to preserve async-signal-safety + * - any allocations/freeing memory + * - dereferencing any pointers other than: exec_env, exec_env->module_inst, + * exec_env->module_inst->module, pointers between stack's bottom and + * top_boundary For more details check wasm_copy_callstack in + * wasm_export.h + */ +#if WASM_ENABLE_DUMP_CALL_STACK + WASMModuleInstance *module_inst = + (WASMModuleInstance *)get_module_inst(exec_env); + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + return wasm_interp_copy_callstack(exec_env, buffer, length, skip_n, + error_buf, error_buf_size); + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + return aot_copy_callstack(exec_env, buffer, length, skip_n, error_buf, + error_buf_size); + } +#endif +#endif + char *err_msg = "No copy_callstack API was actually executed"; + strncpy(error_buf, err_msg, error_buf_size); + return 0; +} +#endif // WASM_ENABLE_COPY_CALL_STACK + +bool +wasm_runtime_init_thread_env(void) +{ +#ifdef BH_PLATFORM_WINDOWS + if (os_thread_env_init() != 0) + return false; +#endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!runtime_signal_init()) { +#ifdef BH_PLATFORM_WINDOWS + os_thread_env_destroy(); +#endif + return false; + } +#endif + +#if WASM_ENABLE_THREAD_MGR != 0 && defined(OS_ENABLE_WAKEUP_BLOCKING_OP) + os_end_blocking_op(); +#endif + + return true; +} + +void +wasm_runtime_destroy_thread_env(void) +{ +#ifdef OS_ENABLE_HW_BOUND_CHECK + runtime_signal_destroy(); +#endif + +#ifdef BH_PLATFORM_WINDOWS + os_thread_env_destroy(); +#endif +} + +bool +wasm_runtime_thread_env_inited(void) +{ +#ifdef BH_PLATFORM_WINDOWS + if (!os_thread_env_inited()) + return false; +#endif + +#if WASM_ENABLE_AOT != 0 +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!os_thread_signal_inited()) + return false; +#endif +#endif + return true; +} + +#if (WASM_ENABLE_MEMORY_PROFILING != 0) || (WASM_ENABLE_MEMORY_TRACING != 0) +void +wasm_runtime_dump_module_mem_consumption(const WASMModuleCommon *module) +{ + WASMModuleMemConsumption mem_conspn = { 0 }; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + wasm_get_module_mem_consumption((WASMModule *)module, &mem_conspn); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + aot_get_module_mem_consumption((AOTModule *)module, &mem_conspn); + } +#endif + + os_printf("WASM module memory consumption, total size: %u\n", + mem_conspn.total_size); + os_printf(" module struct size: %u\n", mem_conspn.module_struct_size); + os_printf(" types size: %u\n", mem_conspn.types_size); + os_printf(" imports size: %u\n", mem_conspn.imports_size); + os_printf(" funcs size: %u\n", mem_conspn.functions_size); + os_printf(" tables size: %u\n", mem_conspn.tables_size); + os_printf(" memories size: %u\n", mem_conspn.memories_size); + os_printf(" globals size: %u\n", mem_conspn.globals_size); + os_printf(" exports size: %u\n", mem_conspn.exports_size); + os_printf(" table segs size: %u\n", mem_conspn.table_segs_size); + os_printf(" data segs size: %u\n", mem_conspn.data_segs_size); + os_printf(" const strings size: %u\n", mem_conspn.const_strs_size); +#if WASM_ENABLE_AOT != 0 + os_printf(" aot code size: %u\n", mem_conspn.aot_code_size); +#endif +} + +void +wasm_runtime_dump_module_inst_mem_consumption( + const WASMModuleInstanceCommon *module_inst) +{ + WASMModuleInstMemConsumption mem_conspn = { 0 }; + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + wasm_get_module_inst_mem_consumption((WASMModuleInstance *)module_inst, + &mem_conspn); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + aot_get_module_inst_mem_consumption((AOTModuleInstance *)module_inst, + &mem_conspn); + } +#endif + + os_printf("WASM module inst memory consumption, total size: %lu\n", + mem_conspn.total_size); + os_printf(" module inst struct size: %u\n", + mem_conspn.module_inst_struct_size); + os_printf(" memories size: %lu\n", mem_conspn.memories_size); + os_printf(" app heap size: %u\n", mem_conspn.app_heap_size); + os_printf(" tables size: %u\n", mem_conspn.tables_size); + os_printf(" functions size: %u\n", mem_conspn.functions_size); + os_printf(" globals size: %u\n", mem_conspn.globals_size); + os_printf(" exports size: %u\n", mem_conspn.exports_size); +} + +void +wasm_runtime_dump_exec_env_mem_consumption(const WASMExecEnv *exec_env) +{ + uint32 total_size = + offsetof(WASMExecEnv, wasm_stack_u.bottom) + exec_env->wasm_stack_size; + + os_printf("Exec env memory consumption, total size: %u\n", total_size); + os_printf(" exec env struct size: %u\n", + offsetof(WASMExecEnv, wasm_stack_u.bottom)); +#if WASM_ENABLE_INTERP != 0 && WASM_ENABLE_FAST_INTERP == 0 + os_printf(" block addr cache size: %u\n", + sizeof(exec_env->block_addr_cache)); +#endif + os_printf(" stack size: %u\n", exec_env->wasm_stack_size); +} + +uint32 +gc_get_heap_highmark_size(void *heap); + +void +wasm_runtime_dump_mem_consumption(WASMExecEnv *exec_env) +{ + WASMModuleInstMemConsumption module_inst_mem_consps; + WASMModuleMemConsumption module_mem_consps; + WASMModuleInstanceCommon *module_inst_common; + WASMModuleCommon *module_common = NULL; + void *heap_handle = NULL; + uint32 app_heap_peak_size = 0; + uint32 max_aux_stack_used = -1; + uint64 total_size = 0; + + module_inst_common = exec_env->module_inst; +#if WASM_ENABLE_INTERP != 0 + if (module_inst_common->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *wasm_module_inst = + (WASMModuleInstance *)module_inst_common; + WASMModule *wasm_module = wasm_module_inst->module; + module_common = (WASMModuleCommon *)wasm_module; + if (wasm_module_inst->memories) { + heap_handle = wasm_module_inst->memories[0]->heap_handle; + } + wasm_get_module_inst_mem_consumption(wasm_module_inst, + &module_inst_mem_consps); + wasm_get_module_mem_consumption(wasm_module, &module_mem_consps); + if (wasm_module_inst->module->aux_stack_top_global_index != (uint32)-1) + max_aux_stack_used = wasm_module_inst->e->max_aux_stack_used; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst_common->module_type == Wasm_Module_AoT) { + AOTModuleInstance *aot_module_inst = + (AOTModuleInstance *)module_inst_common; + AOTModule *aot_module = (AOTModule *)aot_module_inst->module; + module_common = (WASMModuleCommon *)aot_module; + if (aot_module_inst->memories) { + AOTMemoryInstance **memories = aot_module_inst->memories; + heap_handle = memories[0]->heap_handle; + } + aot_get_module_inst_mem_consumption(aot_module_inst, + &module_inst_mem_consps); + aot_get_module_mem_consumption(aot_module, &module_mem_consps); + } +#endif + + bh_assert(module_common != NULL); + + if (heap_handle) { + app_heap_peak_size = gc_get_heap_highmark_size(heap_handle); + } + + total_size = offsetof(WASMExecEnv, wasm_stack_u.bottom) + + exec_env->wasm_stack_size + module_mem_consps.total_size + + module_inst_mem_consps.total_size; + + os_printf("\nMemory consumption summary (bytes):\n"); + wasm_runtime_dump_module_mem_consumption(module_common); + wasm_runtime_dump_module_inst_mem_consumption(module_inst_common); + wasm_runtime_dump_exec_env_mem_consumption(exec_env); + os_printf("\nTotal memory consumption of module, module inst and " + "exec env: %" PRIu64 "\n", + total_size); + os_printf("Total interpreter stack used: %u\n", + exec_env->max_wasm_stack_used); + + if (max_aux_stack_used != (uint32)-1) + os_printf("Total auxiliary stack used: %u\n", max_aux_stack_used); + else + os_printf("Total aux stack used: no enough info to profile\n"); + + /* + * Report the native stack usage estimation. + * + * Unlike the aux stack above, we report the amount unused + * because we don't know the stack "bottom". + * + * Note that this is just about what the runtime itself observed. + * It doesn't cover host func implementations, signal handlers, etc. + */ + if (exec_env->native_stack_top_min != (void *)UINTPTR_MAX) + os_printf("Native stack left: %zd\n", + exec_env->native_stack_top_min + - exec_env->native_stack_boundary); + else + os_printf("Native stack left: no enough info to profile\n"); + + os_printf("Total app heap used: %u\n", app_heap_peak_size); +} +#endif /* end of (WASM_ENABLE_MEMORY_PROFILING != 0) \ + || (WASM_ENABLE_MEMORY_TRACING != 0) */ + +#if WASM_ENABLE_PERF_PROFILING != 0 +void +wasm_runtime_dump_perf_profiling(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + wasm_dump_perf_profiling((WASMModuleInstance *)module_inst); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + aot_dump_perf_profiling((AOTModuleInstance *)module_inst); + } +#endif +} + +double +wasm_runtime_sum_wasm_exec_time(WASMModuleInstanceCommon *inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (inst->module_type == Wasm_Module_Bytecode) + return wasm_summarize_wasm_execute_time((WASMModuleInstance *)inst); +#endif + +#if WASM_ENABLE_AOT != 0 + if (inst->module_type == Wasm_Module_AoT) + return aot_summarize_wasm_execute_time((AOTModuleInstance *)inst); +#endif + + return 0.0; +} + +double +wasm_runtime_get_wasm_func_exec_time(WASMModuleInstanceCommon *inst, + const char *func_name) +{ +#if WASM_ENABLE_INTERP != 0 + if (inst->module_type == Wasm_Module_Bytecode) + return wasm_get_wasm_func_exec_time((WASMModuleInstance *)inst, + func_name); +#endif + +#if WASM_ENABLE_AOT != 0 + if (inst->module_type == Wasm_Module_AoT) + return aot_get_wasm_func_exec_time((AOTModuleInstance *)inst, + func_name); +#endif + + return 0.0; +} +#endif /* WASM_ENABLE_PERF_PROFILING != 0 */ + +WASMModuleInstanceCommon * +wasm_runtime_get_module_inst(WASMExecEnv *exec_env) +{ + return wasm_exec_env_get_module_inst(exec_env); +} + +void +wasm_runtime_set_module_inst(WASMExecEnv *exec_env, + WASMModuleInstanceCommon *const module_inst) +{ + wasm_exec_env_set_module_inst(exec_env, module_inst); +} + +bool +wasm_runtime_get_export_global_inst(WASMModuleInstanceCommon *const module_inst, + char const *name, + wasm_global_inst_t *global_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + const WASMModuleInstance *wasm_module_inst = + (const WASMModuleInstance *)module_inst; + const WASMModule *wasm_module = wasm_module_inst->module; + uint32 i; + for (i = 0; i < wasm_module->export_count; i++) { + const WASMExport *wasm_export = &wasm_module->exports[i]; + if ((wasm_export->kind == WASM_IMPORT_EXPORT_KIND_GLOBAL) + && !strcmp(wasm_export->name, name)) { + const WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)wasm_module_inst->e; + const WASMGlobalInstance *global = + &e->globals[wasm_export->index]; + global_inst->kind = val_type_to_val_kind(global->type); + global_inst->is_mutable = global->is_mutable; +#if WASM_ENABLE_MULTI_MODULE == 0 + global_inst->global_data = + wasm_module_inst->global_data + global->data_offset; +#else + global_inst->global_data = + global->import_global_inst + ? global->import_module_inst->global_data + + global->import_global_inst->data_offset + : wasm_module_inst->global_data + global->data_offset; +#endif + return true; + } + } + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + const AOTModuleInstance *aot_module_inst = + (AOTModuleInstance *)module_inst; + const AOTModule *aot_module = (AOTModule *)aot_module_inst->module; + uint32 i; + for (i = 0; i < aot_module->export_count; i++) { + const AOTExport *aot_export = &aot_module->exports[i]; + if ((aot_export->kind == WASM_IMPORT_EXPORT_KIND_GLOBAL) + && !strcmp(aot_export->name, name)) { + const AOTGlobal *global = + &aot_module->globals[aot_export->index]; + global_inst->kind = val_type_to_val_kind(global->type.val_type); + global_inst->is_mutable = global->type.is_mutable; + global_inst->global_data = + aot_module_inst->global_data + global->data_offset; + return true; + } + } + } +#endif + + return false; +} + +bool +wasm_runtime_get_export_table_inst(WASMModuleInstanceCommon *const module_inst, + char const *name, + wasm_table_inst_t *table_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + const WASMModuleInstance *wasm_module_inst = + (const WASMModuleInstance *)module_inst; + const WASMModule *wasm_module = wasm_module_inst->module; + uint32 i; + for (i = 0; i < wasm_module->export_count; i++) { + const WASMExport *wasm_export = &wasm_module->exports[i]; + if ((wasm_export->kind == WASM_IMPORT_EXPORT_KIND_TABLE) + && !strcmp(wasm_export->name, name)) { + const WASMTableInstance *wasm_table_inst = + wasm_module_inst->tables[wasm_export->index]; + table_inst->elem_kind = + val_type_to_val_kind(wasm_table_inst->elem_type); + table_inst->cur_size = wasm_table_inst->cur_size; + table_inst->max_size = wasm_table_inst->max_size; + table_inst->elems = (void *)wasm_table_inst->elems; + return true; + } + } + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + const AOTModuleInstance *aot_module_inst = + (AOTModuleInstance *)module_inst; + const AOTModule *aot_module = (AOTModule *)aot_module_inst->module; + uint32 i; + for (i = 0; i < aot_module->export_count; i++) { + const AOTExport *aot_export = &aot_module->exports[i]; + if ((aot_export->kind == WASM_IMPORT_EXPORT_KIND_TABLE) + && !strcmp(aot_export->name, name)) { + const AOTTableInstance *aot_table_inst = + aot_module_inst->tables[aot_export->index]; + table_inst->elem_kind = + val_type_to_val_kind(aot_table_inst->elem_type); + table_inst->cur_size = aot_table_inst->cur_size; + table_inst->max_size = aot_table_inst->max_size; + table_inst->elems = (void *)aot_table_inst->elems; + return true; + } + } + } +#endif + + return false; +} + +WASMFunctionInstanceCommon * +wasm_table_get_func_inst(struct WASMModuleInstanceCommon *const module_inst, + const wasm_table_inst_t *table_inst, uint32_t idx) +{ + if (!table_inst) { + bh_assert(0); + return NULL; + } + + if (idx >= table_inst->cur_size) { + bh_assert(0); + return NULL; + } + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + const WASMModuleInstance *wasm_module_inst = + (const WASMModuleInstance *)module_inst; + table_elem_type_t tbl_elem_val = + ((table_elem_type_t *)table_inst->elems)[idx]; + if (tbl_elem_val == NULL_REF) { + return NULL; + } + +#if WASM_ENABLE_GC == 0 + uint32 func_idx = (uint32)tbl_elem_val; +#else + uint32 func_idx = + wasm_func_obj_get_func_idx_bound((WASMFuncObjectRef)tbl_elem_val); +#endif + + bh_assert(func_idx < wasm_module_inst->e->function_count); + return wasm_module_inst->e->functions + func_idx; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstance *aot_module_inst = (AOTModuleInstance *)module_inst; + uint32 func_idx; + table_elem_type_t tbl_elem_val = + ((table_elem_type_t *)table_inst->elems)[idx]; + if (tbl_elem_val == NULL_REF) { + return NULL; + } + +#if WASM_ENABLE_GC == 0 + func_idx = (uint32)tbl_elem_val; +#else + func_idx = + wasm_func_obj_get_func_idx_bound((WASMFuncObjectRef)tbl_elem_val); +#endif + + return aot_get_function_instance(aot_module_inst, func_idx); + } +#endif + + return NULL; +} + +void * +wasm_runtime_get_function_attachment(WASMExecEnv *exec_env) +{ + return exec_env->attachment; +} + +void +wasm_runtime_set_user_data(WASMExecEnv *exec_env, void *user_data) +{ + exec_env->user_data = user_data; +} + +void * +wasm_runtime_get_user_data(WASMExecEnv *exec_env) +{ + return exec_env->user_data; +} + +void +wasm_runtime_set_native_stack_boundary(WASMExecEnv *exec_env, + uint8 *native_stack_boundary) +{ + exec_env->user_native_stack_boundary = native_stack_boundary; +} + +#ifdef OS_ENABLE_HW_BOUND_CHECK +void +wasm_runtime_access_exce_check_guard_page() +{ + if (exec_env_tls && exec_env_tls->handle == os_self_thread()) { + uint32 page_size = os_getpagesize(); + memset(exec_env_tls->exce_check_guard_page, 0, page_size); + } +} +#endif + +#if WASM_ENABLE_INSTRUCTION_METERING != 0 +void +wasm_runtime_set_instruction_count_limit(WASMExecEnv *exec_env, + int instructions_to_execute) +{ + exec_env->instructions_to_execute = instructions_to_execute; +} +#endif + +WASMFuncType * +wasm_runtime_get_function_type(const WASMFunctionInstanceCommon *function, + uint32 module_type) +{ + WASMFuncType *type = NULL; + +#if WASM_ENABLE_INTERP != 0 + if (module_type == Wasm_Module_Bytecode) { + WASMFunctionInstance *wasm_func = (WASMFunctionInstance *)function; + type = wasm_func->is_import_func ? wasm_func->u.func_import->func_type + : wasm_func->u.func->func_type; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_type == Wasm_Module_AoT) { + AOTFunctionInstance *aot_func = (AOTFunctionInstance *)function; + type = aot_func->is_import_func ? aot_func->u.func_import->func_type + : aot_func->u.func.func_type; + } +#endif + + return type; +} + +WASMFunctionInstanceCommon * +wasm_runtime_lookup_function(WASMModuleInstanceCommon *const module_inst, + const char *name) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return (WASMFunctionInstanceCommon *)wasm_lookup_function( + (const WASMModuleInstance *)module_inst, name); +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return (WASMFunctionInstanceCommon *)aot_lookup_function( + (const AOTModuleInstance *)module_inst, name); +#endif + return NULL; +} + +uint32 +wasm_func_get_param_count(WASMFunctionInstanceCommon *const func_inst, + WASMModuleInstanceCommon *const module_inst) +{ + WASMFuncType *type = + wasm_runtime_get_function_type(func_inst, module_inst->module_type); + bh_assert(type); + + return type->param_count; +} + +uint32 +wasm_func_get_result_count(WASMFunctionInstanceCommon *const func_inst, + WASMModuleInstanceCommon *const module_inst) +{ + WASMFuncType *type = + wasm_runtime_get_function_type(func_inst, module_inst->module_type); + bh_assert(type); + + return type->result_count; +} + +static uint8 +val_type_to_val_kind(uint8 value_type) +{ + switch (value_type) { + case VALUE_TYPE_I32: + return WASM_I32; + case VALUE_TYPE_I64: + return WASM_I64; + case VALUE_TYPE_F32: + return WASM_F32; + case VALUE_TYPE_F64: + return WASM_F64; + case VALUE_TYPE_V128: + return WASM_V128; + case VALUE_TYPE_FUNCREF: + return WASM_FUNCREF; + case VALUE_TYPE_EXTERNREF: + return WASM_EXTERNREF; + default: + bh_assert(0); + return 0; + } +} + +void +wasm_func_get_param_types(WASMFunctionInstanceCommon *const func_inst, + WASMModuleInstanceCommon *const module_inst, + wasm_valkind_t *param_types) +{ + WASMFuncType *type = + wasm_runtime_get_function_type(func_inst, module_inst->module_type); + uint32 i; + + bh_assert(type); + + for (i = 0; i < type->param_count; i++) { + param_types[i] = val_type_to_val_kind(type->types[i]); + } +} + +void +wasm_func_get_result_types(WASMFunctionInstanceCommon *const func_inst, + WASMModuleInstanceCommon *const module_inst, + wasm_valkind_t *result_types) +{ + WASMFuncType *type = + wasm_runtime_get_function_type(func_inst, module_inst->module_type); + uint32 i; + + bh_assert(type); + + for (i = 0; i < type->result_count; i++) { + result_types[i] = + val_type_to_val_kind(type->types[type->param_count + i]); + } +} + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 +/* (uintptr_t)externref -> (uint32)index */ +/* argv -> *ret_argv */ +static bool +wasm_runtime_prepare_call_function(WASMExecEnv *exec_env, + WASMFunctionInstanceCommon *function, + uint32 *argv, uint32 argc, uint32 **ret_argv, + uint32 *ret_argc_param, + uint32 *ret_argc_result) +{ + uint32 *new_argv = NULL, argv_i = 0, new_argv_i = 0, param_i = 0, + result_i = 0; + bool need_param_transform = false, need_result_transform = false; + uint64 size = 0; + WASMFuncType *func_type = wasm_runtime_get_function_type( + function, exec_env->module_inst->module_type); + + bh_assert(func_type); + + *ret_argc_param = func_type->param_cell_num; + *ret_argc_result = func_type->ret_cell_num; + for (param_i = 0; param_i < func_type->param_count; param_i++) { + if (VALUE_TYPE_EXTERNREF == func_type->types[param_i]) { + need_param_transform = true; + } + } + + for (result_i = 0; result_i < func_type->result_count; result_i++) { + if (VALUE_TYPE_EXTERNREF + == func_type->types[func_type->param_count + result_i]) { + need_result_transform = true; + } + } + + if (!need_param_transform && !need_result_transform) { + *ret_argv = argv; + return true; + } + + if (func_type->param_cell_num >= func_type->ret_cell_num) { + size = sizeof(uint32) * func_type->param_cell_num; + } + else { + size = sizeof(uint32) * func_type->ret_cell_num; + } + + if (!(new_argv = runtime_malloc(size, exec_env->module_inst, NULL, 0))) { + return false; + } + + if (!need_param_transform) { + bh_memcpy_s(new_argv, (uint32)size, argv, (uint32)size); + } + else { + for (param_i = 0; param_i < func_type->param_count && argv_i < argc + && new_argv_i < func_type->param_cell_num; + param_i++) { + uint8 param_type = func_type->types[param_i]; + if (VALUE_TYPE_EXTERNREF == param_type) { + void *externref_obj; + uint32 externref_index; + +#if UINTPTR_MAX == UINT32_MAX + externref_obj = (void *)argv[argv_i]; +#else + union { + uintptr_t val; + uint32 parts[2]; + } u; + + u.parts[0] = argv[argv_i]; + u.parts[1] = argv[argv_i + 1]; + externref_obj = (void *)u.val; +#endif + if (!wasm_externref_obj2ref(exec_env->module_inst, + externref_obj, &externref_index)) { + wasm_runtime_free(new_argv); + return false; + } + + new_argv[new_argv_i] = externref_index; + argv_i += sizeof(uintptr_t) / sizeof(uint32); + new_argv_i++; + } + else { + uint16 param_cell_num = wasm_value_type_cell_num(param_type); + uint32 param_size = sizeof(uint32) * param_cell_num; + bh_memcpy_s(new_argv + new_argv_i, param_size, argv + argv_i, + param_size); + argv_i += param_cell_num; + new_argv_i += param_cell_num; + } + } + } + + *ret_argv = new_argv; + return true; +} + +/* (uintptr_t)externref <- (uint32)index */ +/* argv <- new_argv */ +static bool +wasm_runtime_finalize_call_function(WASMExecEnv *exec_env, + WASMFunctionInstanceCommon *function, + uint32 *argv, uint32 argc, uint32 *ret_argv) +{ + uint32 argv_i = 0, result_i = 0, ret_argv_i = 0; + WASMFuncType *func_type; + + bh_assert((argv && ret_argv) || (argc == 0)); + + if (argv == ret_argv) { + /* no need to transform externref results */ + return true; + } + + func_type = wasm_runtime_get_function_type( + function, exec_env->module_inst->module_type); + bh_assert(func_type); + + for (result_i = 0; result_i < func_type->result_count && argv_i < argc; + result_i++) { + uint8 result_type = func_type->types[func_type->param_count + result_i]; + if (result_type == VALUE_TYPE_EXTERNREF) { + void *externref_obj; +#if UINTPTR_MAX != UINT32_MAX + union { + uintptr_t val; + uint32 parts[2]; + } u; +#endif + + if (!wasm_externref_ref2obj(argv[argv_i], &externref_obj)) { + wasm_runtime_free(argv); + return false; + } + +#if UINTPTR_MAX == UINT32_MAX + ret_argv[ret_argv_i] = (uintptr_t)externref_obj; +#else + u.val = (uintptr_t)externref_obj; + ret_argv[ret_argv_i] = u.parts[0]; + ret_argv[ret_argv_i + 1] = u.parts[1]; +#endif + argv_i += 1; + ret_argv_i += sizeof(uintptr_t) / sizeof(uint32); + } + else { + uint16 result_cell_num = wasm_value_type_cell_num(result_type); + uint32 result_size = sizeof(uint32) * result_cell_num; + bh_memcpy_s(ret_argv + ret_argv_i, result_size, argv + argv_i, + result_size); + argv_i += result_cell_num; + ret_argv_i += result_cell_num; + } + } + + wasm_runtime_free(argv); + return true; +} +#endif + +bool +wasm_runtime_call_wasm(WASMExecEnv *exec_env, + WASMFunctionInstanceCommon *function, uint32 argc, + uint32 argv[]) +{ + bool ret = false; + uint32 *new_argv = NULL, param_argc; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + uint32 result_argc = 0; +#endif + + if (!wasm_runtime_exec_env_check(exec_env)) { + LOG_ERROR("Invalid exec env stack info."); + return false; + } + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + if (!wasm_runtime_prepare_call_function(exec_env, function, argv, argc, + &new_argv, ¶m_argc, + &result_argc)) { + wasm_runtime_set_exception(exec_env->module_inst, + "the arguments conversion is failed"); + return false; + } +#else + new_argv = argv; + param_argc = argc; +#endif + +#if WASM_ENABLE_INTERP != 0 + if (exec_env->module_inst->module_type == Wasm_Module_Bytecode) + ret = wasm_call_function(exec_env, (WASMFunctionInstance *)function, + param_argc, new_argv); +#endif +#if WASM_ENABLE_AOT != 0 + if (exec_env->module_inst->module_type == Wasm_Module_AoT) + ret = aot_call_function(exec_env, (AOTFunctionInstance *)function, + param_argc, new_argv); +#endif + if (!ret) { + if (new_argv != argv) { + wasm_runtime_free(new_argv); + } + return false; + } + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + if (!wasm_runtime_finalize_call_function(exec_env, function, new_argv, + result_argc, argv)) { + wasm_runtime_set_exception(exec_env->module_inst, + "the result conversion is failed"); + return false; + } +#endif + + return ret; +} + +static void +parse_args_to_uint32_array(WASMFuncType *type, wasm_val_t *args, + uint32 *out_argv) +{ + uint32 i, p; + + for (i = 0, p = 0; i < type->param_count; i++) { + switch (args[i].kind) { + case WASM_I32: + out_argv[p++] = args[i].of.i32; + break; + case WASM_I64: + { + union { + uint64 val; + uint32 parts[2]; + } u; + u.val = args[i].of.i64; + out_argv[p++] = u.parts[0]; + out_argv[p++] = u.parts[1]; + break; + } + case WASM_F32: + { + union { + float32 val; + uint32 part; + } u; + u.val = args[i].of.f32; + out_argv[p++] = u.part; + break; + } + case WASM_F64: + { + union { + float64 val; + uint32 parts[2]; + } u; + u.val = args[i].of.f64; + out_argv[p++] = u.parts[0]; + out_argv[p++] = u.parts[1]; + break; + } + case WASM_V128: + { + bh_assert(0); + break; + } +#if WASM_ENABLE_REF_TYPES != 0 +#if WASM_ENABLE_GC == 0 + case WASM_FUNCREF: + { + out_argv[p++] = args[i].of.i32; + break; + } +#else + case WASM_FUNCREF: +#endif + case WASM_EXTERNREF: + { +#if UINTPTR_MAX == UINT32_MAX + out_argv[p++] = args[i].of.foreign; +#else + union { + uintptr_t val; + uint32 parts[2]; + } u; + + u.val = (uintptr_t)args[i].of.foreign; + out_argv[p++] = u.parts[0]; + out_argv[p++] = u.parts[1]; +#endif + break; + } +#endif + default: + bh_assert(0); + break; + } + } +} + +static void +parse_uint32_array_to_results(WASMFuncType *type, uint32 *argv, + wasm_val_t *out_results) +{ + uint32 i, p; + + for (i = 0, p = 0; i < type->result_count; i++) { + switch (type->types[type->param_count + i]) { + case VALUE_TYPE_I32: + out_results[i].kind = WASM_I32; + out_results[i].of.i32 = (int32)argv[p++]; + break; + case VALUE_TYPE_I64: + { + union { + uint64 val; + uint32 parts[2]; + } u; + u.parts[0] = argv[p++]; + u.parts[1] = argv[p++]; + out_results[i].kind = WASM_I64; + out_results[i].of.i64 = u.val; + break; + } + case VALUE_TYPE_F32: + { + union { + float32 val; + uint32 part; + } u; + u.part = argv[p++]; + out_results[i].kind = WASM_F32; + out_results[i].of.f32 = u.val; + break; + } + case VALUE_TYPE_F64: + { + union { + float64 val; + uint32 parts[2]; + } u; + u.parts[0] = argv[p++]; + u.parts[1] = argv[p++]; + out_results[i].kind = WASM_F64; + out_results[i].of.f64 = u.val; + break; + } + case VALUE_TYPE_V128: + { + bh_assert(0); + break; + } +#if WASM_ENABLE_REF_TYPES != 0 +#if WASM_ENABLE_GC == 0 + case VALUE_TYPE_FUNCREF: + { + out_results[i].kind = WASM_I32; + out_results[i].of.i32 = (int32)argv[p++]; + break; + } + case VALUE_TYPE_EXTERNREF: +#else + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#endif /* end of WASM_ENABLE_GC == 0 */ + { +#if UINTPTR_MAX == UINT32_MAX + out_results[i].kind = WASM_EXTERNREF; + out_results[i].of.foreign = (uintptr_t)argv[p++]; +#else + union { + uintptr_t val; + uint32 parts[2]; + } u; + u.parts[0] = argv[p++]; + u.parts[1] = argv[p++]; + out_results[i].kind = WASM_EXTERNREF; + out_results[i].of.foreign = u.val; +#endif + break; + } +#endif /* end of WASM_ENABLE_REF_TYPES != 0 */ + default: + bh_assert(0); + break; + } + } +} + +bool +wasm_runtime_call_wasm_a(WASMExecEnv *exec_env, + WASMFunctionInstanceCommon *function, + uint32 num_results, wasm_val_t results[], + uint32 num_args, wasm_val_t args[]) +{ + uint32 argc, argv_buf[16] = { 0 }, *argv = argv_buf, cell_num, module_type; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + uint32 i, param_size_in_double_world = 0, result_size_in_double_world = 0; +#endif + uint64 total_size; + WASMFuncType *type; + bool ret = false; + + module_type = exec_env->module_inst->module_type; + type = wasm_runtime_get_function_type(function, module_type); + + if (!type) { + LOG_ERROR("Function type get failed, WAMR Interpreter and AOT must be " + "enabled at least one."); + goto fail1; + } + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + for (i = 0; i < type->param_count; i++) { + param_size_in_double_world += + wasm_value_type_cell_num_outside(type->types[i]); + } + for (i = 0; i < type->result_count; i++) { + result_size_in_double_world += wasm_value_type_cell_num_outside( + type->types[type->param_count + i]); + } + argc = param_size_in_double_world; + cell_num = (argc >= result_size_in_double_world) + ? argc + : result_size_in_double_world; +#else + argc = type->param_cell_num; + cell_num = (argc > type->ret_cell_num) ? argc : type->ret_cell_num; +#endif + + if (num_results != type->result_count) { + LOG_ERROR( + "The result value number does not match the function declaration."); + goto fail1; + } + + if (num_args != type->param_count) { + LOG_ERROR("The argument value number does not match the function " + "declaration."); + goto fail1; + } + + total_size = sizeof(uint32) * (uint64)(cell_num > 2 ? cell_num : 2); + if (total_size > sizeof(argv_buf)) { + if (!(argv = + runtime_malloc(total_size, exec_env->module_inst, NULL, 0))) { + goto fail1; + } + } + + parse_args_to_uint32_array(type, args, argv); + if (!(ret = wasm_runtime_call_wasm(exec_env, function, argc, argv))) + goto fail2; + + parse_uint32_array_to_results(type, argv, results); + +fail2: + if (argv != argv_buf) + wasm_runtime_free(argv); +fail1: + return ret; +} + +bool +wasm_runtime_call_wasm_v(WASMExecEnv *exec_env, + WASMFunctionInstanceCommon *function, + uint32 num_results, wasm_val_t results[], + uint32 num_args, ...) +{ + wasm_val_t args_buf[8] = { 0 }, *args = args_buf; + WASMFuncType *type = NULL; + bool ret = false; + uint64 total_size; + uint32 i = 0, module_type; + va_list vargs; + + module_type = exec_env->module_inst->module_type; + type = wasm_runtime_get_function_type(function, module_type); + + if (!type) { + LOG_ERROR("Function type get failed, WAMR Interpreter and AOT " + "must be enabled at least one."); + goto fail1; + } + + if (num_args != type->param_count) { + LOG_ERROR("The argument value number does not match the " + "function declaration."); + goto fail1; + } + + total_size = sizeof(wasm_val_t) * (uint64)num_args; + if (total_size > sizeof(args_buf)) { + if (!(args = + runtime_malloc(total_size, exec_env->module_inst, NULL, 0))) { + goto fail1; + } + } + + va_start(vargs, num_args); + for (i = 0; i < num_args; i++) { + switch (type->types[i]) { + case VALUE_TYPE_I32: + args[i].kind = WASM_I32; + args[i].of.i32 = va_arg(vargs, uint32); + break; + case VALUE_TYPE_I64: + args[i].kind = WASM_I64; + args[i].of.i64 = va_arg(vargs, uint64); + break; + case VALUE_TYPE_F32: + args[i].kind = WASM_F32; + args[i].of.f32 = (float32)va_arg(vargs, float64); + break; + case VALUE_TYPE_F64: + args[i].kind = WASM_F64; + args[i].of.f64 = va_arg(vargs, float64); + break; + case VALUE_TYPE_V128: + bh_assert(0); + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + { + args[i].kind = WASM_FUNCREF; + args[i].of.i32 = va_arg(vargs, uint32); + break; + } + case VALUE_TYPE_EXTERNREF: + { + args[i].kind = WASM_EXTERNREF; + args[i].of.foreign = va_arg(vargs, uintptr_t); + break; + } +#endif + default: + bh_assert(0); + break; + } + } + va_end(vargs); + + ret = wasm_runtime_call_wasm_a(exec_env, function, num_results, results, + num_args, args); + if (args != args_buf) + wasm_runtime_free(args); + +fail1: + return ret; +} + +bool +wasm_runtime_create_exec_env_singleton( + WASMModuleInstanceCommon *module_inst_comm) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMExecEnv *exec_env = NULL; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + if (module_inst->exec_env_singleton) { + return true; + } + + exec_env = wasm_exec_env_create(module_inst_comm, + module_inst->default_wasm_stack_size); + if (exec_env) + module_inst->exec_env_singleton = exec_env; + + return exec_env ? true : false; +} + +WASMExecEnv * +wasm_runtime_get_exec_env_singleton(WASMModuleInstanceCommon *module_inst_comm) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + if (!module_inst->exec_env_singleton) { + wasm_runtime_create_exec_env_singleton(module_inst_comm); + } + return module_inst->exec_env_singleton; +} + +static void +wasm_set_exception_local(WASMModuleInstance *module_inst, const char *exception) +{ + exception_lock(module_inst); + if (exception) { + snprintf(module_inst->cur_exception, sizeof(module_inst->cur_exception), + "Exception: %s", exception); + } + else { + module_inst->cur_exception[0] = '\0'; + } + exception_unlock(module_inst); +} + +void +wasm_set_exception(WASMModuleInstance *module_inst, const char *exception) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + WASMExecEnv *exec_env = + wasm_clusters_search_exec_env((WASMModuleInstanceCommon *)module_inst); + if (exec_env) { + wasm_cluster_set_exception(exec_env, exception); + } + else { + wasm_set_exception_local(module_inst, exception); + } +#else + wasm_set_exception_local(module_inst, exception); +#endif +} + +/* clang-format off */ +static const char *exception_msgs[] = { + "unreachable", /* EXCE_UNREACHABLE */ + "allocate memory failed", /* EXCE_OUT_OF_MEMORY */ + "out of bounds memory access", /* EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS */ + "integer overflow", /* EXCE_INTEGER_OVERFLOW */ + "integer divide by zero", /* EXCE_INTEGER_DIVIDE_BY_ZERO */ + "invalid conversion to integer", /* EXCE_INVALID_CONVERSION_TO_INTEGER */ + "indirect call type mismatch", /* EXCE_INVALID_FUNCTION_TYPE_INDEX */ + "invalid function index", /* EXCE_INVALID_FUNCTION_INDEX */ + "undefined element", /* EXCE_UNDEFINED_ELEMENT */ + "uninitialized element", /* EXCE_UNINITIALIZED_ELEMENT */ + "failed to call unlinked import function", /* EXCE_CALL_UNLINKED_IMPORT_FUNC */ + "native stack overflow", /* EXCE_NATIVE_STACK_OVERFLOW */ + "unaligned atomic", /* EXCE_UNALIGNED_ATOMIC */ + "wasm auxiliary stack overflow", /* EXCE_AUX_STACK_OVERFLOW */ + "wasm auxiliary stack underflow", /* EXCE_AUX_STACK_UNDERFLOW */ + "out of bounds table access", /* EXCE_OUT_OF_BOUNDS_TABLE_ACCESS */ + "wasm operand stack overflow", /* EXCE_OPERAND_STACK_OVERFLOW */ + "failed to compile fast jit function", /* EXCE_FAILED_TO_COMPILE_FAST_JIT_FUNC */ + /* GC related exceptions */ + "null function reference", /* EXCE_NULL_FUNC_OBJ */ + "null structure reference", /* EXCE_NULL_STRUCT_OBJ */ + "null array reference", /* EXCE_NULL_ARRAY_OBJ */ + "null i31 reference", /* EXCE_NULL_I31_OBJ */ + "null reference", /* EXCE_NULL_REFERENCE */ + "create rtt type failed", /* EXCE_FAILED_TO_CREATE_RTT_TYPE */ + "create struct object failed", /* EXCE_FAILED_TO_CREATE_STRUCT_OBJ */ + "create array object failed", /* EXCE_FAILED_TO_CREATE_ARRAY_OBJ */ + "create externref object failed", /* EXCE_FAILED_TO_CREATE_EXTERNREF_OBJ */ + "cast failure", /* EXCE_CAST_FAILURE */ + "out of bounds array access", /* EXCE_ARRAY_IDX_OOB */ + /* stringref related exceptions */ + "create string object failed", /* EXCE_FAILED_TO_CREATE_STRING */ + "create stringref failed", /* EXCE_FAILED_TO_CREATE_STRINGREF */ + "create stringview failed", /* EXCE_FAILED_TO_CREATE_STRINGVIEW */ + "encode failed", /* EXCE_FAILED_TO_ENCODE_STRING */ + "", /* EXCE_ALREADY_THROWN */ +}; +/* clang-format on */ + +void +wasm_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id) +{ + if (id < EXCE_NUM) + wasm_set_exception(module_inst, exception_msgs[id]); + else + wasm_set_exception(module_inst, "unknown exception"); +} + +const char * +wasm_get_exception(WASMModuleInstance *module_inst) +{ + if (module_inst->cur_exception[0] == '\0') + return NULL; + else + return module_inst->cur_exception; +} + +bool +wasm_copy_exception(WASMModuleInstance *module_inst, char *exception_buf) +{ + bool has_exception = false; + + exception_lock(module_inst); + if (module_inst->cur_exception[0] != '\0') { + /* NULL is passed if the caller is not interested in getting the + * exception content, but only in knowing if an exception has been + * raised + */ + if (exception_buf != NULL) + bh_memcpy_s(exception_buf, sizeof(module_inst->cur_exception), + module_inst->cur_exception, + sizeof(module_inst->cur_exception)); + has_exception = true; + } + exception_unlock(module_inst); + + return has_exception; +} + +void +wasm_runtime_set_exception(WASMModuleInstanceCommon *module_inst_comm, + const char *exception) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + wasm_set_exception(module_inst, exception); +} + +const char * +wasm_runtime_get_exception(WASMModuleInstanceCommon *module_inst_comm) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + return wasm_get_exception(module_inst); +} + +bool +wasm_runtime_copy_exception(WASMModuleInstanceCommon *module_inst_comm, + char *exception_buf) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + return wasm_copy_exception(module_inst, exception_buf); +} + +void +wasm_runtime_clear_exception(WASMModuleInstanceCommon *module_inst_comm) +{ + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + wasm_runtime_set_exception(module_inst_comm, NULL); +} + +void +wasm_runtime_terminate(WASMModuleInstanceCommon *module_inst_comm) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + wasm_set_exception(module_inst, "terminated by user"); +} + +void +wasm_runtime_set_custom_data_internal( + WASMModuleInstanceCommon *module_inst_comm, void *custom_data) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + module_inst->custom_data = custom_data; +} + +void +wasm_runtime_set_custom_data(WASMModuleInstanceCommon *module_inst, + void *custom_data) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_spread_custom_data(module_inst, custom_data); +#else + wasm_runtime_set_custom_data_internal(module_inst, custom_data); +#endif +} + +void * +wasm_runtime_get_custom_data(WASMModuleInstanceCommon *module_inst_comm) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + return module_inst->custom_data; +} + +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 +void +wasm_runtime_set_bounds_checks(WASMModuleInstanceCommon *module_inst, + bool enable) +{ + /* Always disable bounds checks if hw bounds checks is enabled */ +#ifdef OS_ENABLE_HW_BOUND_CHECK + enable = false; +#endif +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + ((WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e) + ->common.disable_bounds_checks = enable ? false : true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + ((AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e) + ->common.disable_bounds_checks = enable ? false : true; + } +#endif +} + +bool +wasm_runtime_is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst) +{ + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + return !((WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst) + ->e) + ->common.disable_bounds_checks; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + return !((AOTModuleInstanceExtra *)((WASMModuleInstance *)module_inst) + ->e) + ->common.disable_bounds_checks; + } +#endif + + return true; +} +#endif + +uint64 +wasm_runtime_module_malloc_internal(WASMModuleInstanceCommon *module_inst, + WASMExecEnv *exec_env, uint64 size, + void **p_native_addr) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return wasm_module_malloc_internal((WASMModuleInstance *)module_inst, + exec_env, size, p_native_addr); +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return aot_module_malloc_internal((AOTModuleInstance *)module_inst, + exec_env, size, p_native_addr); +#endif + return 0; +} + +uint64 +wasm_runtime_module_realloc_internal(WASMModuleInstanceCommon *module_inst, + WASMExecEnv *exec_env, uint64 ptr, + uint64 size, void **p_native_addr) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return wasm_module_realloc_internal((WASMModuleInstance *)module_inst, + exec_env, ptr, size, p_native_addr); +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return aot_module_realloc_internal((AOTModuleInstance *)module_inst, + exec_env, ptr, size, p_native_addr); +#endif + return 0; +} + +void +wasm_runtime_module_free_internal(WASMModuleInstanceCommon *module_inst, + WASMExecEnv *exec_env, uint64 ptr) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + wasm_module_free_internal((WASMModuleInstance *)module_inst, exec_env, + ptr); + return; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + aot_module_free_internal((AOTModuleInstance *)module_inst, exec_env, + ptr); + return; + } +#endif +} + +uint64 +wasm_runtime_module_malloc(WASMModuleInstanceCommon *module_inst, uint64 size, + void **p_native_addr) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return wasm_module_malloc((WASMModuleInstance *)module_inst, size, + p_native_addr); +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return aot_module_malloc((AOTModuleInstance *)module_inst, size, + p_native_addr); +#endif + return 0; +} + +uint64 +wasm_runtime_module_realloc(WASMModuleInstanceCommon *module_inst, uint64 ptr, + uint64 size, void **p_native_addr) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return wasm_module_realloc((WASMModuleInstance *)module_inst, ptr, size, + p_native_addr); +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return aot_module_realloc((AOTModuleInstance *)module_inst, ptr, size, + p_native_addr); +#endif + return 0; +} + +void +wasm_runtime_module_free(WASMModuleInstanceCommon *module_inst, uint64 ptr) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + wasm_module_free((WASMModuleInstance *)module_inst, ptr); + return; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + aot_module_free((AOTModuleInstance *)module_inst, ptr); + return; + } +#endif +} + +uint64 +wasm_runtime_module_dup_data(WASMModuleInstanceCommon *module_inst, + const char *src, uint64 size) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + return wasm_module_dup_data((WASMModuleInstance *)module_inst, src, + size); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + return aot_module_dup_data((AOTModuleInstance *)module_inst, src, size); + } +#endif + return 0; +} + +#if WASM_ENABLE_LIBC_WASI != 0 + +void +wasi_args_set_defaults(WASIArguments *args) +{ + memset(args, 0, sizeof(*args)); +#if WASM_ENABLE_UVWASI == 0 + args->stdio[0] = os_invalid_raw_handle(); + args->stdio[1] = os_invalid_raw_handle(); + args->stdio[2] = os_invalid_raw_handle(); +#else + args->stdio[0] = os_get_invalid_handle(); + args->stdio[1] = os_get_invalid_handle(); + args->stdio[2] = os_get_invalid_handle(); +#endif /* WASM_ENABLE_UVWASI == 0 */ +} + +static WASIArguments * +get_wasi_args_from_module(wasm_module_t module) +{ + WASIArguments *wasi_args = NULL; + +#if WASM_ENABLE_INTERP != 0 || WASM_ENABLE_JIT != 0 + if (module->module_type == Wasm_Module_Bytecode) + wasi_args = &((WASMModule *)module)->wasi_args; +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) + wasi_args = &((AOTModule *)module)->wasi_args; +#endif + + return wasi_args; +} + +void +wasm_runtime_set_wasi_args_ex(WASMModuleCommon *module, const char *dir_list[], + uint32 dir_count, const char *map_dir_list[], + uint32 map_dir_count, const char *env_list[], + uint32 env_count, char *argv[], int argc, + int64 stdinfd, int64 stdoutfd, int64 stderrfd) +{ + WASIArguments *wasi_args = get_wasi_args_from_module(module); + + bh_assert(wasi_args); + + wasi_args->dir_list = dir_list; + wasi_args->dir_count = dir_count; + wasi_args->map_dir_list = map_dir_list; + wasi_args->map_dir_count = map_dir_count; + wasi_args->env = env_list; + wasi_args->env_count = env_count; + wasi_args->argv = argv; + wasi_args->argc = (uint32)argc; + wasi_args->stdio[0] = (os_raw_file_handle)stdinfd; + wasi_args->stdio[1] = (os_raw_file_handle)stdoutfd; + wasi_args->stdio[2] = (os_raw_file_handle)stderrfd; + +#if WASM_ENABLE_MULTI_MODULE != 0 +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + wasm_propagate_wasi_args((WASMModule *)module); + } +#endif +#endif +} + +void +wasm_runtime_set_wasi_args(WASMModuleCommon *module, const char *dir_list[], + uint32 dir_count, const char *map_dir_list[], + uint32 map_dir_count, const char *env_list[], + uint32 env_count, char *argv[], int argc) +{ + wasm_runtime_set_wasi_args_ex(module, dir_list, dir_count, map_dir_list, + map_dir_count, env_list, env_count, argv, + argc, -1, -1, -1); +} + +void +wasm_runtime_set_wasi_addr_pool(wasm_module_t module, const char *addr_pool[], + uint32 addr_pool_size) +{ + WASIArguments *wasi_args = get_wasi_args_from_module(module); + + if (wasi_args) { + wasi_args->addr_pool = addr_pool; + wasi_args->addr_count = addr_pool_size; + } +} + +void +wasm_runtime_set_wasi_ns_lookup_pool(wasm_module_t module, + const char *ns_lookup_pool[], + uint32 ns_lookup_pool_size) +{ + WASIArguments *wasi_args = get_wasi_args_from_module(module); + + if (wasi_args) { + wasi_args->ns_lookup_pool = ns_lookup_pool; + wasi_args->ns_lookup_count = ns_lookup_pool_size; + } +} + +#if WASM_ENABLE_UVWASI == 0 +static bool +copy_string_array(const char *array[], uint32 array_size, char **buf_ptr, + char ***list_ptr, uint64 *out_buf_size) +{ + uint64 buf_size = 0, total_size; + uint32 buf_offset = 0, i; + char *buf = NULL, **list = NULL; + + for (i = 0; i < array_size; i++) + buf_size += strlen(array[i]) + 1; + + /* We add +1 to generate null-terminated array of strings */ + total_size = sizeof(char *) * ((uint64)array_size + 1); + if (total_size >= UINT32_MAX + /* total_size must be larger than 0, don' check it again */ + || !(list = wasm_runtime_malloc((uint32)total_size)) + || buf_size >= UINT32_MAX + || (buf_size > 0 && !(buf = wasm_runtime_malloc((uint32)buf_size)))) { + + if (buf) + wasm_runtime_free(buf); + if (list) + wasm_runtime_free(list); + return false; + } + + for (i = 0; i < array_size; i++) { + list[i] = buf + buf_offset; + bh_strcpy_s(buf + buf_offset, (uint32)buf_size - buf_offset, array[i]); + buf_offset += (uint32)(strlen(array[i]) + 1); + } + list[array_size] = NULL; + + *list_ptr = list; + *buf_ptr = buf; + if (out_buf_size) + *out_buf_size = buf_size; + + return true; +} + +bool +wasm_runtime_init_wasi(WASMModuleInstanceCommon *module_inst, + const char *dir_list[], uint32 dir_count, + const char *map_dir_list[], uint32 map_dir_count, + const char *env[], uint32 env_count, + const char *addr_pool[], uint32 addr_pool_size, + const char *ns_lookup_pool[], uint32 ns_lookup_pool_size, + char *argv[], uint32 argc, os_raw_file_handle stdinfd, + os_raw_file_handle stdoutfd, os_raw_file_handle stderrfd, + char *error_buf, uint32 error_buf_size) +{ + WASIContext *wasi_ctx; + char *argv_buf = NULL; + char **argv_list = NULL; + char *env_buf = NULL; + char **env_list = NULL; + char *ns_lookup_buf = NULL; + char **ns_lookup_list = NULL; + uint64 argv_buf_size = 0, env_buf_size = 0; + struct fd_table *curfds = NULL; + struct fd_prestats *prestats = NULL; + struct argv_environ_values *argv_environ = NULL; + struct addr_pool *apool = NULL; + bool fd_table_inited = false, fd_prestats_inited = false; + bool argv_environ_inited = false; + bool addr_pool_inited = false; + __wasi_fd_t wasm_fd = 3; + os_file_handle file_handle; + char *path, resolved_path[PATH_MAX]; + uint32 i; + + if (!(wasi_ctx = runtime_malloc(sizeof(WASIContext), NULL, error_buf, + error_buf_size))) { + return false; + } + + wasm_runtime_set_wasi_ctx(module_inst, wasi_ctx); + + /* process argv[0], trip the path and suffix, only keep the program name + */ + if (!copy_string_array((const char **)argv, argc, &argv_buf, &argv_list, + &argv_buf_size)) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: allocate memory failed"); + goto fail; + } + + if (!copy_string_array(env, env_count, &env_buf, &env_list, + &env_buf_size)) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: allocate memory failed"); + goto fail; + } + + if (!(curfds = wasm_runtime_malloc(sizeof(struct fd_table))) + || !(prestats = wasm_runtime_malloc(sizeof(struct fd_prestats))) + || !(argv_environ = + wasm_runtime_malloc(sizeof(struct argv_environ_values))) + || !(apool = wasm_runtime_malloc(sizeof(struct addr_pool)))) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: allocate memory failed"); + goto fail; + } + + if (!fd_table_init(curfds)) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: " + "init fd table failed"); + goto fail; + } + fd_table_inited = true; + + if (!fd_prestats_init(prestats)) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: " + "init fd prestats failed"); + goto fail; + } + fd_prestats_inited = true; + + if (!argv_environ_init(argv_environ, argv_buf, argv_buf_size, argv_list, + argc, env_buf, env_buf_size, env_list, env_count)) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: " + "init argument environment failed"); + goto fail; + } + argv_environ_inited = true; + + if (!addr_pool_init(apool)) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: " + "init the address pool failed"); + goto fail; + } + addr_pool_inited = true; + + os_file_handle stdin_file_handle = os_convert_stdin_handle(stdinfd); + os_file_handle stdout_file_handle = os_convert_stdout_handle(stdoutfd); + os_file_handle stderr_file_handle = os_convert_stderr_handle(stderrfd); + + if (!os_is_handle_valid(&stdin_file_handle) + || !os_is_handle_valid(&stdout_file_handle) + || !os_is_handle_valid(&stderr_file_handle)) + goto fail; + + /* Prepopulate curfds with stdin, stdout, and stderr file descriptors. */ + if (!fd_table_insert_existing(curfds, 0, stdin_file_handle, true) + || !fd_table_insert_existing(curfds, 1, stdout_file_handle, true) + || !fd_table_insert_existing(curfds, 2, stderr_file_handle, true)) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: init fd table failed"); + goto fail; + } + + wasm_fd = 3; + for (i = 0; i < dir_count; i++, wasm_fd++) { + path = os_realpath(dir_list[i], resolved_path); + if (!path) { + if (error_buf) + snprintf(error_buf, error_buf_size, + "error while pre-opening directory %s: %d\n", + dir_list[i], errno); + goto fail; + } + + __wasi_errno_t error = os_open_preopendir(path, &file_handle); + + if (error != __WASI_ESUCCESS) { + if (error_buf) + snprintf(error_buf, error_buf_size, + "error while pre-opening directory %s: %d\n", + dir_list[i], error); + goto fail; + } + + if (!fd_table_insert_existing(curfds, wasm_fd, file_handle, false)) { + if (error_buf) + snprintf(error_buf, error_buf_size, + "error inserting preopen fd %u (directory %s) into fd " + "table", + (unsigned int)wasm_fd, dir_list[i]); + goto fail; + } + + if (!fd_prestats_insert(prestats, dir_list[i], wasm_fd)) { + if (error_buf) + snprintf(error_buf, error_buf_size, + "error inserting preopen fd %u (directory %s) into " + "prestats table", + (unsigned int)wasm_fd, dir_list[i]); + goto fail; + } + } + + for (i = 0; i < map_dir_count; i++, wasm_fd++) { + char mapping_copy_buf[256]; + char *mapping_copy = mapping_copy_buf; + char *map_mapped = NULL, *map_host = NULL; + const unsigned long max_len = + (unsigned long)strlen(map_dir_list[i]) * 2 + 3; + + /* Allocation limit for runtime environments with reduced stack size */ + if (max_len > 256) { + if (!(mapping_copy = wasm_runtime_malloc(max_len))) { + snprintf(error_buf, error_buf_size, + "error while allocating for directory mapping\n"); + goto fail; + } + } + + bh_memcpy_s(mapping_copy, max_len, map_dir_list[i], + (uint32)(strlen(map_dir_list[i]) + 1)); + + const char *delim = "::"; + char *delim_pos = strstr(mapping_copy, delim); + if (delim_pos) { + *delim_pos = '\0'; + map_mapped = mapping_copy; + map_host = delim_pos + strlen(delim); + } + + if (!map_mapped || !map_host) { + if (error_buf) + snprintf(error_buf, error_buf_size, + "error while pre-opening mapped directory: " + "invalid map\n"); + if (mapping_copy != mapping_copy_buf) + wasm_runtime_free(mapping_copy); + goto fail; + } + + path = os_realpath(map_host, resolved_path); + if (!path) { + if (error_buf) + snprintf(error_buf, error_buf_size, + "error while pre-opening mapped directory %s: %d\n", + map_host, errno); + if (mapping_copy != mapping_copy_buf) + wasm_runtime_free(mapping_copy); + goto fail; + } + + __wasi_errno_t error = os_open_preopendir(path, &file_handle); + if (error != __WASI_ESUCCESS) { + if (error_buf) + snprintf(error_buf, error_buf_size, + "error while pre-opening mapped directory %s: %d\n", + map_host, errno); + if (mapping_copy != mapping_copy_buf) + wasm_runtime_free(mapping_copy); + goto fail; + } + + if (!fd_table_insert_existing(curfds, wasm_fd, file_handle, false) + || !fd_prestats_insert(prestats, map_mapped, wasm_fd)) { + if (error_buf) + snprintf(error_buf, error_buf_size, + "error while pre-opening mapped directory %s: " + "insertion failed\n", + dir_list[i]); + if (mapping_copy != mapping_copy_buf) + wasm_runtime_free(mapping_copy); + goto fail; + } + + if (mapping_copy != mapping_copy_buf) + wasm_runtime_free(mapping_copy); + } + + /* addr_pool(textual) -> apool */ + for (i = 0; i < addr_pool_size; i++) { + char *cp, *address, *mask; + bool ret = false; + + cp = bh_strdup(addr_pool[i]); + if (!cp) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: copy address failed"); + goto fail; + } + + address = strtok(cp, "/"); + mask = strtok(NULL, "/"); + + if (!mask) { + snprintf(error_buf, error_buf_size, + "Invalid address pool entry: %s, must be in the format of " + "ADDRESS/MASK", + addr_pool[i]); + goto fail; + } + + ret = addr_pool_insert(apool, address, (uint8)atoi(mask)); + wasm_runtime_free(cp); + if (!ret) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: store address failed"); + goto fail; + } + } + + if (!copy_string_array(ns_lookup_pool, ns_lookup_pool_size, &ns_lookup_buf, + &ns_lookup_list, NULL)) { + set_error_buf(error_buf, error_buf_size, + "Init wasi environment failed: allocate memory failed"); + goto fail; + } + + wasi_ctx->curfds = curfds; + wasi_ctx->prestats = prestats; + wasi_ctx->argv_environ = argv_environ; + wasi_ctx->addr_pool = apool; + wasi_ctx->argv_buf = argv_buf; + wasi_ctx->argv_list = argv_list; + wasi_ctx->env_buf = env_buf; + wasi_ctx->env_list = env_list; + wasi_ctx->ns_lookup_buf = ns_lookup_buf; + wasi_ctx->ns_lookup_list = ns_lookup_list; + + return true; + +fail: + if (argv_environ_inited) + argv_environ_destroy(argv_environ); + if (fd_prestats_inited) + fd_prestats_destroy(prestats); + if (fd_table_inited) + fd_table_destroy(curfds); + if (addr_pool_inited) + addr_pool_destroy(apool); + if (curfds) + wasm_runtime_free(curfds); + if (prestats) + wasm_runtime_free(prestats); + if (argv_environ) + wasm_runtime_free(argv_environ); + if (apool) + wasm_runtime_free(apool); + if (argv_buf) + wasm_runtime_free(argv_buf); + if (argv_list) + wasm_runtime_free(argv_list); + if (env_buf) + wasm_runtime_free(env_buf); + if (env_list) + wasm_runtime_free(env_list); + if (ns_lookup_buf) + wasm_runtime_free(ns_lookup_buf); + if (ns_lookup_list) + wasm_runtime_free(ns_lookup_list); + return false; +} +#else /* else of WASM_ENABLE_UVWASI == 0 */ +static void * +wasm_uvwasi_malloc(size_t size, void *mem_user_data) +{ + return runtime_malloc(size, NULL, NULL, 0); + (void)mem_user_data; +} + +static void +wasm_uvwasi_free(void *ptr, void *mem_user_data) +{ + if (ptr) + wasm_runtime_free(ptr); + (void)mem_user_data; +} + +static void * +wasm_uvwasi_calloc(size_t nmemb, size_t size, void *mem_user_data) +{ + uint64 total_size = (uint64)nmemb * size; + return runtime_malloc(total_size, NULL, NULL, 0); + (void)mem_user_data; +} + +static void * +wasm_uvwasi_realloc(void *ptr, size_t size, void *mem_user_data) +{ + if (size >= UINT32_MAX) { + return NULL; + } + return wasm_runtime_realloc(ptr, (uint32)size); +} + +/* clang-format off */ +static uvwasi_mem_t uvwasi_allocator = { + .mem_user_data = 0, + .malloc = wasm_uvwasi_malloc, + .free = wasm_uvwasi_free, + .calloc = wasm_uvwasi_calloc, + .realloc = wasm_uvwasi_realloc +}; +/* clang-format on */ + +bool +wasm_runtime_init_wasi(WASMModuleInstanceCommon *module_inst, + const char *dir_list[], uint32 dir_count, + const char *map_dir_list[], uint32 map_dir_count, + const char *env[], uint32 env_count, + const char *addr_pool[], uint32 addr_pool_size, + const char *ns_lookup_pool[], uint32 ns_lookup_pool_size, + char *argv[], uint32 argc, os_raw_file_handle stdinfd, + os_raw_file_handle stdoutfd, os_raw_file_handle stderrfd, + char *error_buf, uint32 error_buf_size) +{ + WASIContext *ctx; + uvwasi_t *uvwasi; + uvwasi_options_t init_options; + const char **envp = NULL; + uint64 total_size; + uint32 i; + bool ret = false; + + ctx = runtime_malloc(sizeof(*ctx), module_inst, error_buf, error_buf_size); + if (!ctx) + return false; + uvwasi = &ctx->uvwasi; + + /* Setup the initialization options */ + uvwasi_options_init(&init_options); + init_options.allocator = &uvwasi_allocator; + init_options.argc = argc; + init_options.argv = (const char **)argv; + init_options.in = (stdinfd != os_get_invalid_handle()) + ? (uvwasi_fd_t)stdinfd + : init_options.in; + init_options.out = (stdoutfd != os_get_invalid_handle()) + ? (uvwasi_fd_t)stdoutfd + : init_options.out; + init_options.err = (stderrfd != os_get_invalid_handle()) + ? (uvwasi_fd_t)stderrfd + : init_options.err; + + if (dir_count > 0) { + init_options.preopenc = dir_count; + + total_size = sizeof(uvwasi_preopen_t) * (uint64)init_options.preopenc; + init_options.preopens = (uvwasi_preopen_t *)runtime_malloc( + total_size, module_inst, error_buf, error_buf_size); + if (init_options.preopens == NULL) + goto fail; + + for (i = 0; i < init_options.preopenc; i++) { + init_options.preopens[i].real_path = dir_list[i]; + init_options.preopens[i].mapped_path = + (i < map_dir_count) ? map_dir_list[i] : dir_list[i]; + } + } + + if (env_count > 0) { + total_size = sizeof(char *) * (uint64)(env_count + 1); + envp = + runtime_malloc(total_size, module_inst, error_buf, error_buf_size); + if (envp == NULL) + goto fail; + + for (i = 0; i < env_count; i++) { + envp[i] = env[i]; + } + envp[env_count] = NULL; + init_options.envp = envp; + } + + if (UVWASI_ESUCCESS != uvwasi_init(uvwasi, &init_options)) { + set_error_buf(error_buf, error_buf_size, "uvwasi init failed"); + goto fail; + } + + wasm_runtime_set_wasi_ctx(module_inst, ctx); + + ret = true; + +fail: + if (envp) + wasm_runtime_free((void *)envp); + + if (init_options.preopens) + wasm_runtime_free(init_options.preopens); + + if (!ret && uvwasi) + wasm_runtime_free(uvwasi); + + return ret; +} +#endif /* end of WASM_ENABLE_UVWASI */ + +bool +wasm_runtime_is_wasi_mode(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode + && ((WASMModuleInstance *)module_inst)->module->import_wasi_api) + return true; +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT + && ((AOTModule *)((AOTModuleInstance *)module_inst)->module) + ->import_wasi_api) + return true; +#endif + return false; +} + +WASMFunctionInstanceCommon * +wasm_runtime_lookup_wasi_start_function(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstance *wasm_inst = (WASMModuleInstance *)module_inst; + WASMFunctionInstance *func = wasm_lookup_function(wasm_inst, "_start"); + if (func) { + if (func->u.func->func_type->param_count != 0 + || func->u.func->func_type->result_count != 0) { + LOG_ERROR("Lookup wasi _start function failed: " + "invalid function type.\n"); + return NULL; + } + return (WASMFunctionInstanceCommon *)func; + } + return NULL; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstance *aot_inst = (AOTModuleInstance *)module_inst; + AOTFunctionInstance *func = aot_lookup_function(aot_inst, "_start"); + if (func) { + AOTFuncType *func_type = func->u.func.func_type; + if (func_type->param_count != 0 || func_type->result_count != 0) { + LOG_ERROR("Lookup wasi _start function failed: " + "invalid function type.\n"); + return NULL; + } + return func; + } + return NULL; + } +#endif /* end of WASM_ENABLE_AOT */ + + return NULL; +} + +#if WASM_ENABLE_UVWASI == 0 +void +wasm_runtime_destroy_wasi(WASMModuleInstanceCommon *module_inst) +{ + WASIContext *wasi_ctx = wasm_runtime_get_wasi_ctx(module_inst); + + if (wasi_ctx) { + if (wasi_ctx->argv_environ) { + argv_environ_destroy(wasi_ctx->argv_environ); + wasm_runtime_free(wasi_ctx->argv_environ); + } + if (wasi_ctx->curfds) { + fd_table_destroy(wasi_ctx->curfds); + wasm_runtime_free(wasi_ctx->curfds); + } + if (wasi_ctx->prestats) { + fd_prestats_destroy(wasi_ctx->prestats); + wasm_runtime_free(wasi_ctx->prestats); + } + if (wasi_ctx->addr_pool) { + addr_pool_destroy(wasi_ctx->addr_pool); + wasm_runtime_free(wasi_ctx->addr_pool); + } + if (wasi_ctx->argv_buf) + wasm_runtime_free(wasi_ctx->argv_buf); + if (wasi_ctx->argv_list) + wasm_runtime_free(wasi_ctx->argv_list); + if (wasi_ctx->env_buf) + wasm_runtime_free(wasi_ctx->env_buf); + if (wasi_ctx->env_list) + wasm_runtime_free(wasi_ctx->env_list); + if (wasi_ctx->ns_lookup_buf) + wasm_runtime_free(wasi_ctx->ns_lookup_buf); + if (wasi_ctx->ns_lookup_list) + wasm_runtime_free(wasi_ctx->ns_lookup_list); + + wasm_runtime_free(wasi_ctx); + } +} +#else +void +wasm_runtime_destroy_wasi(WASMModuleInstanceCommon *module_inst) +{ + WASIContext *wasi_ctx = wasm_runtime_get_wasi_ctx(module_inst); + + if (wasi_ctx) { + uvwasi_destroy(&wasi_ctx->uvwasi); + wasm_runtime_free(wasi_ctx); + } +} +#endif + +uint32_t +wasm_runtime_get_wasi_exit_code(WASMModuleInstanceCommon *module_inst) +{ + WASIContext *wasi_ctx = wasm_runtime_get_wasi_ctx(module_inst); +#if WASM_ENABLE_THREAD_MGR != 0 + WASMCluster *cluster; + WASMExecEnv *exec_env; + + exec_env = wasm_runtime_get_exec_env_singleton(module_inst); + if (exec_env && (cluster = wasm_exec_env_get_cluster(exec_env))) { + /** + * The main thread may exit earlier than other threads, and + * the exit_code of wasi_ctx may be changed by other thread + * when it runs into wasi_proc_exit, here we wait until all + * other threads exit to avoid getting invalid exit_code. + */ + wasm_cluster_wait_for_all_except_self(cluster, exec_env); + } +#endif + return wasi_ctx->exit_code; +} +#endif /* end of WASM_ENABLE_LIBC_WASI */ + +WASMModuleCommon * +wasm_exec_env_get_module(WASMExecEnv *exec_env) +{ + WASMModuleInstanceCommon *module_inst_comm = + wasm_runtime_get_module_inst(exec_env); + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + return (WASMModuleCommon *)module_inst->module; +} + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 +const uint8 * +wasm_runtime_get_custom_section(WASMModuleCommon *const module_comm, + const char *name, uint32 *len) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_comm->module_type == Wasm_Module_Bytecode) + return wasm_loader_get_custom_section((WASMModule *)module_comm, name, + len); +#endif +#if WASM_ENABLE_AOT != 0 + if (module_comm->module_type == Wasm_Module_AoT) + return aot_get_custom_section((AOTModule *)module_comm, name, len); +#endif + return NULL; +} +#endif /* end of WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 */ + +static union { + int a; + char b; +} __ue = { .a = 1 }; + +#define is_little_endian() (__ue.b == 1) /* NOLINT */ + +int32 +wasm_runtime_get_import_count(WASMModuleCommon *const module) +{ + if (!module) { + bh_assert(0); + return -1; + } + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + const AOTModule *aot_module = (const AOTModule *)module; + return (int32)(aot_module->import_func_count + + aot_module->import_global_count + + aot_module->import_table_count + + aot_module->import_memory_count); + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + const WASMModule *wasm_module = (const WASMModule *)module; + return (int32)wasm_module->import_count; + } +#endif + + return -1; +} + +void +wasm_runtime_get_import_type(WASMModuleCommon *const module, int32 import_index, + wasm_import_t *import_type) +{ + if (!import_type) { + bh_assert(0); + return; + } + + memset(import_type, 0, sizeof(wasm_import_t)); + + if (!module) { + bh_assert(0); + return; + } + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + const AOTModule *aot_module = (const AOTModule *)module; + + uint32 func_index = (uint32)import_index; + if (func_index < aot_module->import_func_count) { + const AOTImportFunc *aot_import_func = + &aot_module->import_funcs[func_index]; + import_type->module_name = aot_import_func->module_name; + import_type->name = aot_import_func->func_name; + import_type->kind = WASM_IMPORT_EXPORT_KIND_FUNC; + import_type->linked = + aot_import_func->func_ptr_linked ? true : false; + import_type->u.func_type = + (WASMFuncType *)aot_import_func->func_type; + return; + } + + uint32 global_index = func_index - aot_module->import_func_count; + if (global_index < aot_module->import_global_count) { + const AOTImportGlobal *aot_import_global = + &aot_module->import_globals[global_index]; + import_type->module_name = aot_import_global->module_name; + import_type->name = aot_import_global->global_name; + import_type->kind = WASM_IMPORT_EXPORT_KIND_GLOBAL; + import_type->linked = aot_import_global->is_linked; + import_type->u.global_type = + (WASMGlobalType *)&aot_import_global->type; + return; + } + + uint32 table_index = global_index - aot_module->import_global_count; + if (table_index < aot_module->import_table_count) { + const AOTImportTable *aot_import_table = + &aot_module->import_tables[table_index]; + import_type->module_name = aot_import_table->module_name; + import_type->name = aot_import_table->table_name; + import_type->kind = WASM_IMPORT_EXPORT_KIND_TABLE; + import_type->linked = false; /* not supported */ + import_type->u.table_type = + (WASMTableType *)&aot_import_table->table_type; + return; + } + + uint32 memory_index = table_index - aot_module->import_table_count; + if (memory_index < aot_module->import_memory_count) { + const AOTImportMemory *aot_import_memory = + &aot_module->import_memories[memory_index]; + import_type->module_name = aot_import_memory->module_name; + import_type->name = aot_import_memory->memory_name; + import_type->kind = WASM_IMPORT_EXPORT_KIND_MEMORY; + import_type->linked = false; /* not supported */ + import_type->u.memory_type = + (WASMMemoryType *)&aot_import_memory->mem_type; + return; + } + + bh_assert(0); + return; + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + const WASMModule *wasm_module = (const WASMModule *)module; + + if ((uint32)import_index >= wasm_module->import_count) { + bh_assert(0); + return; + } + + const WASMImport *wasm_import = &wasm_module->imports[import_index]; + + import_type->module_name = wasm_import->u.names.module_name; + import_type->name = wasm_import->u.names.field_name; + import_type->kind = wasm_import->kind; + switch (import_type->kind) { + case WASM_IMPORT_EXPORT_KIND_FUNC: + import_type->linked = wasm_import->u.function.func_ptr_linked; + import_type->u.func_type = + (WASMFuncType *)wasm_import->u.function.func_type; + break; + case WASM_IMPORT_EXPORT_KIND_GLOBAL: + import_type->linked = wasm_import->u.global.is_linked; + import_type->u.global_type = + (WASMGlobalType *)&wasm_import->u.global.type; + break; + case WASM_IMPORT_EXPORT_KIND_TABLE: + import_type->linked = false; /* not supported */ + import_type->u.table_type = + (WASMTableType *)&wasm_import->u.table.table_type; + break; + case WASM_IMPORT_EXPORT_KIND_MEMORY: + import_type->linked = false; /* not supported */ + import_type->u.memory_type = + (WASMMemoryType *)&wasm_import->u.memory.mem_type; + break; + default: + bh_assert(0); + break; + } + + return; + } +#endif +} + +int32 +wasm_runtime_get_export_count(WASMModuleCommon *const module) +{ + if (!module) { + bh_assert(0); + return -1; + } + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + const AOTModule *aot_module = (const AOTModule *)module; + return (int32)aot_module->export_count; + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + const WASMModule *wasm_module = (const WASMModule *)module; + return (int32)wasm_module->export_count; + } +#endif + + return -1; +} + +void +wasm_runtime_get_export_type(WASMModuleCommon *const module, int32 export_index, + wasm_export_t *export_type) +{ + if (!export_type) { + bh_assert(0); + return; + } + + memset(export_type, 0, sizeof(wasm_export_t)); + + if (!module) { + bh_assert(0); + return; + } + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + const AOTModule *aot_module = (const AOTModule *)module; + + if ((uint32)export_index >= aot_module->export_count) { + bh_assert(0); + return; + } + + const AOTExport *aot_export = &aot_module->exports[export_index]; + export_type->name = aot_export->name; + export_type->kind = aot_export->kind; + switch (export_type->kind) { + case WASM_IMPORT_EXPORT_KIND_FUNC: + { + if (aot_export->index < aot_module->import_func_count) { + export_type->u.func_type = + (AOTFuncType *)aot_module + ->import_funcs[aot_export->index] + .func_type; + } + else { + export_type->u.func_type = + (AOTFuncType *)aot_module + ->types[aot_module->func_type_indexes + [aot_export->index + - aot_module->import_func_count]]; + } + break; + } + case WASM_IMPORT_EXPORT_KIND_GLOBAL: + { + if (aot_export->index < aot_module->import_global_count) { + export_type->u.global_type = + &aot_module->import_globals[aot_export->index].type; + } + else { + export_type->u.global_type = + &aot_module + ->globals[aot_export->index + - aot_module->import_global_count] + .type; + } + break; + } + case WASM_IMPORT_EXPORT_KIND_TABLE: + { + if (aot_export->index < aot_module->import_table_count) { + export_type->u.table_type = + &aot_module->import_tables[aot_export->index] + .table_type; + } + else { + export_type->u.table_type = + &aot_module + ->tables[aot_export->index + - aot_module->import_table_count] + .table_type; + } + break; + } + case WASM_IMPORT_EXPORT_KIND_MEMORY: + { + if (aot_export->index < aot_module->import_memory_count) { + export_type->u.memory_type = + &aot_module->import_memories[aot_export->index] + .mem_type; + } + else { + export_type->u.memory_type = + &aot_module + ->memories[aot_export->index + - aot_module->import_memory_count]; + } + break; + } + default: + bh_assert(0); + break; + } + return; + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + const WASMModule *wasm_module = (const WASMModule *)module; + + if ((uint32)export_index >= wasm_module->export_count) { + bh_assert(0); + return; + } + + const WASMExport *wasm_export = &wasm_module->exports[export_index]; + export_type->name = wasm_export->name; + export_type->kind = wasm_export->kind; + switch (export_type->kind) { + case WASM_IMPORT_EXPORT_KIND_FUNC: + { + if (wasm_export->index < wasm_module->import_function_count) { + export_type->u.func_type = + (WASMFuncType *)wasm_module + ->import_functions[wasm_export->index] + .u.function.func_type; + } + else { + export_type->u.func_type = + wasm_module + ->functions[wasm_export->index + - wasm_module->import_function_count] + ->func_type; + } + + break; + } + case WASM_IMPORT_EXPORT_KIND_GLOBAL: + { + if (wasm_export->index < wasm_module->import_global_count) { + export_type->u.global_type = + (WASMGlobalType *)&wasm_module + ->import_globals[wasm_export->index] + .u.global.type; + } + else { + export_type->u.global_type = + &wasm_module + ->globals[wasm_export->index + - wasm_module->import_global_count] + .type; + } + + break; + } + case WASM_IMPORT_EXPORT_KIND_TABLE: + { + if (wasm_export->index < wasm_module->import_table_count) { + export_type->u.table_type = + (WASMTableType *)&wasm_module + ->import_tables[wasm_export->index] + .u.table.table_type; + } + else { + export_type->u.table_type = + &wasm_module + ->tables[wasm_export->index + - wasm_module->import_table_count] + .table_type; + } + + break; + } + case WASM_IMPORT_EXPORT_KIND_MEMORY: + { + if (wasm_export->index < wasm_module->import_memory_count) { + export_type->u.memory_type = + (WASMMemoryType *)&wasm_module + ->import_memories[wasm_export->index] + .u.memory.mem_type; + } + else { + export_type->u.memory_type = + &wasm_module + ->memories[wasm_export->index + - wasm_module->import_memory_count]; + } + + break; + } + default: + bh_assert(0); + break; + } + return; + } +#endif +} + +uint32 +wasm_func_type_get_param_count(WASMFuncType *const func_type) +{ + bh_assert(func_type); + + return func_type->param_count; +} + +wasm_valkind_t +wasm_func_type_get_param_valkind(WASMFuncType *const func_type, + uint32 param_index) +{ + if (!func_type || (param_index >= func_type->param_count)) { + bh_assert(0); + return (wasm_valkind_t)-1; + } + + switch (func_type->types[param_index]) { + case VALUE_TYPE_I32: + return WASM_I32; + case VALUE_TYPE_I64: + return WASM_I64; + case VALUE_TYPE_F32: + return WASM_F32; + case VALUE_TYPE_F64: + return WASM_F64; + case VALUE_TYPE_V128: + return WASM_V128; + case VALUE_TYPE_FUNCREF: + return WASM_FUNCREF; + case VALUE_TYPE_EXTERNREF: + return WASM_EXTERNREF; + + case VALUE_TYPE_VOID: + default: + { + bh_assert(0); + return (wasm_valkind_t)-1; + } + } +} + +uint32 +wasm_func_type_get_result_count(WASMFuncType *const func_type) +{ + bh_assert(func_type); + + return func_type->result_count; +} + +wasm_valkind_t +wasm_func_type_get_result_valkind(WASMFuncType *const func_type, + uint32 result_index) +{ + if (!func_type || (result_index >= func_type->result_count)) { + bh_assert(0); + return (wasm_valkind_t)-1; + } + + switch (func_type->types[func_type->param_count + result_index]) { + case VALUE_TYPE_I32: + return WASM_I32; + case VALUE_TYPE_I64: + return WASM_I64; + case VALUE_TYPE_F32: + return WASM_F32; + case VALUE_TYPE_F64: + return WASM_F64; + case VALUE_TYPE_FUNCREF: + return WASM_FUNCREF; + +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + return WASM_V128; +#endif +#if WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: +#endif + case VALUE_TYPE_VOID: + default: + { + bh_assert(0); + return (wasm_valkind_t)-1; + } + } +} + +wasm_valkind_t +wasm_global_type_get_valkind(WASMGlobalType *const global_type) +{ + bh_assert(global_type); + + return val_type_to_val_kind(global_type->val_type); +} + +bool +wasm_global_type_get_mutable(WASMGlobalType *const global_type) +{ + bh_assert(global_type); + + return global_type->is_mutable; +} + +bool +wasm_memory_type_get_shared(WASMMemoryType *const memory_type) +{ + bh_assert(memory_type); + + return (memory_type->flags & SHARED_MEMORY_FLAG) ? true : false; +} + +uint32 +wasm_memory_type_get_init_page_count(WASMMemoryType *const memory_type) +{ + bh_assert(memory_type); + + return memory_type->init_page_count; +} + +uint32 +wasm_memory_type_get_max_page_count(WASMMemoryType *const memory_type) +{ + bh_assert(memory_type); + + return memory_type->max_page_count; +} + +wasm_valkind_t +wasm_table_type_get_elem_kind(WASMTableType *const table_type) +{ + bh_assert(table_type); + + return val_type_to_val_kind(table_type->elem_type); +} + +bool +wasm_table_type_get_shared(WASMTableType *const table_type) +{ + bh_assert(table_type); + + return (table_type->flags & 2) ? true : false; +} + +uint32 +wasm_table_type_get_init_size(WASMTableType *const table_type) +{ + bh_assert(table_type); + + return table_type->init_size; +} + +uint32 +wasm_table_type_get_max_size(WASMTableType *const table_type) +{ + bh_assert(table_type); + + return table_type->max_size; +} + +#if WASM_ENABLE_INVOKE_NATIVE != 0 +bool +wasm_runtime_register_natives(const char *module_name, + NativeSymbol *native_symbols, + uint32 n_native_symbols) +{ + return wasm_native_register_natives(module_name, native_symbols, + n_native_symbols); +} + +bool +wasm_runtime_register_natives_raw(const char *module_name, + NativeSymbol *native_symbols, + uint32 n_native_symbols) +{ + return wasm_native_register_natives_raw(module_name, native_symbols, + n_native_symbols); +} + +bool +wasm_runtime_unregister_natives(const char *module_name, + NativeSymbol *native_symbols) +{ + return wasm_native_unregister_natives(module_name, native_symbols); +} + +bool +wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, + const WASMFuncType *func_type, + const char *signature, void *attachment, + uint32 *argv, uint32 argc, uint32 *argv_ret) +{ + WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env); +#if WASM_ENABLE_MEMORY64 != 0 + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module); + bool is_memory64 = memory ? memory->is_memory64 : false; +#endif + typedef void (*NativeRawFuncPtr)(WASMExecEnv *, uint64 *); + NativeRawFuncPtr invoke_native_raw = (NativeRawFuncPtr)func_ptr; + uint64 argv_buf[16] = { 0 }, *argv1 = argv_buf, *argv_dst, size; + uint32 *argv_src = argv, i, argc1, ptr_len; + uint32 arg_i32; + bool ret = false; + + argc1 = func_type->param_count; + if (argc1 > sizeof(argv_buf) / sizeof(uint64)) { + size = sizeof(uint64) * (uint64)argc1; + if (!(argv1 = runtime_malloc((uint32)size, exec_env->module_inst, NULL, + 0))) { + return false; + } + } + + argv_dst = argv1; + + /* Traverse secondly to fill in each argument */ + for (i = 0; i < func_type->param_count; i++, argv_dst++) { + switch (func_type->types[i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#endif + { + *(uint32 *)argv_dst = arg_i32 = *argv_src++; + if (signature +#if WASM_ENABLE_MEMORY64 != 0 + && !is_memory64 +#endif + ) { + if (signature[i + 1] == '*') { + /* param is a pointer */ + if (signature[i + 2] == '~') + /* pointer with length followed */ + ptr_len = *argv_src; + else + /* pointer without length followed */ + ptr_len = 1; + + if (!wasm_runtime_validate_app_addr( + module, (uint64)arg_i32, (uint64)ptr_len)) + goto fail; + + *(uintptr_t *)argv_dst = + (uintptr_t)wasm_runtime_addr_app_to_native( + module, (uint64)arg_i32); + } + else if (signature[i + 1] == '$') { + /* param is a string */ + if (!wasm_runtime_validate_app_str_addr( + module, (uint64)arg_i32)) + goto fail; + + *(uintptr_t *)argv_dst = + (uintptr_t)wasm_runtime_addr_app_to_native( + module, (uint64)arg_i32); + } + } + break; + } + case VALUE_TYPE_I64: +#if WASM_ENABLE_MEMORY64 != 0 + { + uint64 arg_i64; + + PUT_I64_TO_ADDR((uint32 *)argv_dst, + GET_I64_FROM_ADDR(argv_src)); + argv_src += 2; + arg_i64 = *argv_dst; + if (signature && is_memory64) { + /* TODO: memory64 pointer with length need a new symbol + * to represent type i64, with '~' still represent i32 + * length */ + if (signature[i + 1] == '*') { + /* param is a pointer */ + if (signature[i + 2] == '~') + /* pointer with length followed */ + ptr_len = *argv_src; + else + /* pointer without length followed */ + ptr_len = 1; + + if (!wasm_runtime_validate_app_addr(module, arg_i64, + (uint64)ptr_len)) + goto fail; + + *argv_dst = (uint64)wasm_runtime_addr_app_to_native( + module, arg_i64); + } + else if (signature[i + 1] == '$') { + /* param is a string */ + if (!wasm_runtime_validate_app_str_addr(module, + arg_i64)) + goto fail; + + *argv_dst = (uint64)wasm_runtime_addr_app_to_native( + module, arg_i64); + } + } + break; + } +#endif + case VALUE_TYPE_F64: + bh_memcpy_s(argv_dst, sizeof(uint64), argv_src, + sizeof(uint32) * 2); + argv_src += 2; + break; + case VALUE_TYPE_F32: + *(float32 *)argv_dst = *(float32 *)argv_src++; + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + { + uint32 externref_idx = *argv_src++; + + void *externref_obj; + + if (!wasm_externref_ref2obj(externref_idx, &externref_obj)) + goto fail; + + bh_memcpy_s(argv_dst, sizeof(uintptr_t), argv_src, + sizeof(uintptr_t)); + break; + } +#endif +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif + { + bh_memcpy_s(argv_dst, sizeof(uintptr_t), argv_src, + sizeof(uintptr_t)); + argv_src += sizeof(uintptr_t) / sizeof(uint32); + break; + } +#endif + default: + bh_assert(0); + break; + } + } + + exec_env->attachment = attachment; + invoke_native_raw(exec_env, argv1); + exec_env->attachment = NULL; + + if (func_type->result_count > 0) { + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#endif + argv_ret[0] = *(uint32 *)argv1; + break; + case VALUE_TYPE_F32: + *(float32 *)argv_ret = *(float32 *)argv1; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + bh_memcpy_s(argv_ret, sizeof(uint32) * 2, argv1, + sizeof(uint64)); + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + { + uint32 externref_idx; + uint64 externref_obj; + + bh_memcpy_s(&externref_obj, sizeof(uint64), argv1, + sizeof(uint64)); + + if (!wasm_externref_obj2ref(exec_env->module_inst, + (void *)(uintptr_t)externref_obj, + &externref_idx)) + goto fail; + argv_ret[0] = externref_idx; + break; + } +#endif +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif + { + bh_memcpy_s(argv_ret, sizeof(uintptr_t), argv1, + sizeof(uintptr_t)); + break; + } +#endif + default: + bh_assert(0); + break; + } + } + + ret = !wasm_runtime_copy_exception(module, NULL); + +fail: + if (argv1 != argv_buf) + wasm_runtime_free(argv1); + return ret; +} + +/** + * Implementation of wasm_runtime_invoke_native() + */ + +/** + * The invoke native implementation on ARM platform with VFP co-processor, + * RISCV32 platform with/without FPU/DPFPU and ARC platform. + */ +#if defined(BUILD_TARGET_ARM_VFP) || defined(BUILD_TARGET_THUMB_VFP) \ + || defined(BUILD_TARGET_RISCV32_ILP32D) \ + || defined(BUILD_TARGET_RISCV32_ILP32F) \ + || defined(BUILD_TARGET_RISCV32_ILP32) || defined(BUILD_TARGET_ARC) +typedef void (*GenericFunctionPointer)(void); +void +invokeNative(GenericFunctionPointer f, uint32 *args, uint32 n_stacks); + +typedef float64 (*Float64FuncPtr)(GenericFunctionPointer, uint32 *, uint32); +typedef float32 (*Float32FuncPtr)(GenericFunctionPointer, uint32 *, uint32); +typedef int64 (*Int64FuncPtr)(GenericFunctionPointer, uint32 *, uint32); +typedef int32 (*Int32FuncPtr)(GenericFunctionPointer, uint32 *, uint32); +typedef void (*VoidFuncPtr)(GenericFunctionPointer, uint32 *, uint32); + +static volatile Float64FuncPtr invokeNative_Float64 = + (Float64FuncPtr)(uintptr_t)invokeNative; +static volatile Float32FuncPtr invokeNative_Float32 = + (Float32FuncPtr)(uintptr_t)invokeNative; +static volatile Int64FuncPtr invokeNative_Int64 = + (Int64FuncPtr)(uintptr_t)invokeNative; +static volatile Int32FuncPtr invokeNative_Int32 = + (Int32FuncPtr)(uintptr_t)invokeNative; +static volatile VoidFuncPtr invokeNative_Void = + (VoidFuncPtr)(uintptr_t)invokeNative; + +#if defined(BUILD_TARGET_ARM_VFP) || defined(BUILD_TARGET_THUMB_VFP) +#define MAX_REG_INTS 4 +#define MAX_REG_FLOATS 16 +#else +#define MAX_REG_INTS 8 +#define MAX_REG_FLOATS 8 +#endif + +bool +wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, + const WASMFuncType *func_type, const char *signature, + void *attachment, uint32 *argv, uint32 argc, + uint32 *argv_ret) +{ + WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env); + /* argv buf layout: int args(fix cnt) + float args(fix cnt) + stack args + */ + uint32 argv_buf[32], *argv1 = argv_buf, *ints, *stacks, size; + uint32 *argv_src = argv, i, argc1, n_ints = 0, n_stacks = 0; + uint32 arg_i32, ptr_len; + uint32 result_count = func_type->result_count; + uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0; + bool ret = false; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + bool is_aot_func = (NULL == signature); +#endif +#if !defined(BUILD_TARGET_RISCV32_ILP32) && !defined(BUILD_TARGET_ARC) + uint32 *fps; + int n_fps = 0; +#else +#define fps ints +#define n_fps n_ints +#endif + + n_ints++; /* exec env */ + + /* Traverse firstly to calculate stack args count */ + for (i = 0; i < func_type->param_count; i++) { + switch (func_type->types[i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif +#endif +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: + case VALUE_TYPE_EXTERNREF: +#endif + if (n_ints < MAX_REG_INTS) + n_ints++; + else + n_stacks++; + break; + case VALUE_TYPE_I64: + if (n_ints < MAX_REG_INTS - 1) { +#if defined(BUILD_TARGET_ARM_VFP) || defined(BUILD_TARGET_THUMB_VFP) + /* 64-bit data must be 8 bytes aligned in arm */ + if (n_ints & 1) + n_ints++; +#endif + n_ints += 2; + } +#if defined(BUILD_TARGET_RISCV32_ILP32) \ + || defined(BUILD_TARGET_RISCV32_ILP32F) \ + || defined(BUILD_TARGET_RISCV32_ILP32D) || defined(BUILD_TARGET_ARC) + /* part in register, part in stack */ + else if (n_ints == MAX_REG_INTS - 1) { + n_ints++; + n_stacks++; + } +#endif + else { + /* 64-bit data in stack must be 8 bytes aligned + in arm and riscv32 */ +#if !defined(BUILD_TARGET_ARC) + if (n_stacks & 1) + n_stacks++; +#endif + n_stacks += 2; + } + break; +#if !defined(BUILD_TARGET_RISCV32_ILP32D) + case VALUE_TYPE_F32: + if (n_fps < MAX_REG_FLOATS) + n_fps++; +#if defined(BUILD_TARGET_RISCV32_ILP32F) + else if (n_ints < MAX_REG_INTS) { + n_ints++; + } +#endif + else + n_stacks++; + break; + case VALUE_TYPE_F64: +#if defined(BUILD_TARGET_RISCV32_ILP32) \ + || defined(BUILD_TARGET_RISCV32_ILP32F) || defined(BUILD_TARGET_ARC) + if (n_ints < MAX_REG_INTS - 1) { + n_ints += 2; + } + else if (n_ints == MAX_REG_INTS - 1) { + n_ints++; + n_stacks++; + } +#endif +#if defined(BUILD_TARGET_ARM_VFP) || defined(BUILD_TARGET_THUMB_VFP) + if (n_fps < MAX_REG_FLOATS - 1) { + /* 64-bit data must be 8 bytes aligned in arm */ + if (n_fps & 1) + n_fps++; + n_fps += 2; + } + else if (n_fps == MAX_REG_FLOATS - 1) { + n_fps++; + n_stacks++; + } +#endif + else { + /* 64-bit data in stack must be 8 bytes aligned + in arm and riscv32 */ +#if !defined(BUILD_TARGET_ARC) + if (n_stacks & 1) + n_stacks++; +#endif + n_stacks += 2; + } + break; +#else /* BUILD_TARGET_RISCV32_ILP32D */ + case VALUE_TYPE_F32: + case VALUE_TYPE_F64: + if (n_fps < MAX_REG_FLOATS) { + n_fps++; + } + else if (func_type->types[i] == VALUE_TYPE_F32 + && n_ints < MAX_REG_INTS) { + /* use int reg firstly if available */ + n_ints++; + } + else if (func_type->types[i] == VALUE_TYPE_F64 + && n_ints < MAX_REG_INTS - 1) { + /* use int regs firstly if available */ + if (n_ints & 1) + n_ints++; + n_ints += 2; + } + else { + /* 64-bit data in stack must be 8 bytes aligned in riscv32 + */ + if (n_stacks & 1) + n_stacks++; + n_stacks += 2; + } + break; +#endif /* BUILD_TARGET_RISCV32_ILP32D */ + default: + bh_assert(0); + break; + } + } + + for (i = 0; i < ext_ret_count; i++) { + if (n_ints < MAX_REG_INTS) + n_ints++; + else + n_stacks++; + } + +#if defined(BUILD_TARGET_ARM_VFP) || defined(BUILD_TARGET_THUMB_VFP) \ + || defined(BUILD_TARGET_RISCV32_ILP32F) + argc1 = MAX_REG_INTS + MAX_REG_FLOATS + n_stacks; +#elif defined(BUILD_TARGET_RISCV32_ILP32) || defined(BUILD_TARGET_ARC) + argc1 = MAX_REG_INTS + n_stacks; +#else /* for BUILD_TARGET_RISCV32_ILP32D */ + argc1 = MAX_REG_INTS + MAX_REG_FLOATS * 2 + n_stacks; +#endif + + if (argc1 > sizeof(argv_buf) / sizeof(uint32)) { + size = sizeof(uint32) * (uint32)argc1; + if (!(argv1 = runtime_malloc((uint32)size, exec_env->module_inst, NULL, + 0))) { + return false; + } + } + + ints = argv1; +#if defined(BUILD_TARGET_ARM_VFP) || defined(BUILD_TARGET_THUMB_VFP) \ + || defined(BUILD_TARGET_RISCV32_ILP32F) + fps = ints + MAX_REG_INTS; + stacks = fps + MAX_REG_FLOATS; +#elif defined(BUILD_TARGET_RISCV32_ILP32) || defined(BUILD_TARGET_ARC) + stacks = ints + MAX_REG_INTS; +#else /* for BUILD_TARGET_RISCV32_ILP32D */ + fps = ints + MAX_REG_INTS; + stacks = fps + MAX_REG_FLOATS * 2; +#endif + + n_ints = 0; + n_fps = 0; + n_stacks = 0; + ints[n_ints++] = (uint32)(uintptr_t)exec_env; + + /* Traverse secondly to fill in each argument */ + for (i = 0; i < func_type->param_count; i++) { + switch (func_type->types[i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif +#endif +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#endif + { + arg_i32 = *argv_src++; + + if (signature) { + if (signature[i + 1] == '*') { + /* param is a pointer */ + if (signature[i + 2] == '~') + /* pointer with length followed */ + ptr_len = *argv_src; + else + /* pointer without length followed */ + ptr_len = 1; + + if (!wasm_runtime_validate_app_addr( + module, (uint64)arg_i32, (uint64)ptr_len)) + goto fail; + + arg_i32 = (uintptr_t)wasm_runtime_addr_app_to_native( + module, (uint64)arg_i32); + } + else if (signature[i + 1] == '$') { + /* param is a string */ + if (!wasm_runtime_validate_app_str_addr( + module, (uint64)arg_i32)) + goto fail; + + arg_i32 = (uintptr_t)wasm_runtime_addr_app_to_native( + module, (uint64)arg_i32); + } + } + + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = arg_i32; + else + stacks[n_stacks++] = arg_i32; + break; + } + case VALUE_TYPE_I64: + { + if (n_ints < MAX_REG_INTS - 1) { +#if defined(BUILD_TARGET_ARM_VFP) || defined(BUILD_TARGET_THUMB_VFP) + /* 64-bit data must be 8 bytes aligned in arm */ + if (n_ints & 1) + n_ints++; +#endif + ints[n_ints++] = *argv_src++; + ints[n_ints++] = *argv_src++; + } +#if defined(BUILD_TARGET_RISCV32_ILP32) \ + || defined(BUILD_TARGET_RISCV32_ILP32F) \ + || defined(BUILD_TARGET_RISCV32_ILP32D) || defined(BUILD_TARGET_ARC) + else if (n_ints == MAX_REG_INTS - 1) { + ints[n_ints++] = *argv_src++; + stacks[n_stacks++] = *argv_src++; + } +#endif + else { + /* 64-bit data in stack must be 8 bytes aligned + in arm and riscv32 */ +#if !defined(BUILD_TARGET_ARC) + if (n_stacks & 1) + n_stacks++; +#endif + stacks[n_stacks++] = *argv_src++; + stacks[n_stacks++] = *argv_src++; + } + break; + } +#if !defined(BUILD_TARGET_RISCV32_ILP32D) + case VALUE_TYPE_F32: + { + if (n_fps < MAX_REG_FLOATS) + *(float32 *)&fps[n_fps++] = *(float32 *)argv_src++; +#if defined(BUILD_TARGET_RISCV32_ILP32F) + else if (n_ints < MAX_REG_INTS) { + ints[n_ints++] = *argv_src++; + } +#endif + else + *(float32 *)&stacks[n_stacks++] = *(float32 *)argv_src++; + break; + } + case VALUE_TYPE_F64: + { +#if defined(BUILD_TARGET_RISCV32_ILP32) \ + || defined(BUILD_TARGET_RISCV32_ILP32F) || defined(BUILD_TARGET_ARC) + if (n_ints < MAX_REG_INTS - 1) { + ints[n_ints++] = *argv_src++; + ints[n_ints++] = *argv_src++; + } + else if (n_ints == MAX_REG_INTS - 1) { + ints[n_ints++] = *argv_src++; + stacks[n_stacks++] = *argv_src++; + } +#endif +#if defined(BUILD_TARGET_ARM_VFP) || defined(BUILD_TARGET_THUMB_VFP) + if (n_fps < MAX_REG_FLOATS - 1) { + /* 64-bit data must be 8 bytes aligned in arm */ + if (n_fps & 1) + n_fps++; + fps[n_fps++] = *argv_src++; + fps[n_fps++] = *argv_src++; + } + else if (n_fps == MAX_REG_FLOATS - 1) { + fps[n_fps++] = *argv_src++; + stacks[n_stacks++] = *argv_src++; + } +#endif + else { + /* 64-bit data in stack must be 8 bytes aligned + in arm and riscv32 */ +#if !defined(BUILD_TARGET_ARC) + if (n_stacks & 1) + n_stacks++; +#endif + stacks[n_stacks++] = *argv_src++; + stacks[n_stacks++] = *argv_src++; + } + break; + } +#else /* BUILD_TARGET_RISCV32_ILP32D */ + case VALUE_TYPE_F32: + case VALUE_TYPE_F64: + { + if (n_fps < MAX_REG_FLOATS) { + if (func_type->types[i] == VALUE_TYPE_F32) { + *(float32 *)&fps[n_fps * 2] = *(float32 *)argv_src++; + /* NaN boxing, the upper bits of a valid NaN-boxed + value must be all 1s. */ + fps[n_fps * 2 + 1] = 0xFFFFFFFF; + } + else { + *(float64 *)&fps[n_fps * 2] = *(float64 *)argv_src; + argv_src += 2; + } + n_fps++; + } + else if (func_type->types[i] == VALUE_TYPE_F32 + && n_ints < MAX_REG_INTS) { + /* use int reg firstly if available */ + *(float32 *)&ints[n_ints++] = *(float32 *)argv_src++; + } + else if (func_type->types[i] == VALUE_TYPE_F64 + && n_ints < MAX_REG_INTS - 1) { + /* use int regs firstly if available */ + if (n_ints & 1) + n_ints++; + *(float64 *)&ints[n_ints] = *(float64 *)argv_src; + n_ints += 2; + argv_src += 2; + } + else { + /* 64-bit data in stack must be 8 bytes aligned in riscv32 + */ + if (n_stacks & 1) + n_stacks++; + if (func_type->types[i] == VALUE_TYPE_F32) { + *(float32 *)&stacks[n_stacks++] = + *(float32 *)argv_src++; + } + else { + *(float64 *)&stacks[n_stacks] = *(float64 *)argv_src; + argv_src += 2; + n_stacks += 2; + } + } + break; + } +#endif /* BUILD_TARGET_RISCV32_ILP32D */ +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + { + uint32 externref_idx = *argv_src++; + + if (is_aot_func) { + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = externref_idx; + else + stacks[n_stacks++] = externref_idx; + } + else { + void *externref_obj; + + if (!wasm_externref_ref2obj(externref_idx, &externref_obj)) + goto fail; + + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = (uintptr_t)externref_obj; + else + stacks[n_stacks++] = (uintptr_t)externref_obj; + } + break; + } +#endif + default: + bh_assert(0); + break; + } + } + + /* Save extra result values' address to argv1 */ + for (i = 0; i < ext_ret_count; i++) { + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = *(uint32 *)argv_src++; + else + stacks[n_stacks++] = *(uint32 *)argv_src++; + } + + exec_env->attachment = attachment; + if (func_type->result_count == 0) { + invokeNative_Void(func_ptr, argv1, n_stacks); + } + else { + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif +#endif +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#endif + argv_ret[0] = + (uint32)invokeNative_Int32(func_ptr, argv1, n_stacks); + break; + case VALUE_TYPE_I64: + PUT_I64_TO_ADDR(argv_ret, + invokeNative_Int64(func_ptr, argv1, n_stacks)); + break; + case VALUE_TYPE_F32: + *(float32 *)argv_ret = + invokeNative_Float32(func_ptr, argv1, n_stacks); + break; + case VALUE_TYPE_F64: + PUT_F64_TO_ADDR( + argv_ret, invokeNative_Float64(func_ptr, argv1, n_stacks)); + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + { + if (is_aot_func) { + uint32 externref_idx = + (uint32)invokeNative_Int32(func_ptr, argv1, argc1); + argv_ret[0] = externref_idx; + } + else { + uint32 externref_idx; + void *externref_obj; + + externref_obj = (void *)(uintptr_t)invokeNative_Int32( + func_ptr, argv1, argc1); + + if (!wasm_externref_obj2ref(exec_env->module_inst, + externref_obj, &externref_idx)) + goto fail; + + argv_ret[0] = externref_idx; + } + break; + } +#endif + default: + bh_assert(0); + break; + } + } + exec_env->attachment = NULL; + + ret = !wasm_runtime_copy_exception(module, NULL); + +fail: + if (argv1 != argv_buf) + wasm_runtime_free(argv1); + return ret; +} +#endif /* end of defined(BUILD_TARGET_ARM_VFP) \ + || defined(BUILD_TARGET_THUMB_VFP) \ + || defined(BUILD_TARGET_RISCV32_ILP32D) \ + || defined(BUILD_TARGET_RISCV32_ILP32F) \ + || defined(BUILD_TARGET_RISCV32_ILP32) \ + || defined(BUILD_TARGET_ARC) */ + +#if defined(BUILD_TARGET_X86_32) || defined(BUILD_TARGET_ARM) \ + || defined(BUILD_TARGET_THUMB) || defined(BUILD_TARGET_MIPS) \ + || defined(BUILD_TARGET_XTENSA) +typedef void (*GenericFunctionPointer)(void); +void +invokeNative(GenericFunctionPointer f, uint32 *args, uint32 sz); + +typedef float64 (*Float64FuncPtr)(GenericFunctionPointer f, uint32 *, uint32); +typedef float32 (*Float32FuncPtr)(GenericFunctionPointer f, uint32 *, uint32); +typedef int64 (*Int64FuncPtr)(GenericFunctionPointer f, uint32 *, uint32); +typedef int32 (*Int32FuncPtr)(GenericFunctionPointer f, uint32 *, uint32); +typedef void (*VoidFuncPtr)(GenericFunctionPointer f, uint32 *, uint32); + +static volatile Int64FuncPtr invokeNative_Int64 = + (Int64FuncPtr)(uintptr_t)invokeNative; +static volatile Int32FuncPtr invokeNative_Int32 = + (Int32FuncPtr)(uintptr_t)invokeNative; +static volatile Float64FuncPtr invokeNative_Float64 = + (Float64FuncPtr)(uintptr_t)invokeNative; +static volatile Float32FuncPtr invokeNative_Float32 = + (Float32FuncPtr)(uintptr_t)invokeNative; +static volatile VoidFuncPtr invokeNative_Void = + (VoidFuncPtr)(uintptr_t)invokeNative; + +static inline void +word_copy(uint32 *dest, uint32 *src, unsigned num) +{ + for (; num > 0; num--) + *dest++ = *src++; +} + +bool +wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, + const WASMFuncType *func_type, const char *signature, + void *attachment, uint32 *argv, uint32 argc, + uint32 *argv_ret) +{ + WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env); + uint32 argv_buf[32], *argv1 = argv_buf, argc1, i, j = 0; + uint32 arg_i32, ptr_len; + uint32 result_count = func_type->result_count; + uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0; + uint64 size; + bool ret = false; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + bool is_aot_func = (NULL == signature); +#endif + +#if defined(BUILD_TARGET_X86_32) + argc1 = argc + ext_ret_count + 2; +#else + /* arm/thumb/mips/xtensa, 64-bit data must be 8 bytes aligned, + so we need to allocate more memory. */ + argc1 = func_type->param_count * 2 + ext_ret_count + 2; +#endif + + if (argc1 > sizeof(argv_buf) / sizeof(uint32)) { + size = sizeof(uint32) * (uint64)argc1; + if (!(argv1 = runtime_malloc((uint32)size, exec_env->module_inst, NULL, + 0))) { + return false; + } + } + + for (i = 0; i < sizeof(WASMExecEnv *) / sizeof(uint32); i++) + argv1[j++] = ((uint32 *)&exec_env)[i]; + + for (i = 0; i < func_type->param_count; i++) { + switch (func_type->types[i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif +#endif +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#endif + { + arg_i32 = *argv++; + + if (signature) { + if (signature[i + 1] == '*') { + /* param is a pointer */ + if (signature[i + 2] == '~') + /* pointer with length followed */ + ptr_len = *argv; + else + /* pointer without length followed */ + ptr_len = 1; + + if (!wasm_runtime_validate_app_addr( + module, (uint64)arg_i32, (uint64)ptr_len)) + goto fail; + + arg_i32 = (uintptr_t)wasm_runtime_addr_app_to_native( + module, (uint64)arg_i32); + } + else if (signature[i + 1] == '$') { + /* param is a string */ + if (!wasm_runtime_validate_app_str_addr( + module, (uint64)arg_i32)) + goto fail; + + arg_i32 = (uintptr_t)wasm_runtime_addr_app_to_native( + module, (uint64)arg_i32); + } + } + + argv1[j++] = arg_i32; + break; + } + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: +#if !defined(BUILD_TARGET_X86_32) + /* 64-bit data must be 8 bytes aligned in arm, thumb, mips + and xtensa */ + if (j & 1) + j++; +#endif + argv1[j++] = *argv++; + argv1[j++] = *argv++; + break; + case VALUE_TYPE_F32: + argv1[j++] = *argv++; + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + { + uint32 externref_idx = *argv++; + if (is_aot_func) { + argv1[j++] = externref_idx; + } + else { + void *externref_obj; + + if (!wasm_externref_ref2obj(externref_idx, &externref_obj)) + goto fail; + + argv1[j++] = (uintptr_t)externref_obj; + } + break; + } +#endif + default: + bh_assert(0); + break; + } + } + + /* Save extra result values' address to argv1 */ + word_copy(argv1 + j, argv, ext_ret_count); + + argc1 = j + ext_ret_count; + exec_env->attachment = attachment; + if (func_type->result_count == 0) { + invokeNative_Void(func_ptr, argv1, argc1); + } + else { + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif +#endif +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#endif + argv_ret[0] = + (uint32)invokeNative_Int32(func_ptr, argv1, argc1); + break; + case VALUE_TYPE_I64: + PUT_I64_TO_ADDR(argv_ret, + invokeNative_Int64(func_ptr, argv1, argc1)); + break; + case VALUE_TYPE_F32: + *(float32 *)argv_ret = + invokeNative_Float32(func_ptr, argv1, argc1); + break; + case VALUE_TYPE_F64: + PUT_F64_TO_ADDR(argv_ret, + invokeNative_Float64(func_ptr, argv1, argc1)); + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + { + if (is_aot_func) { + uint32 externref_idx = + (uint32)invokeNative_Int32(func_ptr, argv1, argc1); + argv_ret[0] = externref_idx; + } + else { + void *externref_obj = (void *)(uintptr_t)invokeNative_Int32( + func_ptr, argv1, argc1); + uint32 externref_idx; + if (!wasm_externref_obj2ref(exec_env->module_inst, + externref_obj, &externref_idx)) + goto fail; + argv_ret[0] = externref_idx; + } + break; + } +#endif + default: + bh_assert(0); + break; + } + } + exec_env->attachment = NULL; + + ret = !wasm_runtime_copy_exception(module, NULL); + +fail: + if (argv1 != argv_buf) + wasm_runtime_free(argv1); + return ret; +} + +#endif /* end of defined(BUILD_TARGET_X86_32) \ + || defined(BUILD_TARGET_ARM) \ + || defined(BUILD_TARGET_THUMB) \ + || defined(BUILD_TARGET_MIPS) \ + || defined(BUILD_TARGET_XTENSA) */ + +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_AARCH64) || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) + +#endif /* end of WASM_ENABLE_INVOKE_NATIVE != 0 */ + +#if WASM_ENABLE_SIMD != 0 +#ifdef v128 +#undef v128 +#endif + +#if defined(_WIN32) || defined(_WIN32_) +typedef union __declspec(intrin_type) __declspec(align(8)) v128 { + __int8 m128i_i8[16]; + __int16 m128i_i16[8]; + __int32 m128i_i32[4]; + __int64 m128i_i64[2]; + unsigned __int8 m128i_u8[16]; + unsigned __int16 m128i_u16[8]; + unsigned __int32 m128i_u32[4]; + unsigned __int64 m128i_u64[2]; +} v128; +#elif defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) +typedef long long v128 + __attribute__((__vector_size__(16), __may_alias__, __aligned__(1))); +#elif defined(BUILD_TARGET_AARCH64) +#include +typedef uint32x4_t __m128i; +#define v128 __m128i +#endif + +#endif /* end of WASM_ENABLE_SIMD != 0 */ + +typedef void (*GenericFunctionPointer)(void); +void +invokeNative(GenericFunctionPointer f, uint64 *args, uint64 n_stacks); + +typedef float64 (*Float64FuncPtr)(GenericFunctionPointer, uint64 *, uint64); +typedef float32 (*Float32FuncPtr)(GenericFunctionPointer, uint64 *, uint64); +typedef int64 (*Int64FuncPtr)(GenericFunctionPointer, uint64 *, uint64); +typedef int32 (*Int32FuncPtr)(GenericFunctionPointer, uint64 *, uint64); +typedef void (*VoidFuncPtr)(GenericFunctionPointer, uint64 *, uint64); + +/* NOLINTBEGIN */ +static volatile Float64FuncPtr invokeNative_Float64 = + (Float64FuncPtr)(uintptr_t)invokeNative; +static volatile Float32FuncPtr invokeNative_Float32 = + (Float32FuncPtr)(uintptr_t)invokeNative; +static volatile Int64FuncPtr invokeNative_Int64 = + (Int64FuncPtr)(uintptr_t)invokeNative; +static volatile Int32FuncPtr invokeNative_Int32 = + (Int32FuncPtr)(uintptr_t)invokeNative; +static volatile VoidFuncPtr invokeNative_Void = + (VoidFuncPtr)(uintptr_t)invokeNative; + +#if WASM_ENABLE_SIMD != 0 +typedef v128 (*V128FuncPtr)(GenericFunctionPointer, uint64 *, uint64); +static V128FuncPtr invokeNative_V128 = (V128FuncPtr)(uintptr_t)invokeNative; +#endif +/* NOLINTEND */ + +#if defined(_WIN32) || defined(_WIN32_) +#define MAX_REG_FLOATS 4 +#define MAX_REG_INTS 4 +#else /* else of defined(_WIN32) || defined(_WIN32_) */ +#define MAX_REG_FLOATS 8 +#if defined(BUILD_TARGET_AARCH64) || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) +#define MAX_REG_INTS 8 +#else +#define MAX_REG_INTS 6 +#endif /* end of defined(BUILD_TARGET_AARCH64) \ + || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) */ +#endif /* end of defined(_WIN32) || defined(_WIN32_) */ + +/* + * ASAN is not designed to work with custom stack unwind or other low-level + * things. Ignore a function that does some low-level magic. (e.g. walking + * through the thread's stack bypassing the frame boundaries) + */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((no_sanitize_address)) +#endif +bool +wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, + const WASMFuncType *func_type, const char *signature, + void *attachment, uint32 *argv, uint32 argc, + uint32 *argv_ret) +{ + WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env); +#if WASM_ENABLE_MEMORY64 != 0 + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module); + bool is_memory64 = memory ? memory->is_memory64 : false; +#endif + uint64 argv_buf[32] = { 0 }, *argv1 = argv_buf, *ints, *stacks, size, + arg_i64; + uint32 *argv_src = argv, i, argc1, n_ints = 0, n_stacks = 0; + uint32 arg_i32, ptr_len; + uint32 result_count = func_type->result_count; + uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0; + bool ret = false; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + bool is_aot_func = (NULL == signature); +#endif +#ifndef BUILD_TARGET_RISCV64_LP64 +#if WASM_ENABLE_SIMD == 0 + uint64 *fps; +#else + v128 *fps; +#endif +#else /* else of BUILD_TARGET_RISCV64_LP64 */ +#define fps ints +#endif /* end of BUILD_TARGET_RISCV64_LP64 */ + +#if defined(_WIN32) || defined(_WIN32_) || defined(BUILD_TARGET_RISCV64_LP64) + /* important difference in calling conventions */ +#define n_fps n_ints +#else + int n_fps = 0; +#endif + +#if WASM_ENABLE_SIMD == 0 + argc1 = 1 + MAX_REG_FLOATS + (uint32)func_type->param_count + ext_ret_count; +#else + argc1 = 1 + MAX_REG_FLOATS * 2 + (uint32)func_type->param_count * 2 + + ext_ret_count; +#endif + if (argc1 > sizeof(argv_buf) / sizeof(uint64)) { + size = sizeof(uint64) * (uint64)argc1; + if (!(argv1 = runtime_malloc((uint32)size, exec_env->module_inst, NULL, + 0))) { + return false; + } + } + +#ifndef BUILD_TARGET_RISCV64_LP64 +#if WASM_ENABLE_SIMD == 0 + fps = argv1; + ints = fps + MAX_REG_FLOATS; +#else + fps = (v128 *)argv1; + ints = (uint64 *)(fps + MAX_REG_FLOATS); +#endif +#else /* else of BUILD_TARGET_RISCV64_LP64 */ + ints = argv1; +#endif /* end of BUILD_TARGET_RISCV64_LP64 */ + stacks = ints + MAX_REG_INTS; + + ints[n_ints++] = (uint64)(uintptr_t)exec_env; + + for (i = 0; i < func_type->param_count; i++) { + switch (func_type->types[i]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#endif + { + arg_i32 = *argv_src++; + arg_i64 = arg_i32; + if (signature +#if WASM_ENABLE_MEMORY64 != 0 + && !is_memory64 +#endif + ) { + if (signature[i + 1] == '*') { + /* param is a pointer */ + if (signature[i + 2] == '~') + /* pointer with length followed */ + ptr_len = *argv_src; + else + /* pointer without length followed */ + ptr_len = 1; + + if (!wasm_runtime_validate_app_addr( + module, (uint64)arg_i32, (uint64)ptr_len)) + goto fail; + + arg_i64 = (uintptr_t)wasm_runtime_addr_app_to_native( + module, (uint64)arg_i32); + } + else if (signature[i + 1] == '$') { + /* param is a string */ + if (!wasm_runtime_validate_app_str_addr( + module, (uint64)arg_i32)) + goto fail; + + arg_i64 = (uintptr_t)wasm_runtime_addr_app_to_native( + module, (uint64)arg_i32); + } + } + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = arg_i64; + else + stacks[n_stacks++] = arg_i64; + break; + } + case VALUE_TYPE_I64: +#if WASM_ENABLE_MEMORY64 != 0 + { + arg_i64 = GET_I64_FROM_ADDR(argv_src); + argv_src += 2; + if (signature && is_memory64) { + /* TODO: memory64 pointer with length need a new symbol + * to represent type i64, with '~' still represent i32 + * length */ + if (signature[i + 1] == '*') { + /* param is a pointer */ + if (signature[i + 2] == '~') + /* pointer with length followed */ + ptr_len = *argv_src; + else + /* pointer without length followed */ + ptr_len = 1; + + if (!wasm_runtime_validate_app_addr(module, arg_i64, + (uint64)ptr_len)) + goto fail; + + arg_i64 = (uint64)wasm_runtime_addr_app_to_native( + module, arg_i64); + } + else if (signature[i + 1] == '$') { + /* param is a string */ + if (!wasm_runtime_validate_app_str_addr(module, + arg_i64)) + goto fail; + + arg_i64 = (uint64)wasm_runtime_addr_app_to_native( + module, arg_i64); + } + } + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = arg_i64; + else + stacks[n_stacks++] = arg_i64; + break; + } +#endif +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif +#endif + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = *(uint64 *)argv_src; + else + stacks[n_stacks++] = *(uint64 *)argv_src; + argv_src += 2; + break; + case VALUE_TYPE_F32: + if (n_fps < MAX_REG_FLOATS) { + *(float32 *)&fps[n_fps++] = *(float32 *)argv_src++; + } + else { + *(float32 *)&stacks[n_stacks++] = *(float32 *)argv_src++; + } + break; + case VALUE_TYPE_F64: + if (n_fps < MAX_REG_FLOATS) { + *(float64 *)&fps[n_fps++] = *(float64 *)argv_src; + } + else { + *(float64 *)&stacks[n_stacks++] = *(float64 *)argv_src; + } + argv_src += 2; + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + { + uint32 externref_idx = *argv_src++; + if (is_aot_func) { + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = externref_idx; + else + stacks[n_stacks++] = externref_idx; + } + else { + void *externref_obj; + + if (!wasm_externref_ref2obj(externref_idx, &externref_obj)) + goto fail; + + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = (uintptr_t)externref_obj; + else + stacks[n_stacks++] = (uintptr_t)externref_obj; + } + break; + } +#endif +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + if (n_fps < MAX_REG_FLOATS) { + *(v128 *)&fps[n_fps++] = *(v128 *)argv_src; + } + else { + *(v128 *)&stacks[n_stacks++] = *(v128 *)argv_src; + n_stacks++; + } + argv_src += 4; + break; +#endif + default: + bh_assert(0); + break; + } + } + + /* Save extra result values' address to argv1 */ + for (i = 0; i < ext_ret_count; i++) { + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = *(uint64 *)argv_src; + else + stacks[n_stacks++] = *(uint64 *)argv_src; + argv_src += 2; + } + + exec_env->attachment = attachment; + if (result_count == 0) { + invokeNative_Void(func_ptr, argv1, n_stacks); + } + else { + /* Invoke the native function and get the first result value */ + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_FUNCREF: +#endif + argv_ret[0] = + (uint32)invokeNative_Int32(func_ptr, argv1, n_stacks); + break; + case VALUE_TYPE_I64: +#if WASM_ENABLE_GC != 0 + case REF_TYPE_FUNCREF: + case REF_TYPE_EXTERNREF: + case REF_TYPE_ANYREF: + case REF_TYPE_EQREF: + case REF_TYPE_HT_NULLABLE: + case REF_TYPE_HT_NON_NULLABLE: + case REF_TYPE_I31REF: + case REF_TYPE_NULLFUNCREF: + case REF_TYPE_NULLEXTERNREF: + case REF_TYPE_STRUCTREF: + case REF_TYPE_ARRAYREF: + case REF_TYPE_NULLREF: +#if WASM_ENABLE_STRINGREF != 0 + case REF_TYPE_STRINGREF: + case REF_TYPE_STRINGVIEWWTF8: + case REF_TYPE_STRINGVIEWWTF16: + case REF_TYPE_STRINGVIEWITER: +#endif +#endif + PUT_I64_TO_ADDR(argv_ret, + invokeNative_Int64(func_ptr, argv1, n_stacks)); + break; + case VALUE_TYPE_F32: + *(float32 *)argv_ret = + invokeNative_Float32(func_ptr, argv1, n_stacks); + break; + case VALUE_TYPE_F64: + PUT_F64_TO_ADDR( + argv_ret, invokeNative_Float64(func_ptr, argv1, n_stacks)); + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + { + if (is_aot_func) { + argv_ret[0] = invokeNative_Int32(func_ptr, argv1, n_stacks); + } + else { + uint32 externref_idx; + void *externref_obj = (void *)(uintptr_t)invokeNative_Int64( + func_ptr, argv1, n_stacks); + + if (!wasm_externref_obj2ref(exec_env->module_inst, + externref_obj, &externref_idx)) + goto fail; + + argv_ret[0] = externref_idx; + } + break; + } +#endif +#if WASM_ENABLE_SIMD != 0 + case VALUE_TYPE_V128: + *(v128 *)argv_ret = + invokeNative_V128(func_ptr, argv1, n_stacks); + break; +#endif + default: + bh_assert(0); + break; + } + } + exec_env->attachment = NULL; + + ret = !wasm_runtime_copy_exception(module, NULL); +fail: + if (argv1 != argv_buf) + wasm_runtime_free(argv1); + + return ret; +} + +#endif /* end of defined(BUILD_TARGET_X86_64) \ + || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_AARCH64) \ + || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) */ + +bool +wasm_runtime_call_indirect(WASMExecEnv *exec_env, uint32 element_index, + uint32 argc, uint32 argv[]) +{ + bool ret = false; + + if (!wasm_runtime_exec_env_check(exec_env)) { + LOG_ERROR("Invalid exec env stack info."); + return false; + } + + /* this function is called from native code, so exec_env->handle and + exec_env->native_stack_boundary must have been set, we don't set + it again */ + +#if WASM_ENABLE_INTERP != 0 + if (exec_env->module_inst->module_type == Wasm_Module_Bytecode) + ret = wasm_call_indirect(exec_env, 0, element_index, argc, argv); +#endif +#if WASM_ENABLE_AOT != 0 + if (exec_env->module_inst->module_type == Wasm_Module_AoT) + ret = aot_call_indirect(exec_env, 0, element_index, argc, argv); +#endif + + return ret; +} + +static void +exchange_uint32(uint8 *p_data) +{ + uint8 value = *p_data; + *p_data = *(p_data + 3); + *(p_data + 3) = value; + + value = *(p_data + 1); + *(p_data + 1) = *(p_data + 2); + *(p_data + 2) = value; +} + +static void +exchange_uint64(uint8 *p_data) +{ + uint32 value; + + value = *(uint32 *)p_data; + *(uint32 *)p_data = *(uint32 *)(p_data + 4); + *(uint32 *)(p_data + 4) = value; + exchange_uint32(p_data); + exchange_uint32(p_data + 4); +} + +void +wasm_runtime_read_v128(const uint8 *bytes, uint64 *ret1, uint64 *ret2) +{ + uint64 u1, u2; + + bh_memcpy_s(&u1, 8, bytes, 8); + bh_memcpy_s(&u2, 8, bytes + 8, 8); + + if (!is_little_endian()) { + exchange_uint64((uint8 *)&u1); + exchange_uint64((uint8 *)&u2); + *ret1 = u2; + *ret2 = u1; + } + else { + *ret1 = u1; + *ret2 = u2; + } +} + +#if WASM_ENABLE_THREAD_MGR != 0 +typedef struct WASMThreadArg { + WASMExecEnv *new_exec_env; + wasm_thread_callback_t callback; + void *arg; +} WASMThreadArg; + +WASMExecEnv * +wasm_runtime_spawn_exec_env(WASMExecEnv *exec_env) +{ + return wasm_cluster_spawn_exec_env(exec_env); +} + +void +wasm_runtime_destroy_spawned_exec_env(WASMExecEnv *exec_env) +{ + wasm_cluster_destroy_spawned_exec_env(exec_env); +} + +static void * +wasm_runtime_thread_routine(void *arg) +{ + WASMThreadArg *thread_arg = (WASMThreadArg *)arg; + void *ret; + + bh_assert(thread_arg->new_exec_env); + ret = thread_arg->callback(thread_arg->new_exec_env, thread_arg->arg); + + wasm_runtime_destroy_spawned_exec_env(thread_arg->new_exec_env); + wasm_runtime_free(thread_arg); + + os_thread_exit(ret); + return ret; +} + +int32 +wasm_runtime_spawn_thread(WASMExecEnv *exec_env, wasm_thread_t *tid, + wasm_thread_callback_t callback, void *arg) +{ + WASMExecEnv *new_exec_env = wasm_runtime_spawn_exec_env(exec_env); + WASMThreadArg *thread_arg; + int32 ret; + + if (!new_exec_env) + return -1; + + if (!(thread_arg = wasm_runtime_malloc(sizeof(WASMThreadArg)))) { + wasm_runtime_destroy_spawned_exec_env(new_exec_env); + return -1; + } + + thread_arg->new_exec_env = new_exec_env; + thread_arg->callback = callback; + thread_arg->arg = arg; + + ret = os_thread_create((korp_tid *)tid, wasm_runtime_thread_routine, + thread_arg, APP_THREAD_STACK_SIZE_DEFAULT); + + if (ret != 0) { + wasm_runtime_destroy_spawned_exec_env(new_exec_env); + wasm_runtime_free(thread_arg); + } + + return ret; +} + +int32 +wasm_runtime_join_thread(wasm_thread_t tid, void **retval) +{ + return os_thread_join((korp_tid)tid, retval); +} + +#endif /* end of WASM_ENABLE_THREAD_MGR */ + +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + +static korp_mutex externref_lock; +static uint32 externref_global_id = 1; +static HashMap *externref_map; + +typedef struct ExternRefMapNode { + /* The extern object from runtime embedder */ + void *extern_obj; + /* The module instance it belongs to */ + WASMModuleInstanceCommon *module_inst; + /* Whether it is retained */ + bool retained; + /* Whether it is marked by runtime */ + bool marked; + /* cleanup function called when the externref is freed */ + void (*cleanup)(void *); +} ExternRefMapNode; + +static uint32 +wasm_externref_hash(const void *key) +{ + uint32 externref_idx = (uint32)(uintptr_t)key; + return externref_idx; +} + +static bool +wasm_externref_equal(void *key1, void *key2) +{ + uint32 externref_idx1 = (uint32)(uintptr_t)key1; + uint32 externref_idx2 = (uint32)(uintptr_t)key2; + return externref_idx1 == externref_idx2 ? true : false; +} + +static bool +wasm_externref_map_init() +{ + if (os_mutex_init(&externref_lock) != 0) + return false; + + if (!(externref_map = bh_hash_map_create(32, false, wasm_externref_hash, + wasm_externref_equal, NULL, + wasm_runtime_free))) { + os_mutex_destroy(&externref_lock); + return false; + } + + externref_global_id = 1; + return true; +} + +static void +wasm_externref_map_destroy() +{ + bh_hash_map_destroy(externref_map); + os_mutex_destroy(&externref_lock); +} + +typedef struct LookupExtObj_UserData { + ExternRefMapNode node; + bool found; + uint32 externref_idx; +} LookupExtObj_UserData; + +static void +lookup_extobj_callback(void *key, void *value, void *user_data) +{ + uint32 externref_idx = (uint32)(uintptr_t)key; + ExternRefMapNode *node = (ExternRefMapNode *)value; + LookupExtObj_UserData *user_data_lookup = + (LookupExtObj_UserData *)user_data; + + if (node->extern_obj == user_data_lookup->node.extern_obj + && node->module_inst == user_data_lookup->node.module_inst) { + user_data_lookup->found = true; + user_data_lookup->externref_idx = externref_idx; + } +} + +static void +delete_externref(void *key, ExternRefMapNode *node) +{ + bh_hash_map_remove(externref_map, key, NULL, NULL); + if (node->cleanup) { + (*node->cleanup)(node->extern_obj); + } + wasm_runtime_free(node); +} + +static void +delete_extobj_callback(void *key, void *value, void *user_data) +{ + ExternRefMapNode *node = (ExternRefMapNode *)value; + LookupExtObj_UserData *lookup_user_data = + (LookupExtObj_UserData *)user_data; + + if (node->extern_obj == lookup_user_data->node.extern_obj + && node->module_inst == lookup_user_data->node.module_inst) { + lookup_user_data->found = true; + delete_externref(key, node); + } +} + +bool +wasm_externref_objdel(WASMModuleInstanceCommon *module_inst, void *extern_obj) +{ + LookupExtObj_UserData lookup_user_data = { 0 }; + bool ok = false; + + /* in a wrapper, extern_obj could be any value */ + lookup_user_data.node.extern_obj = extern_obj; + lookup_user_data.node.module_inst = module_inst; + lookup_user_data.found = false; + + os_mutex_lock(&externref_lock); + /* Lookup hashmap firstly */ + bh_hash_map_traverse(externref_map, delete_extobj_callback, + (void *)&lookup_user_data); + if (lookup_user_data.found) { + ok = true; + } + os_mutex_unlock(&externref_lock); + + return ok; +} + +bool +wasm_externref_set_cleanup(WASMModuleInstanceCommon *module_inst, + void *extern_obj, void (*extern_obj_cleanup)(void *)) +{ + + LookupExtObj_UserData lookup_user_data = { 0 }; + bool ok = false; + + /* in a wrapper, extern_obj could be any value */ + lookup_user_data.node.extern_obj = extern_obj; + lookup_user_data.node.module_inst = module_inst; + lookup_user_data.found = false; + + os_mutex_lock(&externref_lock); + /* Lookup hashmap firstly */ + bh_hash_map_traverse(externref_map, lookup_extobj_callback, + (void *)&lookup_user_data); + if (lookup_user_data.found) { + void *key = (void *)(uintptr_t)lookup_user_data.externref_idx; + ExternRefMapNode *node = bh_hash_map_find(externref_map, key); + bh_assert(node); + node->cleanup = extern_obj_cleanup; + ok = true; + } + os_mutex_unlock(&externref_lock); + + return ok; +} + +bool +wasm_externref_obj2ref(WASMModuleInstanceCommon *module_inst, void *extern_obj, + uint32 *p_externref_idx) +{ + LookupExtObj_UserData lookup_user_data = { 0 }; + ExternRefMapNode *node; + uint32 externref_idx; + + /* + * to catch a parameter from `wasm_application_execute_func`, + * which represents a string 'null' + */ +#if UINTPTR_MAX == UINT32_MAX + if ((uint32)-1 == (uintptr_t)extern_obj) { +#else + if ((uint64)-1LL == (uintptr_t)extern_obj) { +#endif + *p_externref_idx = NULL_REF; + return true; + } + + /* in a wrapper, extern_obj could be any value */ + lookup_user_data.node.extern_obj = extern_obj; + lookup_user_data.node.module_inst = module_inst; + lookup_user_data.found = false; + + os_mutex_lock(&externref_lock); + + /* Lookup hashmap firstly */ + bh_hash_map_traverse(externref_map, lookup_extobj_callback, + (void *)&lookup_user_data); + if (lookup_user_data.found) { + *p_externref_idx = lookup_user_data.externref_idx; + os_mutex_unlock(&externref_lock); + return true; + } + + /* Not found in hashmap */ + if (externref_global_id == NULL_REF || externref_global_id == 0) { + goto fail1; + } + + if (!(node = wasm_runtime_malloc(sizeof(ExternRefMapNode)))) { + goto fail1; + } + + memset(node, 0, sizeof(ExternRefMapNode)); + node->extern_obj = extern_obj; + node->module_inst = module_inst; + node->cleanup = NULL; + + externref_idx = externref_global_id; + + if (!bh_hash_map_insert(externref_map, (void *)(uintptr_t)externref_idx, + (void *)node)) { + goto fail2; + } + + externref_global_id++; + *p_externref_idx = externref_idx; + os_mutex_unlock(&externref_lock); + return true; +fail2: + wasm_runtime_free(node); +fail1: + os_mutex_unlock(&externref_lock); + return false; +} + +bool +wasm_externref_ref2obj(uint32 externref_idx, void **p_extern_obj) +{ + ExternRefMapNode *node; + + /* catch a `ref.null` variable */ + if (externref_idx == NULL_REF) { + *p_extern_obj = NULL; + return true; + } + + os_mutex_lock(&externref_lock); + node = bh_hash_map_find(externref_map, (void *)(uintptr_t)externref_idx); + os_mutex_unlock(&externref_lock); + + if (!node) + return false; + + *p_extern_obj = node->extern_obj; + return true; +} + +static void +reclaim_extobj_callback(void *key, void *value, void *user_data) +{ + ExternRefMapNode *node = (ExternRefMapNode *)value; + WASMModuleInstanceCommon *module_inst = + (WASMModuleInstanceCommon *)user_data; + + if (node->module_inst == module_inst) { + if (!node->marked && !node->retained) { + delete_externref(key, node); + } + else { + node->marked = false; + } + } +} + +static void +mark_externref(uint32 externref_idx) +{ + ExternRefMapNode *node; + + if (externref_idx != NULL_REF) { + node = + bh_hash_map_find(externref_map, (void *)(uintptr_t)externref_idx); + if (node) { + node->marked = true; + } + } +} + +#if WASM_ENABLE_INTERP != 0 +static void +interp_mark_all_externrefs(WASMModuleInstance *module_inst) +{ + uint32 i, j, externref_idx; + table_elem_type_t *table_data; + uint8 *global_data = module_inst->global_data; + WASMGlobalInstance *global; + WASMTableInstance *table; + + global = module_inst->e->globals; + for (i = 0; i < module_inst->e->global_count; i++, global++) { + if (global->type == VALUE_TYPE_EXTERNREF) { + externref_idx = *(uint32 *)(global_data + global->data_offset); + mark_externref(externref_idx); + } + } + + for (i = 0; i < module_inst->table_count; i++) { + uint8 elem_type = 0; + uint32 init_size, max_size; + + table = wasm_get_table_inst(module_inst, i); + (void)wasm_runtime_get_table_inst_elem_type( + (WASMModuleInstanceCommon *)module_inst, i, &elem_type, &init_size, + &max_size); + + if (elem_type == VALUE_TYPE_EXTERNREF) { + table_data = table->elems; + for (j = 0; j < table->cur_size; j++) { + externref_idx = table_data[j]; + mark_externref(externref_idx); + } + } + (void)init_size; + (void)max_size; + } +} +#endif + +#if WASM_ENABLE_AOT != 0 +static void +aot_mark_all_externrefs(AOTModuleInstance *module_inst) +{ + uint32 i = 0, j = 0; + const AOTModule *module = (AOTModule *)module_inst->module; + const AOTTable *table = module->tables; + const AOTGlobal *global = module->globals; + const AOTTableInstance *table_inst; + + for (i = 0; i < module->global_count; i++, global++) { + if (global->type.val_type == VALUE_TYPE_EXTERNREF) { + mark_externref( + *(uint32 *)(module_inst->global_data + global->data_offset)); + } + } + + for (i = 0; i < module->table_count; i++) { + table_inst = module_inst->tables[i]; + if ((table + i)->table_type.elem_type == VALUE_TYPE_EXTERNREF) { + while (j < table_inst->cur_size) { + mark_externref(table_inst->elems[j++]); + } + } + } +} +#endif + +void +wasm_externref_reclaim(WASMModuleInstanceCommon *module_inst) +{ + os_mutex_lock(&externref_lock); +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + interp_mark_all_externrefs((WASMModuleInstance *)module_inst); +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + aot_mark_all_externrefs((AOTModuleInstance *)module_inst); +#endif + + bh_hash_map_traverse(externref_map, reclaim_extobj_callback, + (void *)module_inst); + os_mutex_unlock(&externref_lock); +} + +static void +cleanup_extobj_callback(void *key, void *value, void *user_data) +{ + ExternRefMapNode *node = (ExternRefMapNode *)value; + WASMModuleInstanceCommon *module_inst = + (WASMModuleInstanceCommon *)user_data; + + if (node->module_inst == module_inst) { + delete_externref(key, node); + } +} + +void +wasm_externref_cleanup(WASMModuleInstanceCommon *module_inst) +{ + os_mutex_lock(&externref_lock); + bh_hash_map_traverse(externref_map, cleanup_extobj_callback, + (void *)module_inst); + os_mutex_unlock(&externref_lock); +} + +bool +wasm_externref_retain(uint32 externref_idx) +{ + ExternRefMapNode *node; + + os_mutex_lock(&externref_lock); + + if (externref_idx != NULL_REF) { + node = + bh_hash_map_find(externref_map, (void *)(uintptr_t)externref_idx); + if (node) { + node->retained = true; + os_mutex_unlock(&externref_lock); + return true; + } + } + + os_mutex_unlock(&externref_lock); + return false; +} +#endif /* end of WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 */ + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 +uint32 +wasm_runtime_dump_line_buf_impl(const char *line_buf, bool dump_or_print, + char **buf, uint32 *len) +{ + if (dump_or_print) { + return (uint32)os_printf("%s", line_buf); + } + else if (*buf) { + uint32 dump_len; + + dump_len = snprintf(*buf, *len, "%s", line_buf); + if (dump_len >= *len) { + dump_len = *len; + } + + *len = *len - dump_len; + *buf = *buf + dump_len; + return dump_len; + } + else { + return (uint32)strlen(line_buf); + } +} + +void +wasm_runtime_dump_call_stack(WASMExecEnv *exec_env) +{ + WASMModuleInstanceCommon *module_inst = + wasm_exec_env_get_module_inst(exec_env); +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + wasm_interp_dump_call_stack(exec_env, true, NULL, 0); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + aot_dump_call_stack(exec_env, true, NULL, 0); + } +#endif +} + +uint32 +wasm_runtime_get_call_stack_buf_size(wasm_exec_env_t exec_env) +{ + WASMModuleInstanceCommon *module_inst = + wasm_exec_env_get_module_inst(exec_env); + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + return wasm_interp_dump_call_stack(exec_env, false, NULL, 0); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + return aot_dump_call_stack(exec_env, false, NULL, 0); + } +#endif + + return 0; +} + +uint32 +wasm_runtime_dump_call_stack_to_buf(wasm_exec_env_t exec_env, char *buf, + uint32 len) +{ + WASMModuleInstanceCommon *module_inst = + wasm_exec_env_get_module_inst(exec_env); + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + return wasm_interp_dump_call_stack(exec_env, false, buf, len); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + return aot_dump_call_stack(exec_env, false, buf, len); + } +#endif + + return 0; +} +#endif /* end of WASM_ENABLE_DUMP_CALL_STACK */ + +#if WASM_ENABLE_STATIC_PGO != 0 +uint32 +wasm_runtime_get_pgo_prof_data_size(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstance *aot_inst = (AOTModuleInstance *)module_inst; + return aot_get_pgo_prof_data_size(aot_inst); + } +#endif + return 0; +} + +uint32 +wasm_runtime_dump_pgo_prof_data_to_buf(WASMModuleInstanceCommon *module_inst, + char *buf, uint32 len) +{ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstance *aot_inst = (AOTModuleInstance *)module_inst; + return aot_dump_pgo_prof_data_to_buf(aot_inst, buf, len); + } +#endif + return 0; +} +#endif /* end of WASM_ENABLE_STATIC_PGO != 0 */ + +bool +wasm_runtime_get_table_elem_type(const WASMModuleCommon *module_comm, + uint32 table_idx, uint8 *out_elem_type, +#if WASM_ENABLE_GC != 0 + WASMRefType **out_ref_type, +#endif + uint32 *out_min_size, uint32 *out_max_size) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_comm->module_type == Wasm_Module_Bytecode) { + WASMModule *module = (WASMModule *)module_comm; + + if (table_idx < module->import_table_count) { + WASMTableImport *import_table = + &((module->import_tables + table_idx)->u.table); + *out_elem_type = import_table->table_type.elem_type; +#if WASM_ENABLE_GC != 0 + *out_ref_type = import_table->table_type.elem_ref_type; +#endif + *out_min_size = import_table->table_type.init_size; + *out_max_size = import_table->table_type.max_size; + } + else { + WASMTable *table = + module->tables + (table_idx - module->import_table_count); + *out_elem_type = table->table_type.elem_type; +#if WASM_ENABLE_GC != 0 + *out_ref_type = table->table_type.elem_ref_type; +#endif + *out_min_size = table->table_type.init_size; + *out_max_size = table->table_type.max_size; + } + return true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_comm->module_type == Wasm_Module_AoT) { + AOTModule *module = (AOTModule *)module_comm; + + if (table_idx < module->import_table_count) { + AOTImportTable *import_table = module->import_tables + table_idx; + *out_elem_type = import_table->table_type.elem_type; +#if WASM_ENABLE_GC != 0 + *out_ref_type = NULL; /* TODO */ +#endif + *out_min_size = import_table->table_type.init_size; + *out_max_size = import_table->table_type.max_size; + } + else { + AOTTable *table = + module->tables + (table_idx - module->import_table_count); + *out_elem_type = table->table_type.elem_type; +#if WASM_ENABLE_GC != 0 + *out_ref_type = NULL; /* TODO */ +#endif + *out_min_size = table->table_type.init_size; + *out_max_size = table->table_type.max_size; + } + return true; + } +#endif + + return false; +} + +bool +wasm_runtime_get_table_inst_elem_type( + const WASMModuleInstanceCommon *module_inst_comm, uint32 table_idx, + uint8 *out_elem_type, +#if WASM_ENABLE_GC != 0 + WASMRefType **out_ref_type, +#endif + uint32 *out_min_size, uint32 *out_max_size) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + + bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode + || module_inst_comm->module_type == Wasm_Module_AoT); + + return wasm_runtime_get_table_elem_type( + (WASMModuleCommon *)module_inst->module, table_idx, out_elem_type, +#if WASM_ENABLE_GC != 0 + out_ref_type, +#endif + out_min_size, out_max_size); +} + +bool +wasm_runtime_get_export_func_type(const WASMModuleCommon *module_comm, + const WASMExport *export, WASMFuncType **out) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_comm->module_type == Wasm_Module_Bytecode) { + WASMModule *module = (WASMModule *)module_comm; + + if (export->index < module->import_function_count) { + *out = module->import_functions[export->index].u.function.func_type; + } + else { + *out = + module->functions[export->index - module->import_function_count] + ->func_type; + } + return true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_comm->module_type == Wasm_Module_AoT) { + AOTModule *module = (AOTModule *)module_comm; + + if (export->index < module->import_func_count) { + *out = (WASMFuncType *) + module->types[module->import_funcs[export->index] + .func_type_index]; + } + else { + *out = (WASMFuncType *)module + ->types[module->func_type_indexes + [export->index - module->import_func_count]]; + } + return true; + } +#endif + return false; +} + +bool +wasm_runtime_get_export_global_type(const WASMModuleCommon *module_comm, + const WASMExport *export, + uint8 *out_val_type, bool *out_mutability) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_comm->module_type == Wasm_Module_Bytecode) { + WASMModule *module = (WASMModule *)module_comm; + + if (export->index < module->import_global_count) { + WASMGlobalImport *import_global = + &((module->import_globals + export->index)->u.global); + *out_val_type = import_global->type.val_type; + *out_mutability = import_global->type.is_mutable; + } + else { + WASMGlobal *global = + module->globals + (export->index - module->import_global_count); + *out_val_type = global->type.val_type; + *out_mutability = global->type.is_mutable; + } + return true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_comm->module_type == Wasm_Module_AoT) { + AOTModule *module = (AOTModule *)module_comm; + + if (export->index < module->import_global_count) { + AOTImportGlobal *import_global = + module->import_globals + export->index; + *out_val_type = import_global->type.val_type; + *out_mutability = import_global->type.is_mutable; + } + else { + AOTGlobal *global = + module->globals + (export->index - module->import_global_count); + *out_val_type = global->type.val_type; + *out_mutability = global->type.is_mutable; + } + return true; + } +#endif + return false; +} + +bool +wasm_runtime_get_export_memory_type(const WASMModuleCommon *module_comm, + const WASMExport *export, + uint32 *out_min_page, uint32 *out_max_page) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_comm->module_type == Wasm_Module_Bytecode) { + WASMModule *module = (WASMModule *)module_comm; + + if (export->index < module->import_memory_count) { + WASMMemoryImport *import_memory = + &((module->import_memories + export->index)->u.memory); + *out_min_page = import_memory->mem_type.init_page_count; + *out_max_page = import_memory->mem_type.max_page_count; + } + else { + WASMMemory *memory = + module->memories + + (export->index - module->import_memory_count); + *out_min_page = memory->init_page_count; + *out_max_page = memory->max_page_count; + } + return true; + } +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_comm->module_type == Wasm_Module_AoT) { + AOTModule *module = (AOTModule *)module_comm; + + if (export->index < module->import_memory_count) { + AOTImportMemory *import_memory = + module->import_memories + export->index; + *out_min_page = import_memory->mem_type.init_page_count; + *out_max_page = import_memory->mem_type.max_page_count; + } + else { + AOTMemory *memory = module->memories + + (export->index - module->import_memory_count); + *out_min_page = memory->init_page_count; + *out_max_page = memory->max_page_count; + } + return true; + } +#endif + return false; +} + +bool +wasm_runtime_get_export_table_type(const WASMModuleCommon *module_comm, + const WASMExport *export, + uint8 *out_elem_type, +#if WASM_ENABLE_GC != 0 + WASMRefType **out_ref_type, +#endif + uint32 *out_min_size, uint32 *out_max_size) +{ + return wasm_runtime_get_table_elem_type(module_comm, export->index, + out_elem_type, +#if WASM_ENABLE_GC != 0 + out_ref_type, +#endif + out_min_size, out_max_size); +} + +static inline bool +argv_to_params(wasm_val_t *out_params, const uint32 *argv, + WASMFuncType *func_type) +{ + wasm_val_t *param = out_params; + uint32 i = 0, *u32; + + for (i = 0; i < func_type->param_count; i++, param++) { + switch (func_type->types[i]) { + case VALUE_TYPE_I32: + param->kind = WASM_I32; + param->of.i32 = *argv++; + break; + case VALUE_TYPE_I64: + param->kind = WASM_I64; + u32 = (uint32 *)¶m->of.i64; + u32[0] = *argv++; + u32[1] = *argv++; + break; + case VALUE_TYPE_F32: + param->kind = WASM_F32; + param->of.f32 = *(float32 *)argv++; + break; + case VALUE_TYPE_F64: + param->kind = WASM_F64; + u32 = (uint32 *)¶m->of.i64; + u32[0] = *argv++; + u32[1] = *argv++; + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + param->kind = WASM_EXTERNREF; + + if (!wasm_externref_ref2obj(*argv, + (void **)¶m->of.foreign)) { + return false; + } + + argv++; + break; +#endif + default: + return false; + } + } + + return true; +} + +static inline bool +results_to_argv(WASMModuleInstanceCommon *module_inst, uint32 *out_argv, + const wasm_val_t *results, WASMFuncType *func_type) +{ + const wasm_val_t *result = results; + uint32 *argv = out_argv, *u32, i; + uint8 *result_types = func_type->types + func_type->param_count; + + for (i = 0; i < func_type->result_count; i++, result++) { + switch (result_types[i]) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: + *(int32 *)argv++ = result->of.i32; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + u32 = (uint32 *)&result->of.i64; + *argv++ = u32[0]; + *argv++ = u32[1]; + break; +#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 + case VALUE_TYPE_EXTERNREF: + if (!wasm_externref_obj2ref(module_inst, + (void *)result->of.foreign, + (uint32 *)argv)) { + return false; + } + argv++; + break; +#endif + default: + return false; + } + } + + return true; +} + +bool +wasm_runtime_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, + void *func_ptr, WASMFuncType *func_type, + uint32 argc, uint32 *argv, bool with_env, + void *wasm_c_api_env) +{ + wasm_val_t params_buf[16] = { 0 }, results_buf[4] = { 0 }; + wasm_val_t *params = params_buf, *results = results_buf; + wasm_trap_t *trap = NULL; + bool ret = false; + wasm_val_vec_t params_vec = { 0 }, results_vec = { 0 }; + + if (func_type->param_count > 16) { + if (!(params = + runtime_malloc(sizeof(wasm_val_t) * func_type->param_count, + module_inst, NULL, 0))) { + wasm_runtime_set_exception(module_inst, "allocate memory failed"); + return false; + } + } + + if (!argv_to_params(params, argv, func_type)) { + wasm_runtime_set_exception(module_inst, "unsupported param type"); + goto fail; + } + + if (func_type->result_count > 4) { + if (!(results = + runtime_malloc(sizeof(wasm_val_t) * func_type->result_count, + module_inst, NULL, 0))) { + wasm_runtime_set_exception(module_inst, "allocate memory failed"); + goto fail; + } + } + + params_vec.data = params; + params_vec.num_elems = func_type->param_count; + params_vec.size = func_type->param_count; + + results_vec.data = results; + results_vec.num_elems = 0; + results_vec.size = func_type->result_count; + + if (!with_env) { + wasm_func_callback_t callback = (wasm_func_callback_t)func_ptr; + trap = callback(¶ms_vec, &results_vec); + } + else { + wasm_func_callback_with_env_t callback = + (wasm_func_callback_with_env_t)func_ptr; + trap = callback(wasm_c_api_env, ¶ms_vec, &results_vec); + } + + if (trap) { + if (trap->message->data) { + /* since trap->message->data does not end with '\0' */ + char trap_message[108] = { 0 }; + uint32 max_size_to_copy = (uint32)sizeof(trap_message) - 1; + uint32 size_to_copy = (trap->message->size < max_size_to_copy) + ? (uint32)trap->message->size + : max_size_to_copy; + bh_memcpy_s(trap_message, (uint32)sizeof(trap_message), + trap->message->data, size_to_copy); + wasm_runtime_set_exception(module_inst, trap_message); + } + else { + wasm_runtime_set_exception( + module_inst, "native function throw unknown exception"); + } + wasm_trap_delete(trap); + goto fail; + } + + if (!results_to_argv(module_inst, argv, results, func_type)) { + wasm_runtime_set_exception(module_inst, "unsupported result type"); + goto fail; + } + ret = true; + +fail: + if (params != params_buf) + wasm_runtime_free(params); + if (results != results_buf) + wasm_runtime_free(results); + return ret; +} + +bool +wasm_runtime_quick_invoke_c_api_native(WASMModuleInstanceCommon *inst_comm, + CApiFuncImport *c_api_import, + wasm_val_t *params, uint32 param_count, + wasm_val_t *results, uint32 result_count) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)inst_comm; + void *func_ptr = c_api_import->func_ptr_linked; + bool with_env_arg = c_api_import->with_env_arg, ret = true; + wasm_val_vec_t params_vec = { 0 }, results_vec = { 0 }; + wasm_trap_t *trap = NULL; + + params_vec.data = params; + params_vec.num_elems = param_count; + params_vec.size = param_count; + + results_vec.data = results; + results_vec.num_elems = 0; + results_vec.size = result_count; + + if (!func_ptr) { + wasm_set_exception_with_id(module_inst, EXCE_CALL_UNLINKED_IMPORT_FUNC); + ret = false; + goto fail; + } + + if (!with_env_arg) { + wasm_func_callback_t callback = (wasm_func_callback_t)func_ptr; + trap = callback(¶ms_vec, &results_vec); + } + else { + void *wasm_c_api_env = c_api_import->env_arg; + wasm_func_callback_with_env_t callback = + (wasm_func_callback_with_env_t)func_ptr; + trap = callback(wasm_c_api_env, ¶ms_vec, &results_vec); + } + + if (trap) { + if (trap->message->data) { + /* since trap->message->data does not end with '\0' */ + char trap_message[108] = { 0 }; + uint32 max_size_to_copy = (uint32)sizeof(trap_message) - 1; + uint32 size_to_copy = (trap->message->size < max_size_to_copy) + ? (uint32)trap->message->size + : max_size_to_copy; + bh_memcpy_s(trap_message, (uint32)sizeof(trap_message), + trap->message->data, size_to_copy); + wasm_set_exception(module_inst, trap_message); + } + else { + wasm_set_exception(module_inst, + "native function throw unknown exception"); + } + wasm_trap_delete(trap); + ret = false; + } + +fail: +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!ret) + wasm_runtime_access_exce_check_guard_page(); +#endif + return ret; +} + +void +wasm_runtime_show_app_heap_corrupted_prompt() +{ + LOG_ERROR("Error: app heap is corrupted, if the wasm file " + "is compiled by wasi-sdk-12.0 or higher version, " + "please add -Wl,--export=malloc -Wl,--export=free " + "to export malloc and free functions. If it is " + "compiled by asc, please add --exportRuntime to " + "export the runtime helpers."); +} + +#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 +void +wasm_runtime_destroy_custom_sections(WASMCustomSection *section_list) +{ + WASMCustomSection *section = section_list, *next; + while (section) { + next = section->next; + wasm_runtime_free(section); + section = next; + } +} +#endif /* end of WASM_ENABLE_LOAD_CUSTOM_SECTION */ + +void +wasm_runtime_get_version(uint32_t *major, uint32_t *minor, uint32_t *patch) +{ + *major = WAMR_VERSION_MAJOR; + *minor = WAMR_VERSION_MINOR; + *patch = WAMR_VERSION_PATCH; +} + +bool +wasm_runtime_is_import_func_linked(const char *module_name, + const char *func_name) +{ +#if WASM_EANBLE_INVOKE_NATIVE != 0 + return wasm_native_resolve_symbol(module_name, func_name, NULL, NULL, NULL, + NULL); +#else + return false; +#endif +} + +bool +wasm_runtime_is_import_global_linked(const char *module_name, + const char *global_name) +{ +#if WASM_ENABLE_LIBC_BUILTIN != 0 + WASMGlobalImport global = { 0 }; + return wasm_native_lookup_libc_builtin_global(module_name, global_name, + &global); +#else + return false; +#endif +} + +#if WASM_ENABLE_LIBC_WASI != 0 || WASM_ENABLE_MULTI_MODULE != 0 +WASMExport * +loader_find_export(const WASMModuleCommon *module, const char *module_name, + const char *field_name, uint8 export_kind, char *error_buf, + uint32 error_buf_size) +{ + WASMExport *exports = NULL, *result = NULL, *export; + uint32 export_count = 0, i; +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + AOTModule *aot_module = (AOTModule *)module; + exports = (WASMExport *)aot_module->exports; + export_count = aot_module->export_count; + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + WASMModule *wasm_module = (WASMModule *)module; + exports = wasm_module->exports; + export_count = wasm_module->export_count; + } +#endif + for (i = 0, export = exports; i < export_count; ++i, ++export) { + if (export->kind == export_kind && !strcmp(field_name, export->name)) { + result = export; + goto exit; + } + } + if (i == export_count) { + LOG_DEBUG("can not find an export %d named %s in the module %s", + export_kind, field_name, module_name); + set_error_buf(error_buf, error_buf_size, + "unknown import or incompatible import type"); + } +exit: + return result; +} +#endif + +#if WASM_ENABLE_MULTI_MODULE != 0 +WASMModuleCommon * +wasm_runtime_search_sub_module(const WASMModuleCommon *parent_module, + const char *sub_module_name) +{ + WASMRegisteredModule *node = NULL; +#if WASM_ENABLE_AOT != 0 + if (parent_module->module_type == Wasm_Module_AoT) { + node = bh_list_first_elem( + ((AOTModule *)parent_module)->import_module_list); + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (parent_module->module_type == Wasm_Module_Bytecode) { + node = bh_list_first_elem( + ((WASMModule *)parent_module)->import_module_list); + } +#endif + while (node && strcmp(sub_module_name, node->module_name)) { + node = bh_list_elem_next(node); + } + return node ? node->module : NULL; +} + +bool +wasm_runtime_register_sub_module(const WASMModuleCommon *parent_module, + const char *sub_module_name, + WASMModuleCommon *sub_module) +{ + /* register sub_module into its parent sub module list */ + WASMRegisteredModule *node = NULL; + bh_list_status ret = BH_LIST_ERROR; + + if (wasm_runtime_search_sub_module(parent_module, sub_module_name)) { + LOG_DEBUG("%s has been registered in its parent", sub_module_name); + return true; + } + + node = loader_malloc(sizeof(WASMRegisteredModule), NULL, 0); + if (!node) { + return false; + } + + node->module_name = sub_module_name; + node->module = sub_module; +#if WASM_ENABLE_AOT != 0 + if (parent_module->module_type == Wasm_Module_AoT) { + ret = bh_list_insert(((AOTModule *)parent_module)->import_module_list, + node); + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (parent_module->module_type == Wasm_Module_Bytecode) { + ret = bh_list_insert(((WASMModule *)parent_module)->import_module_list, + node); + } +#endif + bh_assert(BH_LIST_SUCCESS == ret); + (void)ret; + return true; +} + +WASMModuleCommon * +wasm_runtime_load_depended_module(const WASMModuleCommon *parent_module, + const char *sub_module_name, char *error_buf, + uint32 error_buf_size) +{ + WASMModuleCommon *sub_module = NULL; + bool ret = false; + uint8 *buffer = NULL; + uint32 buffer_size = 0; + LoadArgs args = { 0 }; + + /* check the registered module list of the parent */ + sub_module = wasm_runtime_search_sub_module(parent_module, sub_module_name); + if (sub_module) { + LOG_DEBUG("%s has been loaded before", sub_module_name); + return sub_module; + } + + /* check the global registered module list */ + sub_module = wasm_runtime_find_module_registered(sub_module_name); + if (sub_module) { + LOG_DEBUG("%s has been loaded", sub_module_name); + goto wasm_runtime_register_sub_module; + } + LOG_VERBOSE("loading %s", sub_module_name); + if (!reader) { + set_error_buf_v(parent_module, error_buf, error_buf_size, + "no sub module reader to load %s", sub_module_name); + return NULL; + } + /* start to maintain a loading module list */ + ret = wasm_runtime_is_loading_module(sub_module_name); + if (ret) { + set_error_buf_v(parent_module, error_buf, error_buf_size, + "found circular dependency on %s", sub_module_name); + return NULL; + } + ret = wasm_runtime_add_loading_module(sub_module_name, error_buf, + error_buf_size); + if (!ret) { + LOG_DEBUG("can not add %s into loading module list\n", sub_module_name); + return NULL; + } + + ret = reader(parent_module->module_type, sub_module_name, &buffer, + &buffer_size); + if (!ret) { + LOG_DEBUG("read the file of %s failed", sub_module_name); + set_error_buf_v(parent_module, error_buf, error_buf_size, + "unknown import %s", sub_module_name); + goto delete_loading_module; + } + if (get_package_type(buffer, buffer_size) != parent_module->module_type) { + LOG_DEBUG("module %s type error", sub_module_name); + goto destroy_file_buffer; + } + + args.name = (char *)sub_module_name; + if (get_package_type(buffer, buffer_size) == Wasm_Module_Bytecode) { +#if WASM_ENABLE_INTERP != 0 + sub_module = (WASMModuleCommon *)wasm_load( + buffer, buffer_size, false, &args, error_buf, error_buf_size); +#endif + } + else if (get_package_type(buffer, buffer_size) == Wasm_Module_AoT) { +#if WASM_ENABLE_AOT != 0 + sub_module = (WASMModuleCommon *)aot_load_from_aot_file( + buffer, buffer_size, &args, error_buf, error_buf_size); +#endif + } + if (!sub_module) { + LOG_DEBUG("error: can not load the sub_module %s", sub_module_name); + /* others will be destroyed in runtime_destroy() */ + goto destroy_file_buffer; + } + wasm_runtime_delete_loading_module(sub_module_name); + /* register on a global list */ + ret = wasm_runtime_register_module_internal( + sub_module_name, (WASMModuleCommon *)sub_module, buffer, buffer_size, + error_buf, error_buf_size); + if (!ret) { + LOG_DEBUG("error: can not register module %s globally\n", + sub_module_name); + /* others will be unloaded in runtime_destroy() */ + goto unload_module; + } + + /* register into its parent list */ +wasm_runtime_register_sub_module: + ret = wasm_runtime_register_sub_module(parent_module, sub_module_name, + sub_module); + if (!ret) { + set_error_buf_v(parent_module, error_buf, error_buf_size, + "failed to register sub module %s", sub_module_name); + /* since it is in the global module list, no need to + * unload the module. the runtime_destroy() will do it + */ + return NULL; + } + + return sub_module; + +unload_module: + wasm_runtime_unload(sub_module); + +destroy_file_buffer: + if (destroyer) { + destroyer(buffer, buffer_size); + } + else { + LOG_WARNING("need to release the reading buffer of %s manually", + sub_module_name); + } + +delete_loading_module: + wasm_runtime_delete_loading_module(sub_module_name); + return NULL; +} + +bool +wasm_runtime_sub_module_instantiate(WASMModuleCommon *module, + WASMModuleInstanceCommon *module_inst, + uint32 stack_size, uint32 heap_size, + uint32 max_memory_pages, char *error_buf, + uint32 error_buf_size) +{ + bh_list *sub_module_inst_list = NULL; + WASMRegisteredModule *sub_module_list_node = NULL; + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + sub_module_inst_list = + ((AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e) + ->sub_module_inst_list; + sub_module_list_node = + bh_list_first_elem(((AOTModule *)module)->import_module_list); + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + sub_module_inst_list = + ((WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e) + ->sub_module_inst_list; + sub_module_list_node = + bh_list_first_elem(((WASMModule *)module)->import_module_list); + } +#endif + while (sub_module_list_node) { + WASMSubModInstNode *sub_module_inst_list_node = NULL; + WASMModuleCommon *sub_module = sub_module_list_node->module; + WASMModuleInstanceCommon *sub_module_inst = NULL; + sub_module_inst = wasm_runtime_instantiate_internal( + sub_module, NULL, NULL, stack_size, heap_size, max_memory_pages, + error_buf, error_buf_size); + if (!sub_module_inst) { + LOG_DEBUG("instantiate %s failed", + sub_module_list_node->module_name); + return false; + } + sub_module_inst_list_node = loader_malloc(sizeof(WASMSubModInstNode), + error_buf, error_buf_size); + if (!sub_module_inst_list_node) { + LOG_DEBUG("Malloc WASMSubModInstNode failed, SZ: %zu", + sizeof(WASMSubModInstNode)); + if (sub_module_inst) + wasm_runtime_deinstantiate_internal(sub_module_inst, false); + return false; + } + sub_module_inst_list_node->module_inst = + (WASMModuleInstance *)sub_module_inst; + sub_module_inst_list_node->module_name = + sub_module_list_node->module_name; + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstance *aot_module_inst = + (AOTModuleInstance *)module_inst; + AOTModule *aot_module = (AOTModule *)module; + AOTModuleInstanceExtra *aot_extra = + (AOTModuleInstanceExtra *)aot_module_inst->e; + uint32 i; + AOTImportFunc *import_func; + for (i = 0; i < aot_module->import_func_count; i++) { + if (aot_extra->import_func_module_insts[i]) + continue; + + import_func = &aot_module->import_funcs[i]; + if (strcmp(sub_module_inst_list_node->module_name, + import_func->module_name) + == 0) { + aot_extra->import_func_module_insts[i] = + (WASMModuleInstanceCommon *) + sub_module_inst_list_node->module_inst; + } + } + } +#endif + + bh_list_status ret = + bh_list_insert(sub_module_inst_list, sub_module_inst_list_node); + bh_assert(BH_LIST_SUCCESS == ret); + (void)ret; + sub_module_list_node = bh_list_elem_next(sub_module_list_node); + } + + return true; +} + +void +wasm_runtime_sub_module_deinstantiate(WASMModuleInstanceCommon *module_inst) +{ + bh_list *list = NULL; +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + list = ((AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e) + ->sub_module_inst_list; + } +#endif +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + list = + ((WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e) + ->sub_module_inst_list; + } +#endif + + WASMSubModInstNode *node = bh_list_first_elem(list); + while (node) { + WASMSubModInstNode *next_node = bh_list_elem_next(node); + bh_list_remove(list, node); + wasm_runtime_deinstantiate_internal( + (WASMModuleInstanceCommon *)node->module_inst, false); + wasm_runtime_free(node); + node = next_node; + } +} +#endif /* end of WASM_ENABLE_MULTI_MODULE */ +#if WASM_ENABLE_MODULE_INST_CONTEXT != 0 +void * +wasm_runtime_create_context_key(void (*dtor)(WASMModuleInstanceCommon *inst, + void *ctx)) +{ + return wasm_native_create_context_key(dtor); +} + +void +wasm_runtime_destroy_context_key(void *key) +{ + wasm_native_destroy_context_key(key); +} + +void +wasm_runtime_set_context(WASMModuleInstanceCommon *inst, void *key, void *ctx) +{ + wasm_native_set_context(inst, key, ctx); +} + +void +wasm_runtime_set_context_spread(WASMModuleInstanceCommon *inst, void *key, + void *ctx) +{ + wasm_native_set_context_spread(inst, key, ctx); +} + +void * +wasm_runtime_get_context(WASMModuleInstanceCommon *inst, void *key) +{ + return wasm_native_get_context(inst, key); +} +#endif /* WASM_ENABLE_MODULE_INST_CONTEXT != 0 */ + +#if WASM_ENABLE_LINUX_PERF != 0 +static bool enable_linux_perf = false; + +bool +wasm_runtime_get_linux_perf(void) +{ + return enable_linux_perf; +} + +void +wasm_runtime_set_linux_perf(bool flag) +{ + enable_linux_perf = flag; +} +#endif + +bool +wasm_runtime_set_module_name(wasm_module_t module, const char *name, + char *error_buf, uint32_t error_buf_size) +{ + if (!module) + return false; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) + return wasm_set_module_name((WASMModule *)module, name, error_buf, + error_buf_size); +#endif + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) + return aot_set_module_name((AOTModule *)module, name, error_buf, + error_buf_size); +#endif + + return false; +} + +const char * +wasm_runtime_get_module_name(wasm_module_t module) +{ + if (!module) + return ""; + +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) + return wasm_get_module_name((WASMModule *)module); +#endif + +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) + return aot_get_module_name((AOTModule *)module); +#endif + + return ""; +} + +/* + * wasm_runtime_detect_native_stack_overflow + * + * - raise "native stack overflow" exception if available native stack + * at this point is less than WASM_STACK_GUARD_SIZE. in that case, + * return false. + * + * - update native_stack_top_min. + */ +bool +wasm_runtime_detect_native_stack_overflow(WASMExecEnv *exec_env) +{ + uint8 *boundary = exec_env->native_stack_boundary; + RECORD_STACK_USAGE(exec_env, (uint8 *)&boundary); + if (boundary == NULL) { + /* the platform doesn't support os_thread_get_stack_boundary */ + return true; + } +#if defined(OS_ENABLE_HW_BOUND_CHECK) && WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + uint32 page_size = os_getpagesize(); + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; + boundary = boundary + page_size * guard_page_count; +#endif + if ((uint8 *)&boundary < boundary) { + wasm_runtime_set_exception(wasm_runtime_get_module_inst(exec_env), + "native stack overflow"); + return false; + } + return true; +} + +bool +wasm_runtime_detect_native_stack_overflow_size(WASMExecEnv *exec_env, + uint32 requested_size) +{ + uint8 *boundary = exec_env->native_stack_boundary; + RECORD_STACK_USAGE(exec_env, (uint8 *)&boundary); + if (boundary == NULL) { + /* the platform doesn't support os_thread_get_stack_boundary */ + return true; + } +#if defined(OS_ENABLE_HW_BOUND_CHECK) && WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + uint32 page_size = os_getpagesize(); + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; + boundary = boundary + page_size * guard_page_count; +#endif + /* adjust the boundary for the requested size */ + boundary = boundary - WASM_STACK_GUARD_SIZE + requested_size; + if ((uint8 *)&boundary < boundary) { + wasm_runtime_set_exception(wasm_runtime_get_module_inst(exec_env), + "native stack overflow"); + return false; + } + return true; +} + +bool +wasm_runtime_is_underlying_binary_freeable(WASMModuleCommon *const module) +{ +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { +#if (WASM_ENABLE_JIT != 0 || WASM_ENABLE_FAST_JIT != 0) \ + && (WASM_ENABLE_LAZY_JIT != 0) + return false; +#elif WASM_ENABLE_FAST_INTERP == 0 + return false; +#else + /* Fast interpreter mode */ + if (!((WASMModule *)module)->is_binary_freeable) + return false; +#if WASM_ENABLE_GC != 0 && WASM_ENABLE_STRINGREF != 0 + if (((WASMModule *)module)->string_literal_ptrs) + return false; +#endif +#endif + } +#endif /* WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + if (!((AOTModule *)module)->is_binary_freeable) + return false; +#if WASM_ENABLE_GC != 0 && WASM_ENABLE_STRINGREF != 0 + if (((AOTModule *)module)->string_literal_ptrs) + return false; +#endif + } +#endif /* WASM_ENABLE_AOT != 0 */ + + return true; +} + +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +wasm_runtime_check_and_update_last_used_shared_heap( + WASMModuleInstanceCommon *module_inst, uintptr_t app_offset, size_t bytes, + uintptr_t *shared_heap_start_off_p, uintptr_t *shared_heap_end_off_p, + uint8 **shared_heap_base_addr_adj_p, bool is_memory64) +{ + WASMSharedHeap *heap = wasm_runtime_get_shared_heap(module_inst), *cur; + uint64 shared_heap_start, shared_heap_end; + + if (bytes == 0) { + bytes = 1; + } + + /* Find the exact shared heap that app addr is in, and update last used + * shared heap info in func context */ + for (cur = heap; cur; cur = cur->chain_next) { + shared_heap_start = + is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; + shared_heap_end = shared_heap_start - 1 + cur->size; + if (bytes - 1 <= shared_heap_end && app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1) { + *shared_heap_start_off_p = (uintptr_t)shared_heap_start; + *shared_heap_end_off_p = (uintptr_t)shared_heap_end; + *shared_heap_base_addr_adj_p = + cur->base_addr - (uintptr_t)shared_heap_start; + return true; + } + } + + return false; +} +#endif diff --git a/wamr/stub.cpp b/wamr/stub.cpp new file mode 100644 index 0000000..3f67c4f --- /dev/null +++ b/wamr/stub.cpp @@ -0,0 +1,116 @@ +#include + + +#include "bh_log.h" + +extern "C" { + +static uint32 log_verbose_level = BH_LOG_LEVEL_FATAL; + +void +bh_log_set_verbose_level(uint32 level) +{ + log_verbose_level = level; +} + +void +bh_log(LogLevel log_level, const char *file, int line, const char *fmt, ...) +{ + va_list ap; + + if ((uint32)log_level > log_verbose_level) + return; + + if (file) + os_printf("%s, line %d, ", file, line); + + va_start(ap, fmt); + os_vprintf(fmt, ap); + va_end(ap); + + os_printf("\n"); +} + +void * os_malloc(unsigned size) +{ + return std::malloc(size); +} + +void * os_realloc(void *ptr, unsigned size) +{ + return std::realloc(ptr, size); +} + +void os_free(void *ptr) +{ + std::free(ptr); +} + +int os_dumps_proc_mem_info(char *out, unsigned int size) +{ + return -1; +} + +void * os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file) +{ + void *addr; + + if (size >= UINT32_MAX) + return NULL; + + if ((addr = BH_MALLOC((uint32)size))) + memset(addr, 0, (uint32)size); + + return addr; +} + +void * os_mremap(void *old_addr, size_t old_size, size_t new_size) +{ + return os_mremap_slow(old_addr, old_size, new_size); +} + +void os_munmap(void *addr, size_t size) +{ + return BH_FREE(addr); +} + +int os_mprotect(void *addr, size_t size, int prot) +{ + return 0; +} + + +int bh_platform_init() +{ + return 0; +} + +void bh_platform_destroy() +{} + +int os_printf(const char *format, ...) +{ + int ret = 0; + va_list ap; + + va_start(ap, format); +#ifndef BH_VPRINTF + ret += vprintf(format, ap); +#else + ret += BH_VPRINTF(format, ap); +#endif + va_end(ap); + + return ret; +} + +int os_vprintf(const char *format, va_list ap) +{ +#ifndef BH_VPRINTF + return vprintf(format, ap); +#else + return BH_VPRINTF(format, ap); +#endif +} + +} \ No newline at end of file diff --git a/wamr/update.py b/wamr/update.py new file mode 100644 index 0000000..a00d45a --- /dev/null +++ b/wamr/update.py @@ -0,0 +1,248 @@ +import argparse +from pathlib import Path +import re +import shutil + +opts = argparse.ArgumentParser("update") +opts.add_argument("-s", "--src") + +args = opts.parse_args() + +files = [ + "core/iwasm/interpreter/wasm_loader.c", + "core/iwasm/interpreter/wasm_runtime.c", + "core/iwasm/interpreter/wasm_interp_fast.c", + + "core/shared/utils/bh_assert.c", + #"core/shared/utils/bh_log.c", + "core/shared/utils/bh_vector.c", + #"core/shared/utils/bh_bitmap.c", + #"core/shared/utils/bh_queue.c", + #"core/shared/utils/bh_list.c", + "core/shared/utils/bh_common.c", + #"core/shared/utils/bh_hashmap.c", + "core/shared/utils/bh_leb128.c", + #"core/shared/utils/runtime_timer.c", + + #"core/shared/platform/cosmopolitan/platform_init.c", + "core/shared/platform/common/posix/posix_thread.c", + #"core/shared/platform/common/posix/posix_malloc.c", + #"core/shared/platform/common/posix/posix_memmap.c", + #"core/shared/platform/common/posix/posix_blocking_op.c", + #"core/shared/platform/common/memory/mremap.c", + #"core/shared/platform/common/posix/posix_time.c", + + #"core/iwasm/common/arch/invokeNative_general.c", + + "core/iwasm/common/wasm_memory.c", + "core/iwasm/common/wasm_runtime_common.c", + "core/iwasm/common/wasm_exec_env.c", + "core/iwasm/common/wasm_loader_common.c", + + #"core/iwasm/common/wasm_shared_memory.c", + #"core/iwasm/common/wasm_native.c", + + "core/shared/mem-alloc/mem_alloc.c", + "core/shared/mem-alloc/ems/ems_alloc.c", + "core/shared/mem-alloc/ems/ems_kfc.c", + + "core/iwasm/common/wasm_c_api.c", + #"samples/wasm-c-api/src/hello.c", +] + + + +src = Path(args.src) +dest = Path(__file__).parent.resolve() + +search = list(map(lambda o: src / o, [ + "core/shared/utils", + "core/shared/platform/posix", + "core/shared/platform/include", + "core/iwasm/common", + "core/shared/mem-alloc", + "core/iwasm/include", + "core/iwasm/interpreter", + "core/iwasm/libraries", + "core/shared/platform/cosmopolitan", + "core/shared/mem-alloc/ems", + "core", +])) + +config_text = """ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/* Replaced by godot-wasm */ + +#define WASM_ENABLE_INTERP 1 +#define WASM_ENABLE_FAST_INTERP 1 +#define WASM_ENABLE_SHARED_HEAP 0 +#define WASM_ENABLE_SHARED_MEMORY 0 +#define WASM_ENABLE_LIBC_WASI 0 +#define WASM_ENABLE_GC 0 +#define WASM_ENABLE_WASM_CACHE 0 +#define WASM_ENABLE_INVOKE_NATIVE 0 +#define WASM_ENABLE_JIT 0 +#define WASM_ENABLE_FAST_JIT 0 +#define WASM_ENABLE_SIMDE 0 + +#define BH_MALLOC wasm_runtime_malloc +#define BH_FREE wasm_runtime_free + +#ifndef WASM_TABLE_MAX_SIZE +#define WASM_TABLE_MAX_SIZE 1024 +#endif + +/* Default wasm block address cache size and conflict list size */ +#ifndef BLOCK_ADDR_CACHE_SIZE +#define BLOCK_ADDR_CACHE_SIZE 64 +#endif +#define BLOCK_ADDR_CONFLICT_SIZE 2 + +#ifndef DEFAULT_QUEUE_LENGTH +#define DEFAULT_QUEUE_LENGTH 50 +#endif + +/* Default min/max heap size of each app */ +#ifndef APP_HEAP_SIZE_DEFAULT +#define APP_HEAP_SIZE_DEFAULT (8 * 1024) +#endif +#define APP_HEAP_SIZE_MIN (256) + +/* The ems memory allocator supports maximal heap size 1GB, + see ems_gc_internal.h */ +#define APP_HEAP_SIZE_MAX (1024 * 1024 * 1024) + +/* Default/min native stack size of each app thread */ +#if defined(PTHREAD_STACK_DEFAULT) && defined(PTHREAD_STACK_MIN) +#define APP_THREAD_STACK_SIZE_DEFAULT PTHREAD_STACK_DEFAULT +#define APP_THREAD_STACK_SIZE_MIN PTHREAD_STACK_MIN +#else +#define APP_THREAD_STACK_SIZE_DEFAULT (128 * 1024) +#define APP_THREAD_STACK_SIZE_MIN (24 * 1024) +#endif + +/* Max native stack size of each app thread */ +#if !defined(APP_THREAD_STACK_SIZE_MAX) +#define APP_THREAD_STACK_SIZE_MAX (8 * 1024 * 1024) +#endif + +#ifndef WASM_CONST_EXPR_STACK_SIZE +#if WASM_ENABLE_GC != 0 +#define WASM_CONST_EXPR_STACK_SIZE 8 +#else +#define WASM_CONST_EXPR_STACK_SIZE 4 +#endif +#endif + +#define WASM_STACK_GUARD_SIZE (4096 * 6) +#define DEFAULT_WASM_STACK_SIZE (16 * 1024) +#define AOT_CURRENT_VERSION 5 + + +#ifndef _CONFIG_H_ +#define _CONFIG_H_ + +#endif /* end of _CONFIG_H_ */ +""" + +r_include = re.compile("^#include ([\"<])([^\"<>]*)[>\"]", re.DOTALL | re.MULTILINE ) +r_guard = re.compile(r"^#if\s*([^\n]*)\s*\n#include[^\n]*\n+#endif" , re.DOTALL | re.MULTILINE) + +r_banned = re.compile(r"aot|uvwasi|jit|gc") + +if not dest.exists(): + raise "Destination path doesnt exist" + +if not src.exists(): + raise "Source path doesnt exist" + +for p in dest.glob("src/*.c"): + p.unlink() + +if not (dest / "src").exists(): + (dest / "src").mkdir() + +if not (dest / "include").exists(): + (dest / "include").mkdir() + +shutil.copy(src / "LICENSE", dest / "LICENSE") + +includes = set() + + +done = set() + +def matcher(m): + if m.group(1) == "<": + return m.group(0) + + s = m.group(2).split("/")[-1] + + if r_banned.match(s): + return '' + + includes.add(s) + return f"#include \"{s}\" // {m.group(2)}" + +def guarder(m): + if m.group(1) in [ + "WASM_ENABLE_GC != 0", + "WASM_ENABLE_AOT != 0", + "WASM_ENABLE_SHARED_MEMORY != 0", + "WASM_ENABLE_JIT != 0", + "WASM_ENABLE_DEBUG_AOT != 0", + "WASM_ENABLE_THREAD_MGR != 0", + "WASM_ENABLE_WASM_CACHE != 0" + ]: + return '' + + return m.group(0) + +def process(text): + text = r_guard.sub(guarder, text) + text = r_include.sub(matcher, text) + return text + + +for file in files: + p = src / Path(file) + s = "" + with p.open("r") as f: + s = f.read() + + s = process(s) + + with (dest / "src" / p.name).open("w") as f: + f.write(s) + +while True: + todo = includes - done + if len(todo) < 1: + break + i = todo.pop() + done.add(i) + + found = None + for s in search: + o = s / i + if o.exists(): + found = o + break + + if not found: + print("Cant find", i) + else: + with found.open("r") as f: + text = f.read() + text = process(text) + + if found.name == "config.h": + text = config_text + + with (dest / "include" / found.name).open("w") as f2: + f2.write(text) +