From 7a8070e6f711db5fff0905ca1379527f0e8afc56 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 18 Jun 2024 18:02:50 -0700 Subject: [PATCH] Load Pyodide implementation from external capnproto file This is quite work in progress. The idea is we can upload these binary capnproto files to GCS and be able to select at runtime the capnproto module for the version of Pyodide corresponding to the worker's compatibility date. Currently I am just hardcoding the path. For some reason with `The requested module 'pyodide:python-entrypoint-helper' does not provide an export named 'default'`. Adding logging info shows that the text and type of the module is right so I'm not sure what is going wrong. --- build/wd_js_bundle.bzl | 9 +++++++-- build/wd_ts_bundle.bzl | 21 ++++++++++++++++++--- src/pyodide/BUILD.bazel | 23 ++++++++++++++++++++--- src/workerd/api/BUILD.bazel | 2 -- src/workerd/api/pyodide/pyodide.h | 12 ++++++++++-- src/workerd/jsg/BUILD.bazel | 2 ++ src/workerd/server/BUILD.bazel | 9 +++++++++ 7 files changed, 66 insertions(+), 12 deletions(-) diff --git a/build/wd_js_bundle.bzl b/build/wd_js_bundle.bzl index e76855301f6..30f369a5073 100644 --- a/build/wd_js_bundle.bzl +++ b/build/wd_js_bundle.bzl @@ -114,7 +114,7 @@ def _copy_modules(modules, declarations): result[new_filename] = modules[m] return result, declarations_result -def wd_js_bundle( +def wd_js_bundle_capnp( name, import_name, schema_id, @@ -146,6 +146,7 @@ def wd_js_bundle( internal_json_modules: list of json source files declarations: d.ts label set deps: dependency list + Returns: The set of data dependencies """ builtin_modules_dict = { m: "{}:{}".format(import_name, _to_name(m)) @@ -201,7 +202,7 @@ def wd_js_bundle( gen_api_bundle_capnpn( name = name + "@gen", - out = name + ".capnp", + out = name, schema_id = schema_id, const_name = import_name + "Bundle", builtin_modules = builtin_modules_dict, @@ -213,7 +214,11 @@ def wd_js_bundle( data = data, deps = deps, ) + return data + +def wd_js_bundle(name, import_name, *args, **kwargs): + data = wd_js_bundle_capnp(name + ".capnp", import_name, *args, **kwargs) cc_capnp_library( name = name, srcs = [name + ".capnp"], diff --git a/build/wd_ts_bundle.bzl b/build/wd_ts_bundle.bzl index 1c25e88a751..aaed27a3e79 100644 --- a/build/wd_ts_bundle.bzl +++ b/build/wd_ts_bundle.bzl @@ -1,6 +1,7 @@ load("@aspect_rules_ts//ts:defs.bzl", "ts_config", "ts_project") load("@npm//:eslint/package_json.bzl", eslint_bin = "bin") -load("@workerd//:build/wd_js_bundle.bzl", "wd_js_bundle") +load("@workerd//:build/wd_js_bundle.bzl", "wd_js_bundle_capnp") +load("@capnp-cpp//src/capnp:cc_capnp_library.bzl", "cc_capnp_library") def _to_js(file_name): if file_name.endswith(".ts"): @@ -10,7 +11,7 @@ def _to_js(file_name): def _to_d_ts(file_name): return file_name.removesuffix(".ts") + ".d.ts" -def wd_ts_bundle( +def wd_ts_bundle_capnp( name, import_name, schema_id, @@ -59,7 +60,7 @@ def wd_ts_bundle( deps = deps, ) - wd_js_bundle( + data = wd_js_bundle_capnp( name = name, import_name = import_name, # builtin modules are accessible under ":" name @@ -96,3 +97,17 @@ def wd_ts_bundle( "//conditions:default": [], }), ) + return data + + +def wd_ts_bundle(name, import_name, *args, **kwargs): + data = wd_ts_bundle_capnp(name + ".capnp", import_name, *args, **kwargs) + cc_capnp_library( + name = name, + srcs = [name + ".capnp"], + strip_include_prefix = "", + visibility = ["//visibility:public"], + data = data, + deps = ["@workerd//src/workerd/jsg:modules_capnp"], + include_prefix = import_name, + ) diff --git a/src/pyodide/BUILD.bazel b/src/pyodide/BUILD.bazel index 738fdbeace7..d020d5c3d80 100644 --- a/src/pyodide/BUILD.bazel +++ b/src/pyodide/BUILD.bazel @@ -3,7 +3,7 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@bazel_skylib//rules:expand_template.bzl", "expand_template") load("@capnp-cpp//src/capnp:cc_capnp_library.bzl", "cc_capnp_library") load("//:build/capnp_embed.bzl", "capnp_embed") -load("//:build/wd_ts_bundle.bzl", "wd_ts_bundle") +load("//:build/wd_ts_bundle.bzl", "wd_ts_bundle_capnp") copy_file( name = "pyodide_packages_archive", @@ -146,8 +146,8 @@ expand_template( template = "@pyodide//:pyodide/pyodide.asm.js", ) -wd_ts_bundle( - name = "pyodide", +data = wd_ts_bundle_capnp( + name = "pyodide.capnp", modules = ["python-entrypoint-helper.ts"], import_name = "pyodide", internal_data_modules = ["generated/python_stdlib.zip"] + glob([ @@ -179,3 +179,20 @@ wd_ts_bundle( "pyodide-bucket.json@rule", ], ) + +genrule( + name = "pyodide.capnp.bin@rule", + tools = ["@capnp-cpp//src/capnp:capnp_tool"], + srcs = ["pyodide.capnp", "//src/workerd/jsg:modules.capnp"] + data, + outs = ["pyodide.capnp.bin"], + visibility = ["//visibility:public"], + cmd = " ".join([ + "$(execpath @capnp-cpp//src/capnp:capnp_tool)", + "eval", + "$(location :pyodide.capnp)", + "pyodideBundle", + "-I external/workerd/src/", + "-o binary", + "> $@", + ]) +) diff --git a/src/workerd/api/BUILD.bazel b/src/workerd/api/BUILD.bazel index 7ffe65251c8..095f457ab68 100644 --- a/src/workerd/api/BUILD.bazel +++ b/src/workerd/api/BUILD.bazel @@ -69,7 +69,6 @@ wd_cc_library( visibility = ["//visibility:public"], deps = [ ":html-rewriter", - "//src/pyodide", "//src/pyodide:pyodide_extra_capnp", "//src/workerd/io", "//src/workerd/jsg:rtti", @@ -103,7 +102,6 @@ wd_cc_library( ], visibility = ["//visibility:public"], deps = [ - "//src/pyodide", "//src/pyodide:pyodide_extra_capnp", "//src/workerd/io", "//src/workerd/jsg", diff --git a/src/workerd/api/pyodide/pyodide.h b/src/workerd/api/pyodide/pyodide.h index 6e7d9d9f3f8..826bcc2b904 100644 --- a/src/workerd/api/pyodide/pyodide.h +++ b/src/workerd/api/pyodide/pyodide.h @@ -4,8 +4,8 @@ #include "kj/debug.h" #include #include +#include #include -#include #include #include #include @@ -343,10 +343,18 @@ bool hasPythonModules(capnp::List::Reader module api::pyodide::DisabledInternalJaeger,\ api::pyodide::SimplePythonLimiter +#include + +static kj::Own reader; + template void registerPyodideModules(Registry& registry, auto featureFlags) { if (featureFlags.getPythonWorkers()) { // We add `pyodide:` packages here including python-entrypoint-helper.js. - registry.addBuiltinBundle(PYODIDE_BUNDLE, kj::none); + int fd = open("/home/rchatham/edgeworker/deps/workerd/bazel-bin/src/workerd/server/pyodide.capnp.bin", O_RDONLY); + constexpr capnp::ReaderOptions CONFIG_READER_OPTIONS = {}; + reader = kj::heap(fd, CONFIG_READER_OPTIONS); + auto bundle = reader->getRoot(); + registry.addBuiltinBundle(bundle, kj::none); registry.template addBuiltinModule( "pyodide-internal:packages_tar_reader", workerd::jsg::ModuleRegistry::Type::INTERNAL); } diff --git a/src/workerd/jsg/BUILD.bazel b/src/workerd/jsg/BUILD.bazel index 97a90cb5b45..3f94ea990b8 100644 --- a/src/workerd/jsg/BUILD.bazel +++ b/src/workerd/jsg/BUILD.bazel @@ -4,6 +4,8 @@ load("//:build/kj_test.bzl", "kj_test") load("//:build/wd_cc_capnp_library.bzl", "wd_cc_capnp_library") load("//:build/wd_cc_library.bzl", "wd_cc_library") +exports_files(["modules.capnp"]) + wd_cc_library( name = "jsg", srcs = [ diff --git a/src/workerd/server/BUILD.bazel b/src/workerd/server/BUILD.bazel index dcf64c0f571..f949492f7f2 100644 --- a/src/workerd/server/BUILD.bazel +++ b/src/workerd/server/BUILD.bazel @@ -1,5 +1,6 @@ load("@bazel_skylib//lib:selects.bzl", "selects") load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("//:build/kj_test.bzl", "kj_test") load("//:build/wd_cc_binary.bzl", "wd_cc_binary") load("//:build/wd_cc_capnp_library.bzl", "wd_cc_capnp_library") @@ -146,3 +147,11 @@ wd_cc_capnp_library( "//src/workerd/util:test-util", ], ) for f in glob(["*-test.c++"])] + + +copy_file( + name = "pyodide.capnp.bin@rule", + src = "//src/pyodide:pyodide.capnp.bin@rule", + out = "pyodide.capnp.bin", +) +