Load Pyodide implementation from external capnproto file

This is quite work in progress. The idea is we can upload these binary capnproto
files to GCS and be able to select at runtime the capnproto module for the
version of Pyodide corresponding to the worker's compatibility date.

Currently I am just hardcoding the path. For some reason with `The requested
module 'pyodide:python-entrypoint-helper' does not provide an export named
'default'`. Adding logging info shows that the text and type of the module is
right so I'm not sure what is going wrong.

build/wd_js_bundle.bzl

@@ -114,7 +114,7 @@ def _copy_modules(modules, declarations):
         result[new_filename] = modules[m]
     return result, declarations_result
 
-def wd_js_bundle(
+def wd_js_bundle_capnp(
         name,
         import_name,
         schema_id,
@@ -146,6 +146,7 @@ def wd_js_bundle(
      internal_json_modules: list of json source files
      declarations: d.ts label set
      deps: dependency list
+    Returns: The set of data dependencies
     """
     builtin_modules_dict = {
         m: "{}:{}".format(import_name, _to_name(m))
@@ -201,7 +202,7 @@ def wd_js_bundle(
 
     gen_api_bundle_capnpn(
         name = name + "@gen",
-        out = name + ".capnp",
+        out = name,
         schema_id = schema_id,
         const_name = import_name + "Bundle",
         builtin_modules = builtin_modules_dict,
@@ -213,7 +214,11 @@ def wd_js_bundle(
         data = data,
         deps = deps,
     )
+    return data
 
+
+def wd_js_bundle(name, import_name, *args, **kwargs):
+    data = wd_js_bundle_capnp(name + ".capnp", import_name, *args, **kwargs)
     cc_capnp_library(
         name = name,
         srcs = [name + ".capnp"],

build/wd_ts_bundle.bzl

@@ -1,6 +1,7 @@
 load("@aspect_rules_ts//ts:defs.bzl", "ts_config", "ts_project")
 load("@npm//:eslint/package_json.bzl", eslint_bin = "bin")
-load("@workerd//:build/wd_js_bundle.bzl", "wd_js_bundle")
+load("@workerd//:build/wd_js_bundle.bzl", "wd_js_bundle_capnp")
+load("@capnp-cpp//src/capnp:cc_capnp_library.bzl", "cc_capnp_library")
 
 def _to_js(file_name):
     if file_name.endswith(".ts"):
@@ -10,7 +11,7 @@ def _to_js(file_name):
 def _to_d_ts(file_name):
     return file_name.removesuffix(".ts") + ".d.ts"
 
-def wd_ts_bundle(
+def wd_ts_bundle_capnp(
         name,
         import_name,
         schema_id,
@@ -59,7 +60,7 @@ def wd_ts_bundle(
         deps = deps,
     )
 
-    wd_js_bundle(
+    data = wd_js_bundle_capnp(
         name = name,
         import_name = import_name,
         # builtin modules are accessible under "<import_name>:<module_name>" name
@@ -96,3 +97,17 @@ def wd_ts_bundle(
                 "//conditions:default": [],
             }),
         )
+    return data
+
+
+def wd_ts_bundle(name, import_name, *args, **kwargs):
+    data = wd_ts_bundle_capnp(name + ".capnp", import_name, *args, **kwargs)
+    cc_capnp_library(
+        name = name,
+        srcs = [name + ".capnp"],
+        strip_include_prefix = "",
+        visibility = ["//visibility:public"],
+        data = data,
+        deps = ["@workerd//src/workerd/jsg:modules_capnp"],
+        include_prefix = import_name,
+    )

src/pyodide/BUILD.bazel

@@ -3,7 +3,7 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file")
 load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
 load("@capnp-cpp//src/capnp:cc_capnp_library.bzl", "cc_capnp_library")
 load("//:build/capnp_embed.bzl", "capnp_embed")
-load("//:build/wd_ts_bundle.bzl", "wd_ts_bundle")
+load("//:build/wd_ts_bundle.bzl", "wd_ts_bundle_capnp")
 
 copy_file(
     name = "pyodide_packages_archive",
@@ -146,8 +146,8 @@ expand_template(
     template = "@pyodide//:pyodide/pyodide.asm.js",
 )
 
-wd_ts_bundle(
-    name = "pyodide",
+data = wd_ts_bundle_capnp(
+    name = "pyodide.capnp",
     modules = ["python-entrypoint-helper.ts"],
     import_name = "pyodide",
     internal_data_modules = ["generated/python_stdlib.zip"] + glob([
@@ -179,3 +179,20 @@ wd_ts_bundle(
         "pyodide-bucket.json@rule",
     ],
 )
+
+genrule(
+    name = "pyodide.capnp.bin@rule",
+    tools = ["@capnp-cpp//src/capnp:capnp_tool"],
+    srcs = ["pyodide.capnp", "//src/workerd/jsg:modules.capnp"] + data,
+    outs = ["pyodide.capnp.bin"],
+    visibility = ["//visibility:public"],
+    cmd = " ".join([
+        "$(execpath @capnp-cpp//src/capnp:capnp_tool)",
+        "eval",
+        "$(location :pyodide.capnp)",
+        "pyodideBundle",
+        "-I external/workerd/src/",
+        "-o binary",
+        "> $@",
+    ])
+)

src/workerd/api/BUILD.bazel

@@ -69,7 +69,6 @@ wd_cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":html-rewriter",
-        "//src/pyodide",
         "//src/pyodide:pyodide_extra_capnp",
         "//src/workerd/io",
         "//src/workerd/jsg:rtti",
@@ -103,7 +102,6 @@ wd_cc_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
-        "//src/pyodide",
         "//src/pyodide:pyodide_extra_capnp",
         "//src/workerd/io",
         "//src/workerd/jsg",

src/workerd/api/pyodide/pyodide.h

@@ -4,8 +4,8 @@
 #include "kj/debug.h"
 #include <kj/common.h>
 #include <kj/filesystem.h>
+#include <capnp/serialize.h>
 #include <pyodide/generated/pyodide_extra.capnp.h>
-#include <pyodide/pyodide.capnp.h>
 #include <workerd/jsg/jsg.h>
 #include <workerd/jsg/modules-new.h>
 #include <workerd/jsg/url.h>
@@ -343,10 +343,18 @@ bool hasPythonModules(capnp::List<server::config::Worker::Module>::Reader module
   api::pyodide::DisabledInternalJaeger,\
   api::pyodide::SimplePythonLimiter
 
+#include <fcntl.h>
+
+static kj::Own<capnp::StreamFdMessageReader> reader;
+
 template <class Registry> void registerPyodideModules(Registry& registry, auto featureFlags) {
   if (featureFlags.getPythonWorkers()) {
     // We add `pyodide:` packages here including python-entrypoint-helper.js.
-    registry.addBuiltinBundle(PYODIDE_BUNDLE, kj::none);
+    int fd = open("/home/rchatham/edgeworker/deps/workerd/bazel-bin/src/workerd/server/pyodide.capnp.bin", O_RDONLY);
+    constexpr capnp::ReaderOptions CONFIG_READER_OPTIONS = {};
+    reader = kj::heap<capnp::StreamFdMessageReader>(fd, CONFIG_READER_OPTIONS);
+    auto bundle = reader->getRoot<jsg::Bundle>();
+    registry.addBuiltinBundle(bundle, kj::none);
     registry.template addBuiltinModule<PackagesTarReader>(
         "pyodide-internal:packages_tar_reader", workerd::jsg::ModuleRegistry::Type::INTERNAL);
   }

src/workerd/jsg/BUILD.bazel

@@ -4,6 +4,8 @@ load("//:build/kj_test.bzl", "kj_test")
 load("//:build/wd_cc_capnp_library.bzl", "wd_cc_capnp_library")
 load("//:build/wd_cc_library.bzl", "wd_cc_library")
 
+exports_files(["modules.capnp"])
+
 wd_cc_library(
     name = "jsg",
     srcs = [

src/workerd/server/BUILD.bazel

@@ -1,5 +1,6 @@
 load("@bazel_skylib//lib:selects.bzl", "selects")
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
+load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 load("//:build/kj_test.bzl", "kj_test")
 load("//:build/wd_cc_binary.bzl", "wd_cc_binary")
 load("//:build/wd_cc_capnp_library.bzl", "wd_cc_capnp_library")
@@ -146,3 +147,11 @@ wd_cc_capnp_library(
         "//src/workerd/util:test-util",
     ],
 ) for f in glob(["*-test.c++"])]
+
+
+copy_file(
+    name = "pyodide.capnp.bin@rule",
+    src = "//src/pyodide:pyodide.capnp.bin@rule",
+    out = "pyodide.capnp.bin",
+)
+