From 618f15a1b796c6d55572e2f5b8770db03fecde20 Mon Sep 17 00:00:00 2001 From: Ian Denhardt Date: Sat, 22 Aug 2020 14:16:09 -0400 Subject: [PATCH 1/3] Remove the custom node-capnp import logic. This removes the custom logic to import node-capnp, and instead installs deps/node-capnp as an npm dependency of the shell directly. To make this work, we also need to get the schema into meteor's node_modules, so we create a package for those too. --- .../server/payments-api-server.js | 2 +- .../imports/sandstorm-db/scheduled-jobs-db.js | 2 +- .../sandstorm-ui-powerbox/powerbox-server.js | 2 +- shell/imports/server/backend.js | 2 +- shell/imports/server/backup.js | 2 +- shell/imports/server/capnp.js | 43 ------------------- shell/imports/server/core.js | 2 +- .../server/drivers/external-ui-view.js | 2 +- shell/imports/server/drivers/ip.js | 2 +- shell/imports/server/drivers/mail.js | 2 +- shell/imports/server/gateway-router.js | 2 +- shell/imports/server/hack-session.js | 2 +- shell/imports/server/header-whitelist.js | 2 +- shell/imports/server/identity.js | 2 +- shell/imports/server/persistent.js | 2 +- shell/imports/server/scheduled-job.js | 2 +- shell/imports/server/shell-cli.js | 2 +- shell/npm-shrinkwrap.json | 6 +++ shell/package.json | 2 + src/sandstorm/package.json | 11 +++++ 20 files changed, 35 insertions(+), 59 deletions(-) delete mode 100644 shell/imports/server/capnp.js create mode 100644 src/sandstorm/package.json diff --git a/shell/imports/blackrock-payments/server/payments-api-server.js b/shell/imports/blackrock-payments/server/payments-api-server.js index 981b27cf1a..5da01f80e0 100644 --- a/shell/imports/blackrock-payments/server/payments-api-server.js +++ b/shell/imports/blackrock-payments/server/payments-api-server.js @@ -20,7 +20,7 @@ import { check } from "meteor/check"; import { stripe } from "/imports/blackrock-payments/server/payments-server.js"; -import Capnp from "/imports/server/capnp.js"; +import Capnp from "capnp"; const PaymentsRpc = Capnp.importSystem("sandstorm/payments.capnp"); function wrapAsyncAsPromise(obj, func) { diff --git a/shell/imports/sandstorm-db/scheduled-jobs-db.js b/shell/imports/sandstorm-db/scheduled-jobs-db.js index 94e664338f..6658773010 100644 --- a/shell/imports/sandstorm-db/scheduled-jobs-db.js +++ b/shell/imports/sandstorm-db/scheduled-jobs-db.js @@ -16,7 +16,7 @@ import Crypto from "crypto"; import { Match, check } from "meteor/check"; -import Capnp from "/imports/server/capnp.js"; +import Capnp from "capnp"; import { SandstormDb } from "./db.js"; const MINIMUM_SCHEDULING_SLACK_NANO = Capnp.importSystem("sandstorm/grain.capnp").minimumSchedulingSlack; diff --git a/shell/imports/sandstorm-ui-powerbox/powerbox-server.js b/shell/imports/sandstorm-ui-powerbox/powerbox-server.js index 1b25128806..c5ae3e3410 100644 --- a/shell/imports/sandstorm-ui-powerbox/powerbox-server.js +++ b/shell/imports/sandstorm-ui-powerbox/powerbox-server.js @@ -19,7 +19,7 @@ import { check } from "meteor/check"; import { _ } from "meteor/underscore"; import { SandstormPermissions } from "/imports/sandstorm-permissions/permissions.js"; -import Capnp from "/imports/server/capnp.js"; +import Capnp from "capnp"; import { waitPromise } from '../server/async-helpers.ts'; diff --git a/shell/imports/server/backend.js b/shell/imports/server/backend.js index 313bb91557..d4e90ddc5e 100644 --- a/shell/imports/server/backend.js +++ b/shell/imports/server/backend.js @@ -16,7 +16,7 @@ import { Meteor } from "meteor/meteor"; import { inMeteor, waitPromise } from "/imports/server/async-helpers.ts"; -import Capnp from "/imports/server/capnp.js"; +import Capnp from "capnp"; import { globalDb } from "/imports/db-deprecated.js"; let storageUsageUnimplemented = false; diff --git a/shell/imports/server/backup.js b/shell/imports/server/backup.js index 93781d9926..7bee2a98c5 100644 --- a/shell/imports/server/backup.js +++ b/shell/imports/server/backup.js @@ -22,7 +22,7 @@ import { Router } from "meteor/iron:router"; import { inMeteor, waitPromise } from "/imports/server/async-helpers.ts"; -import Capnp from "/imports/server/capnp.js"; +import Capnp from "capnp"; import { SandstormDb } from "/imports/sandstorm-db/db.js"; import { globalDb } from "/imports/db-deprecated.js"; diff --git a/shell/imports/server/capnp.js b/shell/imports/server/capnp.js deleted file mode 100644 index 5bf1f8e57d..0000000000 --- a/shell/imports/server/capnp.js +++ /dev/null @@ -1,43 +0,0 @@ -// Sandstorm - Personal Cloud Sandbox -// Copyright (c) 2017 Sandstorm Development Group, Inc. and contributors -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Finds the capnp module by searching up the directory tree. We expect it to be located in a -// node_modules subdirectory of one of our parent directories. -// -// We used to use Npm.require() for this, but Meteor disabled the "search up the tree" behavior -// in: https://github.com/meteor/meteor/pull/9095 - -import fs from "fs"; - -const pathParts = process.cwd().split("/"); - -let Capnp; -for (;;) { - const path = pathParts.join("/"); - - if (fs.existsSync(`${path}/node_modules/capnp.node`)) { - Capnp = Npm.require(`${path}/node_modules/capnp.js`); - break; - } - - if (pathParts.length === 0) { - throw new Error(`Can't find capnp.node, starting from: ${process.cwd()}`); - } - - pathParts.pop(); -} - -export default Capnp; diff --git a/shell/imports/server/core.js b/shell/imports/server/core.js index 0aa1e9901e..2091dc3b99 100644 --- a/shell/imports/server/core.js +++ b/shell/imports/server/core.js @@ -29,7 +29,7 @@ import { globalDb } from "/imports/db-deprecated.js"; import { schedulePeriodic, scheduleOneShot } from "/imports/server/scheduled-job.js"; import { makeGatewayRouter } from "/imports/server/gateway-router.js"; import { makeShellCli } from "/imports/server/shell-cli.js"; -import Capnp from "/imports/server/capnp.js"; +import Capnp from "capnp"; const PersistentHandle = Capnp.importSystem("sandstorm/supervisor.capnp").PersistentHandle; const SandstormCore = Capnp.importSystem("sandstorm/supervisor.capnp").SandstormCore; diff --git a/shell/imports/server/drivers/external-ui-view.js b/shell/imports/server/drivers/external-ui-view.js index c53169aa26..5d9ece0e11 100644 --- a/shell/imports/server/drivers/external-ui-view.js +++ b/shell/imports/server/drivers/external-ui-view.js @@ -29,7 +29,7 @@ import { globalDb } from "/imports/db-deprecated.js"; import Url from "url"; import Http from "http"; import Https from "https"; -import Capnp from "/imports/server/capnp.js"; +import Capnp from "capnp"; const ApiSession = Capnp.importSystem("sandstorm/api-session.capnp").ApiSession; const PersistentApiSession = Capnp.importSystem("sandstorm/api-session-impl.capnp").PersistentApiSession; diff --git a/shell/imports/server/drivers/ip.js b/shell/imports/server/drivers/ip.js index 39c81af9c7..0dd7b13fd6 100644 --- a/shell/imports/server/drivers/ip.js +++ b/shell/imports/server/drivers/ip.js @@ -22,7 +22,7 @@ import { PersistentImpl } from "/imports/server/persistent.js"; import Net from "net"; import Tls from "tls"; import Dgram from "dgram"; -import Capnp from "/imports/server/capnp.js"; +import Capnp from "capnp"; const IpRpc = Capnp.importSystem("sandstorm/ip.capnp"); diff --git a/shell/imports/server/drivers/mail.js b/shell/imports/server/drivers/mail.js index c302f2b7a2..cc9d41b4ed 100644 --- a/shell/imports/server/drivers/mail.js +++ b/shell/imports/server/drivers/mail.js @@ -34,7 +34,7 @@ import { makeHackSessionContext } from "/imports/server/hack-session.js"; import Crypto from "crypto"; import Net from "net"; import Url from "url"; -import Capnp from "/imports/server/capnp.js"; +import Capnp from "capnp"; const EmailRpc = Capnp.importSystem("sandstorm/email.capnp"); const EmailImpl = Capnp.importSystem("sandstorm/email-impl.capnp"); diff --git a/shell/imports/server/gateway-router.js b/shell/imports/server/gateway-router.js index c4e839f09b..e1c9ae3e82 100644 --- a/shell/imports/server/gateway-router.js +++ b/shell/imports/server/gateway-router.js @@ -23,7 +23,7 @@ import { Match, check } from "meteor/check"; import { _ } from "meteor/underscore"; import { Random } from "meteor/random"; -import Capnp from "/imports/server/capnp.js"; +import Capnp from "capnp"; import { SandstormDb } from "/imports/sandstorm-db/db.js"; import { globalDb } from "/imports/db-deprecated.js"; import { SandstormPermissions } from "/imports/sandstorm-permissions/permissions.js"; diff --git a/shell/imports/server/hack-session.js b/shell/imports/server/hack-session.js index 4ed1a95aad..15a432d5fd 100644 --- a/shell/imports/server/hack-session.js +++ b/shell/imports/server/hack-session.js @@ -26,7 +26,7 @@ import { Random } from "meteor/random"; import { hashSturdyRef, checkRequirements, fetchApiToken } from "/imports/server/persistent.js"; import { inMeteor, waitPromise } from "/imports/server/async-helpers.ts"; import { ssrfSafeLookup } from "/imports/server/networking.js"; -import Capnp from "/imports/server/capnp.js"; +import Capnp from "capnp"; import { SandstormDb } from "/imports/sandstorm-db/db.js"; import { globalDb } from "/imports/db-deprecated.js"; diff --git a/shell/imports/server/header-whitelist.js b/shell/imports/server/header-whitelist.js index 2330a9ca08..e9977cd402 100644 --- a/shell/imports/server/header-whitelist.js +++ b/shell/imports/server/header-whitelist.js @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -import Capnp from "/imports/server/capnp.js"; +import Capnp from "capnp"; const WebSession = Capnp.importSystem("sandstorm/web-session.capnp").WebSession; class HeaderWhitelist { diff --git a/shell/imports/server/identity.js b/shell/imports/server/identity.js index fbae03c916..9464f10553 100644 --- a/shell/imports/server/identity.js +++ b/shell/imports/server/identity.js @@ -23,7 +23,7 @@ import { globalDb } from "/imports/db-deprecated.js"; import { SandstormPermissions } from "/imports/sandstorm-permissions/permissions.js"; import { PersistentImpl } from "/imports/server/persistent.js"; import { StaticAssetImpl, IdenticonStaticAssetImpl } from "/imports/server/static-asset.js"; -import Capnp from "/imports/server/capnp.js"; +import Capnp from "capnp"; const IdentityRpc = Capnp.importSystem("sandstorm/identity-impl.capnp"); const Identity = Capnp.importSystem("sandstorm/identity.capnp").Identity; diff --git a/shell/imports/server/persistent.js b/shell/imports/server/persistent.js index 98155c6cb5..8a5c5a39d1 100644 --- a/shell/imports/server/persistent.js +++ b/shell/imports/server/persistent.js @@ -21,7 +21,7 @@ import { Random } from "meteor/random"; import Crypto from "crypto"; import { inMeteor } from "/imports/server/async-helpers.ts"; -import Capnp from "/imports/server/capnp.js"; +import Capnp from "capnp"; import { SandstormPermissions } from "/imports/sandstorm-permissions/permissions.js"; const privateDb = Symbol("PersistentImpl.db"); diff --git a/shell/imports/server/scheduled-job.js b/shell/imports/server/scheduled-job.js index 6f2cb77dec..9b2005d761 100644 --- a/shell/imports/server/scheduled-job.js +++ b/shell/imports/server/scheduled-job.js @@ -18,7 +18,7 @@ import { Meteor } from "meteor/meteor"; import { waitPromise } from "/imports/server/async-helpers.ts"; import { fetchApiToken } from "/imports/server/persistent.js"; -import Capnp from "/imports/server/capnp.js"; +import Capnp from "capnp"; import { SandstormDb } from "/imports/sandstorm-db/db.js"; import { globalDb } from "/imports/db-deprecated.js"; diff --git a/shell/imports/server/shell-cli.js b/shell/imports/server/shell-cli.js index 06e40c57f0..025223d6a4 100644 --- a/shell/imports/server/shell-cli.js +++ b/shell/imports/server/shell-cli.js @@ -19,7 +19,7 @@ import { createAcmeAccount, renewCertificateNow } from "/imports/server/acme.js" import { SandstormDb } from "/imports/sandstorm-db/db.js"; import { globalDb } from "/imports/db-deprecated.js"; -import Capnp from "/imports/server/capnp.js"; +import Capnp from "capnp"; const ShellCli = Capnp.importSystem("sandstorm/backend.capnp").ShellCli; class ShellCliImpl { diff --git a/shell/npm-shrinkwrap.json b/shell/npm-shrinkwrap.json index 2c263af1ab..52a00e0e5f 100644 --- a/shell/npm-shrinkwrap.json +++ b/shell/npm-shrinkwrap.json @@ -732,6 +732,9 @@ "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", "dev": true }, + "capnp": { + "version": "file:../deps/node-capnp" + }, "capture-stack-trace": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.1.tgz", @@ -2372,6 +2375,9 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "sandstorm": { + "version": "file:../src/sandstorm" + }, "sax": { "version": "0.5.8", "resolved": "https://registry.npmjs.org/sax/-/sax-0.5.8.tgz", diff --git a/shell/package.json b/shell/package.json index efb4a2fec7..4e252c8434 100644 --- a/shell/package.json +++ b/shell/package.json @@ -36,6 +36,7 @@ "babel-runtime": "^6.26.0", "bignum": "^0.13.0", "body-parser": "^1.19.0", + "capnp": "file:../deps/node-capnp", "content-type": "^1.0.4", "fibers": "^4.0.3", "heapdump": "^0.3.15", @@ -45,6 +46,7 @@ "node-forge": "^0.6.49", "nodemailer": "^2.7.2", "nodemailer-smtp-pool": "^2.8.3", + "sandstorm": "file:../src/sandstorm", "smtp-server": "^1.17.0", "stripe": "^8.81.0", "xml-crypto": "0.0.20", diff --git a/src/sandstorm/package.json b/src/sandstorm/package.json new file mode 100644 index 0000000000..7a0482a657 --- /dev/null +++ b/src/sandstorm/package.json @@ -0,0 +1,11 @@ +{ + "name": "sandstorm", + "version": "1.0.0", + "description": "Sandstorm's capnp schema.", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "Apache 2.0" +} From 3c30d82f6b6febd4a9c7a24774724be92a7bb9bc Mon Sep 17 00:00:00 2001 From: Ian Denhardt Date: Mon, 7 Dec 2020 10:27:50 -0500 Subject: [PATCH 2/3] Re-use the capnp.node from the ekam build ...rather than letting npm build it again (and possibly link against the wrong capnp). --- Makefile | 10 ++++++++-- deps/node-capnp | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e2979759b6..49dca69cd2 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,9 @@ CFLAGS2=$(CFLAGS) -pthread -fPIC -DKJ_STD_COMPAT # -lrt is not used by sandstorm itself, but the test app uses it. It would be # nice if we could not link everything against it. LIBS2=$(LIBS) deps/libsodium/build/src/libsodium/.libs/libsodium.a deps/boringssl/build/ssl/libssl.a deps/boringssl/build/crypto/libcrypto.a -lz -ldl -pthread -lrt +# Note: despite the name, we actually do build this in-tree; it is pre-built from +# perspective of npm, since we build it with ekam. +PREBUILT_NODE_CAPNP_BINARY=$(shell pwd)/tmp/node-capnp/capnp.node define color printf '\033[0;34m==== $1 ====\033[0m\n' @@ -373,14 +376,17 @@ shell/public/%-m.svg: icons/%.svg shell-build: shell/imports/* shell/imports/*/* shell/imports/*/*/* shell/imports/*/*/*/* shell/client/main.ts shell/server/main.ts shell/public/* shell/i18n/* shell/.meteor/packages shell/.meteor/release shell/.meteor/versions tmp/.shell-env @$(call color,building meteor frontend) @test -z "$$(find -L shell/* -type l)" || (echo "error: broken symlinks in shell: $$(find -L shell/* -type l)" >&2 && exit 1) - @OLD=`pwd` && cd shell && meteor build --directory "$$OLD/shell-build" + @OLD=`pwd` && \ + cd shell && \ + PREBUILT_NODE_CAPNP_BINARY="$(PREBUILT_NODE_CAPNP_BINARY)" \ + meteor build --directory "$$OLD/shell-build" # ==================================================================== # Bundle bundle: tmp/.ekam-run shell-build make-bundle.sh localedata-C meteor-bundle-main.js @$(call color,bundle) - @CC=$(CC) ./make-bundle.sh + @CC=$(CC) PREBUILT_NODE_CAPNP_BINARY=$(PREBUILT_NODE_CAPNP_BINARY) ./make-bundle.sh sandstorm-$(BUILD).tar.xz: bundle @$(call color,compress release bundle) diff --git a/deps/node-capnp b/deps/node-capnp index 544bf09271..25af0c6c80 160000 --- a/deps/node-capnp +++ b/deps/node-capnp @@ -1 +1 @@ -Subproject commit 544bf09271ecc1334b4a57b7d247a41007eb4c5a +Subproject commit 25af0c6c80425ea717d8f86cd0e6add330f4d1ab From 2cd680eb139d70fee1101b6bbc41b4e785c7dbb9 Mon Sep 17 00:00:00 2001 From: Ian Denhardt Date: Mon, 7 Dec 2020 10:29:07 -0500 Subject: [PATCH 3/3] Get rid of now-superfluous top-level node_modules directory. --- make-bundle.sh | 3 --- src/sandstorm/sandstorm.ekam-manifest | 19 ------------------- 2 files changed, 22 deletions(-) diff --git a/make-bundle.sh b/make-bundle.sh index 4e217542a7..03dbe5b754 100755 --- a/make-bundle.sh +++ b/make-bundle.sh @@ -154,9 +154,6 @@ mkdir -p bundle/usr/include/{capnp,sandstorm} cp src/capnp/!(*test*).capnp bundle/usr/include/capnp cp src/sandstorm/!(*-internal).capnp bundle/usr/include/sandstorm -# Copy over node_modules. -cp -r node_modules bundle - # Copy over all necessary shared libraries. (ldd bundle/bin/* $(find bundle -name '*.node') || true) | grep -o '[[:space:]]/[^ ]*' | copyDeps diff --git a/src/sandstorm/sandstorm.ekam-manifest b/src/sandstorm/sandstorm.ekam-manifest index 774275985d..2c9ad4bec3 100644 --- a/src/sandstorm/sandstorm.ekam-manifest +++ b/src/sandstorm/sandstorm.ekam-manifest @@ -1,22 +1,3 @@ run-bundle bin/sandstorm spk-standalone bin/spk sandstorm-http-bridge bin -email.capnp node_modules/sandstorm/email.capnp -email-impl.capnp node_modules/sandstorm/email-impl.capnp -grain.capnp node_modules/sandstorm/grain.capnp -api-session.capnp node_modules/sandstorm/api-session.capnp -api-session-impl.capnp node_modules/sandstorm/api-session-impl.capnp -hack-session.capnp node_modules/sandstorm/hack-session.capnp -ip.capnp node_modules/sandstorm/ip.capnp -package.capnp node_modules/sandstorm/package.capnp -supervisor.capnp node_modules/sandstorm/supervisor.capnp -util.capnp node_modules/sandstorm/util.capnp -web-session.capnp node_modules/sandstorm/web-session.capnp -backend.capnp node_modules/sandstorm/backend.capnp -persistentuiview.capnp node_modules/sandstorm/persistentuiview.capnp -update-tool.capnp node_modules/sandstorm/update-tool.capnp -powerbox.capnp node_modules/sandstorm/powerbox.capnp -activity.capnp node_modules/sandstorm/activity.capnp -identity.capnp node_modules/sandstorm/identity.capnp -identity-impl.capnp node_modules/sandstorm/identity-impl.capnp -payments.capnp node_modules/sandstorm/payments.capnp