From d0d2e19dcecced848ddbf8541bed40981fb026f5 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Tue, 16 Dec 2014 22:04:28 -0800 Subject: [PATCH 01/43] casing fixup --- src/colony/modules/net.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/colony/modules/net.js b/src/colony/modules/net.js index e14cbfa4..dc6619c0 100644 --- a/src/colony/modules/net.js +++ b/src/colony/modules/net.js @@ -14,7 +14,7 @@ var tm = process.binding('tm'); var util = require('util'); var dns = require('dns'); -var Stream = require('stream'); +var stream = require('stream'); var tls = require('tls'); /** @@ -40,8 +40,10 @@ function toNumber(x) { return (x = Number(x)) >= 0 ? x : false; } * TCPSocket */ +// TODO: this will need to inherit from Socket too + function TCPSocket (socket, _secure) { - Stream.Duplex.call(this); + stream.Duplex.call(this); if (typeof socket === 'object') { this.socket = socket.fd; @@ -74,7 +76,7 @@ function TCPSocket (socket, _secure) { process.on('tcp-close', this._closehandler) } -util.inherits(TCPSocket, Stream.Duplex); +util.inherits(TCPSocket, stream.Duplex); TCPSocket._portsUsed = Object.create(null); From a959ccf6b1c842ffd748a649972e2550268bcfab Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Tue, 16 Dec 2014 22:04:54 -0800 Subject: [PATCH 02/43] rough w.i.p. streamplex/proxy client integration into net module --- package.json | 3 +- src/colony/modules/net-tmp.js | 72 +++++++++++++++++++++++++++++++++++ t_proxyclient.js | 6 +++ 3 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 src/colony/modules/net-tmp.js create mode 100644 t_proxyclient.js diff --git a/package.json b/package.json index bd75d3e9..97a8c633 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,8 @@ "bindings": "~1.2.0", "colony-compiler": "~0.6.23", "mkdirp": "~0.3.5", - "semver": "^4.1.0" + "semver": "^4.1.0", + "streamplex": "^0.8.0" }, "devDependencies": { "nodejs-websocket": "^1.0.0", diff --git a/src/colony/modules/net-tmp.js b/src/colony/modules/net-tmp.js new file mode 100644 index 00000000..2bca6152 --- /dev/null +++ b/src/colony/modules/net-tmp.js @@ -0,0 +1,72 @@ +// NOTE: keeping separate for quicker standalone testing right now + +var util = require('util'), + stream = require('stream'), + events = require('events'), + connect = require('net').connect; + +/** + * Socket + */ + +function Socket(opts) { + stream.Duplex.call(this, opts); +} +util.inherits(Socket, stream.Duplex); + + + +/** + * ProxiedSocket + */ + + +function createTunnel(tokenServer, proxyServer, cb) { + var streamplex = require('streamplex'); + + var tokenSocket = connect(tokenServer, function () { + var token = []; + tokenSocket.write("DEV-CRED"); + tokenSocket.on('data', function (chunk) { + token.push(chunk); + }); + tokenSocket.on('end', function () { + token = Buffer.concat(token); + if (!token.length) return cb(new Error("Credentials rejected (or token server down).")); + + var proxySocket = connect(proxyServer, function () { + proxySocket.write(token); + var tunnel = streamplex(streamplex.B_SIDE, {subclass:ProxiedSocket}); + tunnel.pipe(proxySocket).pipe(tunnel); + cb(null, tunnel); + }); + // TODO: more error handling, etc. + }); + }); + tokenSocket.on('error', function (e) { + tokenSocket.destroy(); + cb(e); + }); +} + +var tunnel, + emitter = new events.EventEmitter(); +createTunnel({port:5006}, {port:5005}, function (e, _tunnel) { + if (e) return console.error(e); + tunnel = _tunnel; + emitter.emit('ready'); +}); + +function ProxiedSocket(opts) { + return tunnel.createStream(); +} +util.inherits(ProxiedSocket, Socket); + +module.exports = emitter; + +exports.createConnection = function (opts, cb) { + var socket = new ProxiedSocket(); + if (cb) socket.on('connect', cb); + ProxiedSocket.prototype.connect.apply(socket, arguments); + return socket; +}; diff --git a/t_proxyclient.js b/t_proxyclient.js new file mode 100644 index 00000000..eb20634b --- /dev/null +++ b/t_proxyclient.js @@ -0,0 +1,6 @@ +var net = require("./src/colony/modules/net-tmp.js"); + +net.on('ready', function () { + console.log(net.createConnection) +}); + From 461fce7b168426d0848e7c1d9e39992c3a85d68a Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Thu, 18 Dec 2014 14:45:51 -0600 Subject: [PATCH 03/43] finish hooking ProxiedSocket up into net-tmp --- src/colony/modules/net-tmp.js | 56 ++++++++++++++++++++++++----------- t_proxyclient.js | 7 +++-- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/src/colony/modules/net-tmp.js b/src/colony/modules/net-tmp.js index 2bca6152..4888c92b 100644 --- a/src/colony/modules/net-tmp.js +++ b/src/colony/modules/net-tmp.js @@ -6,20 +6,8 @@ var util = require('util'), connect = require('net').connect; /** - * Socket + * Temporary tunnel globals */ - -function Socket(opts) { - stream.Duplex.call(this, opts); -} -util.inherits(Socket, stream.Duplex); - - - -/** - * ProxiedSocket - */ - function createTunnel(tokenServer, proxyServer, cb) { var streamplex = require('streamplex'); @@ -57,16 +45,48 @@ createTunnel({port:5006}, {port:5005}, function (e, _tunnel) { emitter.emit('ready'); }); + +/** + * Socket + */ + +function Socket(opts) { + stream.Duplex.call(this, opts); +} +util.inherits(Socket, stream.Duplex); + + +/** + * ProxiedSocket + */ + +function ProxiedSocketConstructor() { + return tunnel.createStream(); +} + + function ProxiedSocket(opts) { - return tunnel.createStream(); + Socket.call(this); } util.inherits(ProxiedSocket, Socket); -module.exports = emitter; +ProxiedSocket.prototype.connect = function (port, host, cb) { + if (typeof host === 'function') { + cb = host; + host = null; // 'localhost' not useful here… + } + // NOTE: avoid sending garbage, but proxy must still guard itself properly! + if (typeof host !== 'string') throw Error("Host string must be provided"); + if (typeof port !== 'number') throw Error("Port number must be provided"); + this.remoteEmit('_pls_connect', port, host); + if (cb) this.on('connect', cb); +}; + + +exports = module.exports = emitter; -exports.createConnection = function (opts, cb) { - var socket = new ProxiedSocket(); - if (cb) socket.on('connect', cb); +exports.createConnection = function (port, host, cb) { + var socket = new ProxiedSocketConstructor(); ProxiedSocket.prototype.connect.apply(socket, arguments); return socket; }; diff --git a/t_proxyclient.js b/t_proxyclient.js index eb20634b..9853de5a 100644 --- a/t_proxyclient.js +++ b/t_proxyclient.js @@ -1,6 +1,9 @@ var net = require("./src/colony/modules/net-tmp.js"); net.on('ready', function () { - console.log(net.createConnection) + net.createConnection(80, 'ipcalf.com', function () { + this.write(["GET / HTTP/1.1", "Host: ipcalf.com", "Connection: close", '',''].join('\r\n')); + this.on('data', function (d) { console.log(d.toString()); }); + this.on('end', function () { console.log("[[end]]"); }); + }); }); - From a84d76ddea5ad9f91a196354f31450f0e4fbc41b Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Tue, 23 Dec 2014 11:28:45 -0600 Subject: [PATCH 04/43] (broken at least due to streamplex require) start towards class cluster pattern --- config/libcolony.gyp | 1 + src/colony/modules/net.js | 27 ++++++++++++++++--- .../modules/{net-tmp.js => net_proxied.js} | 11 +------- 3 files changed, 26 insertions(+), 13 deletions(-) rename src/colony/modules/{net-tmp.js => net_proxied.js} (93%) diff --git a/config/libcolony.gyp b/config/libcolony.gyp index 619ba549..aac6c4f5 100644 --- a/config/libcolony.gyp +++ b/config/libcolony.gyp @@ -148,6 +148,7 @@ '<(runtime_path)/colony/modules/http.js', '<(runtime_path)/colony/modules/https.js', '<(runtime_path)/colony/modules/net.js', + '<(runtime_path)/colony/modules/net_proxied.js', '<(runtime_path)/colony/modules/os.js', '<(node_libs_path)/path.js', '<(node_libs_path)/punycode.js', diff --git a/src/colony/modules/net.js b/src/colony/modules/net.js index dc6619c0..003a254b 100644 --- a/src/colony/modules/net.js +++ b/src/colony/modules/net.js @@ -16,6 +16,8 @@ var util = require('util'); var dns = require('dns'); var stream = require('stream'); var tls = require('tls'); +var net_proxied = require('net_proxied'); + /** * ip/helpers @@ -36,6 +38,25 @@ function isPipeName(s) { function toNumber(x) { return (x = Number(x)) >= 0 ? x : false; } +/** + * Socket + */ + +function Socket(socket, _secure) { + // NOTE: this implements a "class cluster" pattern + // Do *not* call this from subclasses! use `Socket._Constructor` instead + + if (net_proxied.tunnel) return net_proxied.tunnel.createStream(); // TODO: need on-demand (async!) tunnel, *and* per-target routing… + else return new TCPSocket(socket, _secure); +} +Socket._Constructor = function () { // actual + stream.Duplex.call(this); +}; +util.inherits(Socket, stream.Duplex); + + + + /** * TCPSocket */ @@ -43,7 +64,7 @@ function toNumber(x) { return (x = Number(x)) >= 0 ? x : false; } // TODO: this will need to inherit from Socket too function TCPSocket (socket, _secure) { - stream.Duplex.call(this); + Socket._Constructor.call(this); if (typeof socket === 'object') { this.socket = socket.fd; @@ -76,7 +97,7 @@ function TCPSocket (socket, _secure) { process.on('tcp-close', this._closehandler) } -util.inherits(TCPSocket, stream.Duplex); +util.inherits(TCPSocket, Socket); TCPSocket._portsUsed = Object.create(null); @@ -659,6 +680,6 @@ exports.isIPv4 = isIPv4; exports.connect = exports.createConnection = connect; exports._secureConnect = _secureConnect; exports.createServer = createServer; -exports.Socket = TCPSocket; +exports.Socket = Socket; exports.Server = TCPServer; exports._normalizeConnectArgs = normalizeConnectArgs; diff --git a/src/colony/modules/net-tmp.js b/src/colony/modules/net_proxied.js similarity index 93% rename from src/colony/modules/net-tmp.js rename to src/colony/modules/net_proxied.js index 4888c92b..aa263d84 100644 --- a/src/colony/modules/net-tmp.js +++ b/src/colony/modules/net_proxied.js @@ -3,7 +3,7 @@ var util = require('util'), stream = require('stream'), events = require('events'), - connect = require('net').connect; + net = require('net'); /** * Temporary tunnel globals @@ -46,14 +46,6 @@ createTunnel({port:5006}, {port:5005}, function (e, _tunnel) { }); -/** - * Socket - */ - -function Socket(opts) { - stream.Duplex.call(this, opts); -} -util.inherits(Socket, stream.Duplex); /** @@ -64,7 +56,6 @@ function ProxiedSocketConstructor() { return tunnel.createStream(); } - function ProxiedSocket(opts) { Socket.call(this); } From df6b08d73d172b74ceae725b732bcb3873af5541 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Tue, 23 Dec 2014 13:36:15 -0600 Subject: [PATCH 05/43] clean up tls suite output a bit --- test/suite/tls.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/suite/tls.js b/test/suite/tls.js index e3f4869f..e1e9244d 100644 --- a/test/suite/tls.js +++ b/test/suite/tls.js @@ -10,16 +10,16 @@ var options = { }; var socket = tls.connect(options, function connected() { - tap.eq(true, true, 'connect callback is called'); + tap.ok(true, 'connect callback is called'); }); socket.once('secureConnect', function() { - tap.eq(true, true, 'secureConnect event is called'); + tap.ok(true, 'secureConnect event is called'); socket.write('GET / HTTP/1.1\nAccept: */*\nHost: www.google.com\nUser-Agent: HTTPie/0.7.2\n\n'); }); socket.once('data', function(data) { - tap.eq(data.length > 0, true, 'we got data back from google over a secure TCP socket'); + tap.ok(data.length > 0, 'we got data back from google over a secure TCP socket'); socket.destroy(); }); From 1fd0f2349f8e8765c0847827e947275361721b34 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Tue, 23 Dec 2014 13:39:47 -0600 Subject: [PATCH 06/43] factor Socket subclasses in a different direction: must be handled in .connect method rather than instantiation --- src/colony/modules/net.js | 167 +++++++++++++++++++++----------------- src/colony/modules/tls.js | 20 +---- 2 files changed, 92 insertions(+), 95 deletions(-) diff --git a/src/colony/modules/net.js b/src/colony/modules/net.js index 003a254b..ab397f12 100644 --- a/src/colony/modules/net.js +++ b/src/colony/modules/net.js @@ -16,7 +16,7 @@ var util = require('util'); var dns = require('dns'); var stream = require('stream'); var tls = require('tls'); -var net_proxied = require('net_proxied'); +//var net_proxied = require('net_proxied'); /** @@ -42,42 +42,66 @@ function toNumber(x) { return (x = Number(x)) >= 0 ? x : false; } * Socket */ -function Socket(socket, _secure) { - // NOTE: this implements a "class cluster" pattern - // Do *not* call this from subclasses! use `Socket._Constructor` instead +function Socket(opts) { + //if (!(this instanceof Socket)) return new Socket(opts); + if (!(this instanceof TCPSocket)) return new TCPSocket(opts); + switch (typeof opts) { + case 'number': + opts = { fd: opts }; // Legacy interface. + break; + case 'undefined': + opts = {}; + break; + } + stream.Duplex.call(this, opts); - if (net_proxied.tunnel) return net_proxied.tunnel.createStream(); // TODO: need on-demand (async!) tunnel, *and* per-target routing… - else return new TCPSocket(socket, _secure); + this._opts = opts; + this._secure = opts._secure; + // TODO: finish core setup } -Socket._Constructor = function () { // actual - stream.Duplex.call(this); -}; util.inherits(Socket, stream.Duplex); +Socket.prototype._read = function () {}; +Socket.prototype._write = function (buf, enc, cb) { + throw Error("NOT READY!"); + // NOTE: under node v0.12 we could simply call `this.cork()` in constructor and leave this unimplemented + // holds the chunk (without calling cb) until we know what type of socket we are (post-connect) + this._pending = {buf:buf, enc:enc, cb:cb}; +}; +Socket.prototype.connect = function (opts, cb) { + if (typeof opts !== 'object') { + var args = normalizeConnectArgs(arguments, this._secure); + return Socket.prototype.connect.apply(this, args); + } + + if (cb && this._secure) this.once('secureConnect', cb); + else if (cb) this.once('connect', cb); + + // TODO: convert to necessary subclass *HERE* based on configuration + // TODO: handle _pending stuff (or subclass responsibility?) + + this._connect(+opts.port, opts.host || "127.0.0.1", cb); + return this; +}; + +// TODO: what other "abstract" methods do we need? /** * TCPSocket */ -// TODO: this will need to inherit from Socket too - -function TCPSocket (socket, _secure) { - Socket._Constructor.call(this); +function TCPSocket (opts) { + if (!(this instanceof TCPSocket)) return new TCPSocket(opts); + Socket.call(this, opts); - if (typeof socket === 'object') { - this.socket = socket.fd; - // TODO: respect readable/writable flags - if (socket.allowHalfOpen) console.warn("Ignoring allowHalfOpen option."); - } - - this._secure = _secure; this._outgoing = []; this._sending = false; this._queueEnd = false; - this.socket = (socket === undefined) ? null : socket; - + // TODO: this.socket should be this._socket — it is not public! + this.socket = (this._opts.fd === undefined) ? null : this._opts.fd; + var self = this; self.on('finish', function () { @@ -111,52 +135,14 @@ TCPSocket._requestPort = function (port) { return port; }; -function normalizeConnectArgs(args) { - var options = {}; - - if (util.isObject(args[0])) { - // connect(options, [cb]) - options = args[0]; - } else if (isPipeName(args[0])) { - // connect(path, [cb]); - options.path = args[0]; - } else { - // connect(port, [host], [cb]) - options.port = args[0]; - if (util.isString(args[1])) { - options.host = args[1]; - } - } - - var cb = args[args.length - 1]; - return util.isFunction(cb) ? [options, cb] : [options]; -} - -TCPSocket.prototype.connect = function (/*options | [port], [host], [cb]*/) { +TCPSocket.prototype._connect = function (port, host) { var self = this; - - var args = normalizeConnectArgs(arguments); - var opts = args[0]; - if (opts.allowHalfOpen) console.warn("Ignoring allowHalfOpen option."); - var port = +opts.port; - var host = opts.host || "127.0.0.1"; - var cb = args[1]; self.remotePort = port; self.remoteAddress = host; // TODO: proper value for these? self.localPort = 0; self.localAddress = "0.0.0.0"; - - if (cb) { - if (self._secure) { - self.once('secureConnect', cb); - } - else { - self.once('connect', cb); - } - - } - + if (isIP(host)) { setUpConnection(host); } else { @@ -555,20 +541,50 @@ TCPSocket.prototype.setNoDelay = function (val) { if (val) console.warn("Ignoring call to setNoDelay. TCP_NODELAY socket option not supported."); }; -function connect (port, host, callback) { - var client = new TCPSocket(null); - TCPSocket.prototype.connect.apply(client, arguments); - return client; -}; -// HACK: this is a quick solution to the regressions introduced by 5fb859605b183b70b246328bff24f4e4f8b50dab -// a more complete solution is implemented in a different PR: c015017492980271fa583fce57d798de26a12dab -function _secureConnect (options, callback) { - var client = new TCPSocket(null, true); - TCPSocket.prototype.connect.apply(client, arguments); - return client; +function normalizeConnectArgs(args, _secure) { + var options = {}; + + if (util.isObject(args[0])) { + // connect(options, [cb]) + options = args[0]; + } else if (isPipeName(args[0])) { + // connect(path, [cb]); + options.path = args[0]; + } else { + // connect(port, [host], [cb]) + options.port = args[0]; + if (util.isString(args[1])) { + options.host = args[1]; + } + } + + if (_secure) { + var listArgs = args; + if (util.isObject(listArgs[1])) { + options = util._extend(options, listArgs[1]); + } else if (util.isObject(listArgs[2])) { + options = util._extend(options, listArgs[2]); + } + } + + var cb = args[args.length - 1]; + return util.isFunction(cb) ? [options, cb] : [options]; +} + +function connect () { + var args = normalizeConnectArgs(arguments); + var s = new Socket(args[0]); + return Socket.prototype.connect.apply(s, args); }; +function secureConnect () { + var args = normalizeConnectArgs(arguments, true); + args[0]._secure = true; + var s = new Socket(args[0]); + return Socket.prototype.connect.apply(s, args); +} + /** * Server @@ -626,7 +642,7 @@ TCPServer.prototype.listen = function (port, host, backlog, cb) { , port = _[2]; if (client >= 0) { - var clientsocket = new TCPSocket(client); + var clientsocket = new TCPSocket({fd:client}); clientsocket.connected = true; clientsocket.localAddress = self.localAddress; // TODO: https://forums.tessel.io/t/get-ip-address-of-tessel-in-code/203 clientsocket.localPort = self.localPort; @@ -678,8 +694,7 @@ function createServer (opts, onsocket) { exports.isIP = isIP; exports.isIPv4 = isIPv4; exports.connect = exports.createConnection = connect; -exports._secureConnect = _secureConnect; +exports._secureConnect = secureConnect; // TLS module uses this exports.createServer = createServer; exports.Socket = Socket; exports.Server = TCPServer; -exports._normalizeConnectArgs = normalizeConnectArgs; diff --git a/src/colony/modules/tls.js b/src/colony/modules/tls.js index 291c518f..c37b4044 100644 --- a/src/colony/modules/tls.js +++ b/src/colony/modules/tls.js @@ -14,25 +14,7 @@ function NotImplementedException () { throw new Error('Not yet implemented.'); } -exports.connect = function connect () { - var arguments = normalizeConnectArgs(arguments); - var options = arguments[0]; - var callback = arguments[1]; - return net._secureConnect(options, callback); -} - -function normalizeConnectArgs(listArgs) { - var args = net._normalizeConnectArgs(listArgs); - var options = args[0]; - var cb = args[1]; - if (util.isObject(listArgs[1])) { - options = util._extend(options, listArgs[1]); - } else if (util.isObject(listArgs[2])) { - options = util._extend(options, listArgs[2]); - } - - return (cb) ? [options, cb] : [options]; -} +exports.connect = net._secureConnect; function checkServerIdentity (host, cert) { // Create regexp to much hostnames From 88549e2c8143e164577ad41669e3d1c389688053 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Thu, 25 Dec 2014 21:11:59 -0600 Subject: [PATCH 07/43] implement dynamic subclass conversion for net.Socket --- src/colony/modules/net.js | 21 +++++++++++++++------ test/suite/net.js | 3 ++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/colony/modules/net.js b/src/colony/modules/net.js index ab397f12..915fbdd9 100644 --- a/src/colony/modules/net.js +++ b/src/colony/modules/net.js @@ -43,8 +43,7 @@ function toNumber(x) { return (x = Number(x)) >= 0 ? x : false; } */ function Socket(opts) { - //if (!(this instanceof Socket)) return new Socket(opts); - if (!(this instanceof TCPSocket)) return new TCPSocket(opts); + if (!(this instanceof Socket)) return new Socket(opts); switch (typeof opts) { case 'number': opts = { fd: opts }; // Legacy interface. @@ -78,7 +77,12 @@ Socket.prototype.connect = function (opts, cb) { if (cb && this._secure) this.once('secureConnect', cb); else if (cb) this.once('connect', cb); - // TODO: convert to necessary subclass *HERE* based on configuration + // HACK: convert to necessary subclass here, now that we know necessary type + if (1) { + this.__proto__ = TCPSocket.prototype; + this._setup(this._opts); // pass original (constructor) opts + } + // TODO: handle _pending stuff (or subclass responsibility?) this._connect(+opts.port, opts.host || "127.0.0.1", cb); @@ -95,7 +99,12 @@ Socket.prototype.connect = function (opts, cb) { function TCPSocket (opts) { if (!(this instanceof TCPSocket)) return new TCPSocket(opts); Socket.call(this, opts); - + this._setup(opts); +} +util.inherits(TCPSocket, Socket); + + +TCPSocket.prototype._setup = function (opts) { this._outgoing = []; this._sending = false; this._queueEnd = false; @@ -119,9 +128,9 @@ function TCPSocket (opts) { } } process.on('tcp-close', this._closehandler) -} +}; + -util.inherits(TCPSocket, Socket); TCPSocket._portsUsed = Object.create(null); diff --git a/test/suite/net.js b/test/suite/net.js index 454cf654..006e36a1 100644 --- a/test/suite/net.js +++ b/test/suite/net.js @@ -1,6 +1,7 @@ // NOTE: see https://github.com/tcr/tinytap/issues/4 — not all tests get applied?! var test = require('tinytap'), - net = require('net'); + //net = require('net'); + net = require("../../src/colony/modules/net.js"); test.count(74); From ea020f1a46d8af7f18df8dcd4b53d744f98a7927 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Thu, 25 Dec 2014 21:59:18 -0600 Subject: [PATCH 08/43] bit closer to being able to asyncronously switch Socket subclass --- src/colony/modules/net.js | 23 +++++++++++-------- src/colony/modules/net_proxied.js | 38 ++++++++++++++++--------------- t_proxyclient.js | 9 -------- 3 files changed, 33 insertions(+), 37 deletions(-) delete mode 100644 t_proxyclient.js diff --git a/src/colony/modules/net.js b/src/colony/modules/net.js index 915fbdd9..6be14266 100644 --- a/src/colony/modules/net.js +++ b/src/colony/modules/net.js @@ -16,8 +16,6 @@ var util = require('util'); var dns = require('dns'); var stream = require('stream'); var tls = require('tls'); -//var net_proxied = require('net_proxied'); - /** * ip/helpers @@ -69,6 +67,9 @@ Socket.prototype._write = function (buf, enc, cb) { }; Socket.prototype.connect = function (opts, cb) { + // NOTE: imported here to avoid circular dependency + var net_proxied = require("./net_proxied.js"); + if (typeof opts !== 'object') { var args = normalizeConnectArgs(arguments, this._secure); return Socket.prototype.connect.apply(this, args); @@ -77,15 +78,16 @@ Socket.prototype.connect = function (opts, cb) { if (cb && this._secure) this.once('secureConnect', cb); else if (cb) this.once('connect', cb); - // HACK: convert to necessary subclass here, now that we know necessary type - if (1) { - this.__proto__ = TCPSocket.prototype; - this._setup(this._opts); // pass original (constructor) opts - } - - // TODO: handle _pending stuff (or subclass responsibility?) + var self = this, + port = +opts.port, + host = opts.host || "127.0.0.1"; + net_proxied._protoForConnection(host, port, function (e, proto) { + self.__proto__ = proto; // HACK: convert to "concrete" subclass here, now that we know necessary type + self._setup(self._opts); // pass original (constructor) opts + // TODO: handle _pending stuff (or subclass responsibility?) + self._connect(port, host, cb); + }); - this._connect(+opts.port, opts.host || "127.0.0.1", cb); return this; }; @@ -707,3 +709,4 @@ exports._secureConnect = secureConnect; // TLS module uses this exports.createServer = createServer; exports.Socket = Socket; exports.Server = TCPServer; +exports._CC3KSocket = TCPSocket; // net-proxied module uses this diff --git a/src/colony/modules/net_proxied.js b/src/colony/modules/net_proxied.js index aa263d84..37c52c8e 100644 --- a/src/colony/modules/net_proxied.js +++ b/src/colony/modules/net_proxied.js @@ -3,7 +3,12 @@ var util = require('util'), stream = require('stream'), events = require('events'), - net = require('net'); +// net = require('net'); + net = require("./net.js"); + + +var PROXY_TOKEN = "DEV-CRED", + PROXY_LOCAL = "10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16"; /** * Temporary tunnel globals @@ -14,7 +19,7 @@ function createTunnel(tokenServer, proxyServer, cb) { var tokenSocket = connect(tokenServer, function () { var token = []; - tokenSocket.write("DEV-CRED"); + tokenSocket.write(PROXY_TOKEN); tokenSocket.on('data', function (chunk) { token.push(chunk); }); @@ -37,16 +42,19 @@ function createTunnel(tokenServer, proxyServer, cb) { }); } -var tunnel, - emitter = new events.EventEmitter(); -createTunnel({port:5006}, {port:5005}, function (e, _tunnel) { - if (e) return console.error(e); - tunnel = _tunnel; - emitter.emit('ready'); -}); - +//var tunnel, +// emitter = new events.EventEmitter(); +//createTunnel({port:5006}, {port:5005}, function (e, _tunnel) { +// if (e) return console.error(e); +// tunnel = _tunnel; +// emitter.emit('ready'); +//}); +function protoForConnection(host, port, cb) { // CAUTION: this may callback syncronously! +console.log("HOST??", host) + cb(null, net._CC3KSocket.prototype); +} /** * ProxiedSocket @@ -59,7 +67,7 @@ function ProxiedSocketConstructor() { function ProxiedSocket(opts) { Socket.call(this); } -util.inherits(ProxiedSocket, Socket); +util.inherits(ProxiedSocket, net.Socket); ProxiedSocket.prototype.connect = function (port, host, cb) { if (typeof host === 'function') { @@ -74,10 +82,4 @@ ProxiedSocket.prototype.connect = function (port, host, cb) { }; -exports = module.exports = emitter; - -exports.createConnection = function (port, host, cb) { - var socket = new ProxiedSocketConstructor(); - ProxiedSocket.prototype.connect.apply(socket, arguments); - return socket; -}; +exports._protoForConnection = protoForConnection; \ No newline at end of file diff --git a/t_proxyclient.js b/t_proxyclient.js deleted file mode 100644 index 9853de5a..00000000 --- a/t_proxyclient.js +++ /dev/null @@ -1,9 +0,0 @@ -var net = require("./src/colony/modules/net-tmp.js"); - -net.on('ready', function () { - net.createConnection(80, 'ipcalf.com', function () { - this.write(["GET / HTTP/1.1", "Host: ipcalf.com", "Connection: close", '',''].join('\r\n')); - this.on('data', function (d) { console.log(d.toString()); }); - this.on('end', function () { console.log("[[end]]"); }); - }); -}); From 3517f12f1443e0da3ba14a055992ffcf0ccd388f Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Fri, 26 Dec 2014 11:26:35 -0600 Subject: [PATCH 09/43] =?UTF-8?q?further=20progress=20and=20experiment=20w?= =?UTF-8?q?ith=20returning=20ProxiedSocket=20instance=20as=20prototype=20?= =?UTF-8?q?=E2=80=94=C2=A0this=20does=20not=20work=20because=20pre-registe?= =?UTF-8?q?red=20events=20(e.g.=20connect=20listener)=20are=20not=20emitte?= =?UTF-8?q?d=20correctly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/colony/modules/net.js | 20 ++++--- src/colony/modules/net_proxied.js | 87 ++++++++++++++++++++----------- 2 files changed, 72 insertions(+), 35 deletions(-) diff --git a/src/colony/modules/net.js b/src/colony/modules/net.js index 6be14266..4f25600a 100644 --- a/src/colony/modules/net.js +++ b/src/colony/modules/net.js @@ -30,6 +30,13 @@ function isIP (host) { else return 0; } +function ipStrToInt(ip) { + var addr = ip.split('.').map(Number); + addr = ((addr[0] << 24) | (addr[1] << 16) | (addr[2] << 8) | addr[3]) >>> 0; + return addr; +} + + function isPipeName(s) { return util.isString(s) && toNumber(s) === false; } @@ -81,7 +88,9 @@ Socket.prototype.connect = function (opts, cb) { var self = this, port = +opts.port, host = opts.host || "127.0.0.1"; - net_proxied._protoForConnection(host, port, function (e, proto) { + net_proxied._protoForConnection(host, port, this._opts, function (e, proto) { + if (e) return self.emit('error', e); +console.log("HERE?", host, port); self.__proto__ = proto; // HACK: convert to "concrete" subclass here, now that we know necessary type self._setup(self._opts); // pass original (constructor) opts // TODO: handle _pending stuff (or subclass responsibility?) @@ -101,7 +110,7 @@ Socket.prototype.connect = function (opts, cb) { function TCPSocket (opts) { if (!(this instanceof TCPSocket)) return new TCPSocket(opts); Socket.call(this, opts); - this._setup(opts); + this._setup(this._opts); } util.inherits(TCPSocket, Socket); @@ -199,10 +208,8 @@ TCPSocket.prototype._connect = function (port, host) { var retries = 0; setImmediate(function doConnect() { - var addr = ip.split('.').map(Number); - addr = ((addr[0] << 24) | (addr[1] << 16) | (addr[2] << 8) | addr[3]) >>> 0; - - var ret = tm.tcp_connect(self.socket, addr, port); + var addr = ipStrToInt(ip), + ret = tm.tcp_connect(self.socket, addr, port); if (ret == -tm.ENETUNREACH) { // we're not connected to the internet self.emit('error', new Error("ENETUNREACH: Wifi is not connected")); @@ -704,6 +711,7 @@ function createServer (opts, onsocket) { exports.isIP = isIP; exports.isIPv4 = isIPv4; +exports._ipStrToInt = ipStrToInt; exports.connect = exports.createConnection = connect; exports._secureConnect = secureConnect; // TLS module uses this exports.createServer = createServer; diff --git a/src/colony/modules/net_proxied.js b/src/colony/modules/net_proxied.js index 37c52c8e..2ca79e1b 100644 --- a/src/colony/modules/net_proxied.js +++ b/src/colony/modules/net_proxied.js @@ -8,7 +8,8 @@ var util = require('util'), var PROXY_TOKEN = "DEV-CRED", - PROXY_LOCAL = "10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16"; + // see also https://tools.ietf.org/html/rfc5735#section-4 + PROXY_LOCAL = "10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 127.0.0.0/8 localhost"; /** * Temporary tunnel globals @@ -17,7 +18,7 @@ var PROXY_TOKEN = "DEV-CRED", function createTunnel(tokenServer, proxyServer, cb) { var streamplex = require('streamplex'); - var tokenSocket = connect(tokenServer, function () { + var tokenSocket = net.connect(tokenServer, function () { var token = []; tokenSocket.write(PROXY_TOKEN); tokenSocket.on('data', function (chunk) { @@ -42,44 +43,72 @@ function createTunnel(tokenServer, proxyServer, cb) { }); } -//var tunnel, -// emitter = new events.EventEmitter(); -//createTunnel({port:5006}, {port:5005}, function (e, _tunnel) { -// if (e) return console.error(e); -// tunnel = _tunnel; -// emitter.emit('ready'); -//}); -function protoForConnection(host, port, cb) { // CAUTION: this may callback syncronously! -console.log("HOST??", host) - cb(null, net._CC3KSocket.prototype); +var tunnelKeeper = new events.EventEmitter(); + +tunnelKeeper.getTunnel = function (cb) { // CAUTION: syncronous callback! + if (this._tunnel) return cb(null, this._tunnel); + + var self = this; + if (!this._pending) createTunnel({port:5006}, {port:5005}, function (e, tunnel) { + delete self._pending; + self._tunnel = tunnel; + self.emit('tunnel', e, tunnel); + }); + this._pending = true; + this.on('tunnel', cb); +}; + +var local_matchers = PROXY_LOCAL.split(' ').map(function (str) { + var parts = str.split('/'); + if (parts.length > 1) { + // IPv4 + mask + var bits = +parts[1], + mask = 0xFFFFFFFF << (32-bits) >>> 0, + base = net._ipStrToInt(parts[0]) & mask; // NOTE: left signed to match test below + return function (addr, host) { + return ((addr & mask) === base); + }; + } else if (str[0] === '.') { + // base including subdomains + str = str.slice(1); + return function (addr, host) { + var idx = host.lastIndexOf(str); + return (~idx && idx + str.length === host.length); + }; + } else return function (addr, host) { + // exact domain/address + return (host === str); + } +}); + +function protoForConnection(host, port, opts, cb) { // CAUTION: syncronous callback! + var addr = (net.isIPv4(host)) ? net._ipStrToInt(host) : null, + local = local_matchers.some(function (matcher) { return matcher(addr, host); }); + if (local) cb(null, net._CC3KSocket.prototype); + else tunnelKeeper.getTunnel(function (e, tunnel) { + if (e) return cb(e); + opts.name = (opts._secure) ? 'tls' : 'net'; + cb(null, tunnel.createStream(opts)); + }); } /** * ProxiedSocket */ -function ProxiedSocketConstructor() { - return tunnel.createStream(); -} - function ProxiedSocket(opts) { - Socket.call(this); + // NOTE: only intended for instantiation via protoForConnection! + Socket.call(this, opts); } util.inherits(ProxiedSocket, net.Socket); -ProxiedSocket.prototype.connect = function (port, host, cb) { - if (typeof host === 'function') { - cb = host; - host = null; // 'localhost' not useful here… - } - // NOTE: avoid sending garbage, but proxy must still guard itself properly! - if (typeof host !== 'string') throw Error("Host string must be provided"); - if (typeof port !== 'number') throw Error("Port number must be provided"); - this.remoteEmit('_pls_connect', port, host); - if (cb) this.on('connect', cb); -}; +ProxiedSocket.prototype._setup = function () {}; +ProxiedSocket.prototype._connect = function (port, host) { +console.log("HERE", port, host); + this.remoteEmit('_pls_connect', port, host); +}; -exports._protoForConnection = protoForConnection; \ No newline at end of file +exports._protoForConnection = protoForConnection; From 5eaabb1e5b1887274a62082d301cdc4acb96bda0 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Sat, 27 Dec 2014 08:02:56 -0600 Subject: [PATCH 10/43] continued tweaking --- src/colony/modules/net.js | 1 - src/colony/modules/net_proxied.js | 42 ++++++++++++++++++++++++------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/colony/modules/net.js b/src/colony/modules/net.js index 4f25600a..012a3768 100644 --- a/src/colony/modules/net.js +++ b/src/colony/modules/net.js @@ -90,7 +90,6 @@ Socket.prototype.connect = function (opts, cb) { host = opts.host || "127.0.0.1"; net_proxied._protoForConnection(host, port, this._opts, function (e, proto) { if (e) return self.emit('error', e); -console.log("HERE?", host, port); self.__proto__ = proto; // HACK: convert to "concrete" subclass here, now that we know necessary type self._setup(self._opts); // pass original (constructor) opts // TODO: handle _pending stuff (or subclass responsibility?) diff --git a/src/colony/modules/net_proxied.js b/src/colony/modules/net_proxied.js index 2ca79e1b..b89e549d 100644 --- a/src/colony/modules/net_proxied.js +++ b/src/colony/modules/net_proxied.js @@ -23,14 +23,16 @@ function createTunnel(tokenServer, proxyServer, cb) { tokenSocket.write(PROXY_TOKEN); tokenSocket.on('data', function (chunk) { token.push(chunk); + // HACK: workaround missing tcp-close event + tokenSocket.close(); }); tokenSocket.on('end', function () { token = Buffer.concat(token); if (!token.length) return cb(new Error("Credentials rejected (or token server down).")); - var proxySocket = connect(proxyServer, function () { + var proxySocket = net.connect(proxyServer, function () { proxySocket.write(token); - var tunnel = streamplex(streamplex.B_SIDE, {subclass:ProxiedSocket}); + var tunnel = streamplex(streamplex.B_SIDE); tunnel.pipe(proxySocket).pipe(tunnel); cb(null, tunnel); }); @@ -54,6 +56,11 @@ tunnelKeeper.getTunnel = function (cb) { // CAUTION: syncronous callback! if (!this._pending) createTunnel({port:5006}, {port:5005}, function (e, tunnel) { delete self._pending; self._tunnel = tunnel; + if (tunnel) { + var streamProto = Object.create(ProxiedSocket.prototype); + streamProto._tunnel = tunnel; + tunnel._streamProto = streamProto; + } self.emit('tunnel', e, tunnel); }); this._pending = true; @@ -89,8 +96,7 @@ function protoForConnection(host, port, opts, cb) { // CAUTION: syncronous cal if (local) cb(null, net._CC3KSocket.prototype); else tunnelKeeper.getTunnel(function (e, tunnel) { if (e) return cb(e); - opts.name = (opts._secure) ? 'tls' : 'net'; - cb(null, tunnel.createStream(opts)); + cb(null, tunnel._streamProto); }); } @@ -99,16 +105,34 @@ function protoForConnection(host, port, opts, cb) { // CAUTION: syncronous cal */ function ProxiedSocket(opts) { - // NOTE: only intended for instantiation via protoForConnection! - Socket.call(this, opts); + if (!(this instanceof ProxiedSocket)) return new ProxiedSocket(opts); + net.Socket.call(this, opts); + this._tunnel = this._opts.tunnel; + this._setup(this._opts); } util.inherits(ProxiedSocket, net.Socket); -ProxiedSocket.prototype._setup = function () {}; +ProxiedSocket.prototype._setup = function () { + var type = (this._secure) ? 'tls' : 'net'; + this._transport = this._tunnel.createStream(type); + + var self = this; + // TODO: it'd be great if we is-a substream instead of has-a… + this._transport.on('data', function () { + var more = self.push(d); + if (!more) self._transport.pause(); + }); +}; + +ProxiedSocket.prototype._read = function () { + this._transport.resume(); +}; +ProxiedSocket.prototype._write = function (buf, enc, cb) { + this._transport.write(buf, enc, cb); +}; ProxiedSocket.prototype._connect = function (port, host) { -console.log("HERE", port, host); - this.remoteEmit('_pls_connect', port, host); + this._transport.remoteEmit('_pls_connect', port, host); }; exports._protoForConnection = protoForConnection; From 37b4ef699a230fb0b0336044355a497b8a9e167f Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Sat, 27 Dec 2014 08:32:30 -0600 Subject: [PATCH 11/43] =?UTF-8?q?properly=20forward=20needed=20events=20to?= =?UTF-8?q?=20pass=20test=20suite=20(most=20of=20the=20time=E2=80=A6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/colony/modules/net_proxied.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/colony/modules/net_proxied.js b/src/colony/modules/net_proxied.js index b89e549d..a981d8e1 100644 --- a/src/colony/modules/net_proxied.js +++ b/src/colony/modules/net_proxied.js @@ -118,10 +118,21 @@ ProxiedSocket.prototype._setup = function () { var self = this; // TODO: it'd be great if we is-a substream instead of has-a… - this._transport.on('data', function () { + this._transport.on('data', function (d) { var more = self.push(d); if (!more) self._transport.pause(); }); + this._transport.on('end', function () { + self.push(null); + }); + + function reEmit(evt) { + self._transport.on(evt, function test() { + var args = Array.prototype.concat.apply([evt], arguments); + self.emit.apply(self, args); + }); + } + ['connect', 'secureConnect', 'error'].forEach(reEmit); }; ProxiedSocket.prototype._read = function () { From 68d177b69a12beff152346fa25fbb25a2bdda554 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Tue, 30 Dec 2014 08:13:14 -0600 Subject: [PATCH 12/43] more event forwarding --- src/colony/modules/net_proxied.js | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/colony/modules/net_proxied.js b/src/colony/modules/net_proxied.js index a981d8e1..c47eeb66 100644 --- a/src/colony/modules/net_proxied.js +++ b/src/colony/modules/net_proxied.js @@ -132,7 +132,7 @@ ProxiedSocket.prototype._setup = function () { self.emit.apply(self, args); }); } - ['connect', 'secureConnect', 'error'].forEach(reEmit); + ['connect', 'secureConnect', 'error', 'timeout'].forEach(reEmit); }; ProxiedSocket.prototype._read = function () { @@ -146,4 +146,16 @@ ProxiedSocket.prototype._connect = function (port, host) { this._transport.remoteEmit('_pls_connect', port, host); }; +ProxiedSocket.prototype.setTimeout = function (msecs, cb) { + this._transport.remoteEmit('_pls_timeout', msecs); + if (cb) { + if (msecs) this.once('timeout', cb); + else this.removeListener('timeout', cb); + } +}; + +ProxiedSocket.prototype.destroy = function () { + this._transport.destroy(); +}; + exports._protoForConnection = protoForConnection; From cb8ebfc92231008d82de4e4bab23185b012378c6 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Wed, 31 Dec 2014 09:07:33 -0600 Subject: [PATCH 13/43] avoid test failure due to multi-chunk larger response --- test/suite/net.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/suite/net.js b/test/suite/net.js index 006e36a1..8b400de6 100644 --- a/test/suite/net.js +++ b/test/suite/net.js @@ -78,7 +78,7 @@ test('client-basic', function (t) { t.ok(client instanceof net.Socket, "returned socket"); client.on('connect', function () { t.pass("socket connected"); - client.write("GET / HTTP/1.1\nHost: tessel-httpbin.herokuapp.com\nAccept: text/plain\n\n"); + client.write("GET /ip HTTP/1.1\nHost: tessel-httpbin.herokuapp.com\nAccept: text/plain\n\n"); }); client.on('error', function () { t.fail("socket error"); From c79791ee29d78de13b5b505dcc05b469f22fadef Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Wed, 31 Dec 2014 09:12:09 -0600 Subject: [PATCH 14/43] =?UTF-8?q?t.ok(true=E2=80=A6=20->=20t.pass(?= =?UTF-8?q?=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/suite/net.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/suite/net.js b/test/suite/net.js index 8b400de6..dfdd34cd 100644 --- a/test/suite/net.js +++ b/test/suite/net.js @@ -90,7 +90,7 @@ test('client-basic', function (t) { client.end(); }); client.on('end', function () { - t.ok(true, "socket closed"); + t.pass("socket closed"); t.end(); }); }); From 73c42036b72ae6111d278099467b57a9056fb857 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Wed, 31 Dec 2014 10:34:49 -0600 Subject: [PATCH 15/43] avoid delay of socket end event due to keep-alive --- test/suite/net.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/suite/net.js b/test/suite/net.js index dfdd34cd..53df9f13 100644 --- a/test/suite/net.js +++ b/test/suite/net.js @@ -78,7 +78,7 @@ test('client-basic', function (t) { t.ok(client instanceof net.Socket, "returned socket"); client.on('connect', function () { t.pass("socket connected"); - client.write("GET /ip HTTP/1.1\nHost: tessel-httpbin.herokuapp.com\nAccept: text/plain\n\n"); + client.write("GET /ip HTTP/1.1\nHost: tessel-httpbin.herokuapp.com\nConnection: close\nAccept: text/plain\n\n"); }); client.on('error', function () { t.fail("socket error"); From df064357f29c29e062ba03d3d46718f598fbc3b8 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Wed, 31 Dec 2014 10:36:14 -0600 Subject: [PATCH 16/43] use new tunnel-level auth, gets rid of now-extraneous tokenServer and avoids auth failure where initial stream messages got treated as part of proxy token --- src/colony/modules/net_proxied.js | 41 +++++++++++-------------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/src/colony/modules/net_proxied.js b/src/colony/modules/net_proxied.js index c47eeb66..14f534c8 100644 --- a/src/colony/modules/net_proxied.js +++ b/src/colony/modules/net_proxied.js @@ -15,34 +15,21 @@ var PROXY_TOKEN = "DEV-CRED", * Temporary tunnel globals */ -function createTunnel(tokenServer, proxyServer, cb) { +function createTunnel(proxyServer, cb) { var streamplex = require('streamplex'); - var tokenSocket = net.connect(tokenServer, function () { - var token = []; - tokenSocket.write(PROXY_TOKEN); - tokenSocket.on('data', function (chunk) { - token.push(chunk); - // HACK: workaround missing tcp-close event - tokenSocket.close(); - }); - tokenSocket.on('end', function () { - token = Buffer.concat(token); - if (!token.length) return cb(new Error("Credentials rejected (or token server down).")); - - var proxySocket = net.connect(proxyServer, function () { - proxySocket.write(token); - var tunnel = streamplex(streamplex.B_SIDE); - tunnel.pipe(proxySocket).pipe(tunnel); - cb(null, tunnel); - }); - // TODO: more error handling, etc. - }); - }); - tokenSocket.on('error', function (e) { - tokenSocket.destroy(); - cb(e); - }); + net.connect(proxyServer, function () { + // TODO: proxySocket/tunnel error handling + var proxySocket = this, + tunnel = streamplex(streamplex.B_SIDE); + tunnel.pipe(proxySocket).pipe(tunnel); + tunnel.sendMessage({token:PROXY_TOKEN}); + tunnel.once('message', function (d) { + proxySocket.removeListener('error', cb); + if (!d.authed) cb(new Error("Authorization failed.")); + else cb(null, tunnel); + }); + }).on('error', cb); } @@ -53,7 +40,7 @@ tunnelKeeper.getTunnel = function (cb) { // CAUTION: syncronous callback! if (this._tunnel) return cb(null, this._tunnel); var self = this; - if (!this._pending) createTunnel({port:5006}, {port:5005}, function (e, tunnel) { + if (!this._pending) createTunnel({port:5005}, function (e, tunnel) { delete self._pending; self._tunnel = tunnel; if (tunnel) { From 81e2541eee7b78244a1bc41b0460046609170d84 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Wed, 31 Dec 2014 11:08:30 -0600 Subject: [PATCH 17/43] buffer abstract socket methods and call them after subclass setup --- src/colony/modules/net.js | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/colony/modules/net.js b/src/colony/modules/net.js index 012a3768..5168bcf9 100644 --- a/src/colony/modules/net.js +++ b/src/colony/modules/net.js @@ -61,18 +61,25 @@ function Socket(opts) { this._opts = opts; this._secure = opts._secure; - // TODO: finish core setup + this._pendingCalls = []; } util.inherits(Socket, stream.Duplex); -Socket.prototype._read = function () {}; -Socket.prototype._write = function (buf, enc, cb) { - throw Error("NOT READY!"); - // NOTE: under node v0.12 we could simply call `this.cork()` in constructor and leave this unimplemented - // holds the chunk (without calling cb) until we know what type of socket we are (post-connect) - this._pending = {buf:buf, enc:enc, cb:cb}; +['_read', '_write', 'close', 'destroy', 'setTimeout', 'setNoDelay'].forEach(function (method) { + Socket.prototype[method] = function () { + this._pendingCalls.push({method:method, arguments:arguments}); + }; +}); + +Socket.prototype._doPending = function (proto) { + var self = this; + this._pendingCalls.forEach(function (call) { + proto[call.method].apply(self, call.arguments); + }); + delete this._pendingCalls; }; + Socket.prototype.connect = function (opts, cb) { // NOTE: imported here to avoid circular dependency var net_proxied = require("./net_proxied.js"); @@ -92,15 +99,13 @@ Socket.prototype.connect = function (opts, cb) { if (e) return self.emit('error', e); self.__proto__ = proto; // HACK: convert to "concrete" subclass here, now that we know necessary type self._setup(self._opts); // pass original (constructor) opts - // TODO: handle _pending stuff (or subclass responsibility?) + self._doPending(proto); self._connect(port, host, cb); }); return this; }; -// TODO: what other "abstract" methods do we need? - /** * TCPSocket From 1de84f0a61c6ddc1042e8582bba57c5a41de97fe Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Wed, 31 Dec 2014 11:42:13 -0600 Subject: [PATCH 18/43] extra whitespace --- test/suite/tls.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/suite/tls.js b/test/suite/tls.js index e1e9244d..50a554d7 100644 --- a/test/suite/tls.js +++ b/test/suite/tls.js @@ -22,5 +22,3 @@ socket.once('data', function(data) { tap.ok(data.length > 0, 'we got data back from google over a secure TCP socket'); socket.destroy(); }); - - From 45ad7d52c6c8d1fbfe5e130144881f9181d0aea0 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Wed, 31 Dec 2014 14:54:06 -0600 Subject: [PATCH 19/43] fussing with require statements, start getting tls working --- src/colony/modules/_net_proxied.js | 145 +++++++++++++++++++++++ src/colony/modules/_streamplex_module.js | 4 + src/colony/modules/tls.js | 3 +- test/suite/tls.js | 3 +- 4 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 src/colony/modules/_net_proxied.js create mode 100644 src/colony/modules/_streamplex_module.js diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js new file mode 100644 index 00000000..988f2b73 --- /dev/null +++ b/src/colony/modules/_net_proxied.js @@ -0,0 +1,145 @@ +// NOTE: keeping separate for quicker standalone testing right now +console.log("PARSING NET PROXIED"); + +var util = require('util'), + events = require('events'), + net = require('net'), + streamplex = require('_streamplex_module'); + +var PROXY_TOKEN = "DEV-CRED", + // see also https://tools.ietf.org/html/rfc5735#section-4 + PROXY_LOCAL = "10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 127.0.0.0/8 localhost"; + +/** + * Temporary tunnel globals + */ + +function createTunnel(proxyServer, cb) { + net.connect(proxyServer, function () { + // TODO: proxySocket/tunnel error handling + var proxySocket = this, + tunnel = streamplex(streamplex.B_SIDE); + tunnel.pipe(proxySocket).pipe(tunnel); + tunnel.sendMessage({token:PROXY_TOKEN}); + tunnel.once('message', function (d) { + proxySocket.removeListener('error', cb); + if (!d.authed) cb(new Error("Authorization failed.")); + else cb(null, tunnel); + }); + }).on('error', cb); +} + +var tunnelKeeper = new events.EventEmitter(); + +tunnelKeeper.getTunnel = function (cb) { // CAUTION: syncronous callback! + if (this._tunnel) return cb(null, this._tunnel); + + var self = this; + if (!this._pending) createTunnel({port:5005}, function (e, tunnel) { + delete self._pending; + self._tunnel = tunnel; + if (tunnel) { + var streamProto = Object.create(ProxiedSocket.prototype); + streamProto._tunnel = tunnel; + tunnel._streamProto = streamProto; + } + self.emit('tunnel', e, tunnel); + }); + this._pending = true; + this.on('tunnel', cb); +}; + +var local_matchers = PROXY_LOCAL.split(' ').map(function (str) { + var parts = str.split('/'); + if (parts.length > 1) { + // IPv4 + mask + var bits = +parts[1], + mask = 0xFFFFFFFF << (32-bits) >>> 0, + base = net._ipStrToInt(parts[0]) & mask; // NOTE: left signed to match test below + return function (addr, host) { + return ((addr & mask) === base); + }; + } else if (str[0] === '.') { + // base including subdomains + str = str.slice(1); + return function (addr, host) { + var idx = host.lastIndexOf(str); + return (~idx && idx + str.length === host.length); + }; + } else return function (addr, host) { + // exact domain/address + return (host === str); + } +}); + +function protoForConnection(host, port, opts, cb) { // CAUTION: syncronous callback! + var addr = (net.isIPv4(host)) ? net._ipStrToInt(host) : null, + local = local_matchers.some(function (matcher) { return matcher(addr, host); }); + if (local) cb(null, net._CC3KSocket.prototype); + else tunnelKeeper.getTunnel(function (e, tunnel) { + if (e) return cb(e); + cb(null, tunnel._streamProto); + }); +} + +/** + * ProxiedSocket + */ + +function ProxiedSocket(opts) { + if (!(this instanceof ProxiedSocket)) return new ProxiedSocket(opts); + net.Socket.call(this, opts); + this._tunnel = this._opts.tunnel; + this._setup(this._opts); +} +util.inherits(ProxiedSocket, net.Socket); + +ProxiedSocket.prototype._setup = function () { + var type = (this._secure) ? 'tls' : 'net'; + this._transport = this._tunnel.createStream(type); + + var self = this; + // TODO: it'd be great if we is-a substream instead of has-a… + this._transport.on('data', function (d) { + var more = self.push(d); + if (!more) self._transport.pause(); + }); + this._transport.on('end', function () { + self.push(null); + }); + + function reEmit(evt) { + self._transport.on(evt, function test() { + var args = Array.prototype.concat.apply([evt], arguments); + self.emit.apply(self, args); + }); + } + ['connect', 'secureConnect', 'error', 'timeout'].forEach(reEmit); +}; + +ProxiedSocket.prototype._read = function () { + this._transport.resume(); +}; +ProxiedSocket.prototype._write = function (buf, enc, cb) { + this._transport.write(buf, enc, cb); +}; + +ProxiedSocket.prototype._connect = function (port, host) { + this._transport.remoteEmit('_pls_connect', port, host); +}; + +ProxiedSocket.prototype.setTimeout = function (msecs, cb) { + this._transport.remoteEmit('_pls_timeout', msecs); + if (cb) { + if (msecs) this.once('timeout', cb); + else this.removeListener('timeout', cb); + } +}; + +ProxiedSocket.prototype.destroy = function () { +console.log("HERE"); + this._transport.destroy(); + this.end(); +}; + +exports._protoForConnection = protoForConnection; diff --git a/src/colony/modules/_streamplex_module.js b/src/colony/modules/_streamplex_module.js new file mode 100644 index 00000000..a7ca393c --- /dev/null +++ b/src/colony/modules/_streamplex_module.js @@ -0,0 +1,4 @@ +// TODO: how to build this external module in? +//module.exports = require('streamplex'); +//module.exports = require("/Users/natevw/Desktop/Clients/Technical_Machine/streamplex/index.js"); +module.exports = require("./node_modules/streamplex/index.js"); diff --git a/src/colony/modules/tls.js b/src/colony/modules/tls.js index c37b4044..76056ce8 100644 --- a/src/colony/modules/tls.js +++ b/src/colony/modules/tls.js @@ -7,7 +7,8 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -var net = require('net'); +//var net = require('net'); +var net = require("./net.js"); var util = require('util'); function NotImplementedException () { diff --git a/test/suite/tls.js b/test/suite/tls.js index 50a554d7..4b75a835 100644 --- a/test/suite/tls.js +++ b/test/suite/tls.js @@ -1,4 +1,5 @@ -var tls = require('tls'), +var //tls = require('tls'), + tls = require("../../src/colony/modules/tls.js"), tap = require('../tap'); tap.count(3); From e1de8fa8f672fca2a2449a8cfa00559cf7704826 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Wed, 31 Dec 2014 15:31:25 -0600 Subject: [PATCH 20/43] more dependency fussing --- src/colony/modules/_net_proxied.js | 11 +++++------ src/colony/modules/_streamplex_module.js | 5 ++--- src/colony/modules/net.js | 9 +++++---- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index 988f2b73..0e317c61 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -1,10 +1,10 @@ -// NOTE: keeping separate for quicker standalone testing right now -console.log("PARSING NET PROXIED"); - var util = require('util'), events = require('events'), - net = require('net'), - streamplex = require('_streamplex_module'); + //net = require('net'), + //streamplex = require('_streamplex_module'); + net = require("./net.js"), + streamplex = require("./_streamplex_module.js"); + var PROXY_TOKEN = "DEV-CRED", // see also https://tools.ietf.org/html/rfc5735#section-4 @@ -137,7 +137,6 @@ ProxiedSocket.prototype.setTimeout = function (msecs, cb) { }; ProxiedSocket.prototype.destroy = function () { -console.log("HERE"); this._transport.destroy(); this.end(); }; diff --git a/src/colony/modules/_streamplex_module.js b/src/colony/modules/_streamplex_module.js index a7ca393c..6aeb0dd7 100644 --- a/src/colony/modules/_streamplex_module.js +++ b/src/colony/modules/_streamplex_module.js @@ -1,4 +1,3 @@ // TODO: how to build this external module in? -//module.exports = require('streamplex'); -//module.exports = require("/Users/natevw/Desktop/Clients/Technical_Machine/streamplex/index.js"); -module.exports = require("./node_modules/streamplex/index.js"); +module.exports = require('streamplex'); +//module.exports = require("./node_modules/streamplex/index.js"); diff --git a/src/colony/modules/net.js b/src/colony/modules/net.js index 5168bcf9..2dc48562 100644 --- a/src/colony/modules/net.js +++ b/src/colony/modules/net.js @@ -15,7 +15,8 @@ var tm = process.binding('tm'); var util = require('util'); var dns = require('dns'); var stream = require('stream'); -var tls = require('tls'); +//var tls = require('tls'); +var tls = require("./tls.js"); /** * ip/helpers @@ -65,7 +66,7 @@ function Socket(opts) { } util.inherits(Socket, stream.Duplex); -['_read', '_write', 'close', 'destroy', 'setTimeout', 'setNoDelay'].forEach(function (method) { +['_read', '_write', 'close', 'zzz-destroy', 'setTimeout', 'setNoDelay'].forEach(function (method) { Socket.prototype[method] = function () { this._pendingCalls.push({method:method, arguments:arguments}); }; @@ -82,7 +83,7 @@ Socket.prototype._doPending = function (proto) { Socket.prototype.connect = function (opts, cb) { // NOTE: imported here to avoid circular dependency - var net_proxied = require("./net_proxied.js"); + var net_proxied = require("./_net_proxied.js"); if (typeof opts !== 'object') { var args = normalizeConnectArgs(arguments, this._secure); @@ -598,7 +599,7 @@ function connect () { var args = normalizeConnectArgs(arguments); var s = new Socket(args[0]); return Socket.prototype.connect.apply(s, args); -}; +} function secureConnect () { var args = normalizeConnectArgs(arguments, true); From 23c617706cc9e3d9852463b2d6a6db9324d61f4f Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Wed, 31 Dec 2014 16:36:39 -0600 Subject: [PATCH 21/43] close tunnel when not in use (tls test now terminates) --- package.json | 2 +- src/colony/modules/_net_proxied.js | 20 +++++++++++++------- src/colony/modules/net.js | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 97a8c633..311bcfc4 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "colony-compiler": "~0.6.23", "mkdirp": "~0.3.5", "semver": "^4.1.0", - "streamplex": "^0.8.0" + "streamplex": "^0.12.0" }, "devDependencies": { "nodejs-websocket": "^1.0.0", diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index 0e317c61..e902da26 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -20,6 +20,9 @@ function createTunnel(proxyServer, cb) { var proxySocket = this, tunnel = streamplex(streamplex.B_SIDE); tunnel.pipe(proxySocket).pipe(tunnel); + tunnel.once('inactive', function () { + proxySocket.destroy(); + }); tunnel.sendMessage({token:PROXY_TOKEN}); tunnel.once('message', function (d) { proxySocket.removeListener('error', cb); @@ -37,16 +40,19 @@ tunnelKeeper.getTunnel = function (cb) { // CAUTION: syncronous callback! var self = this; if (!this._pending) createTunnel({port:5005}, function (e, tunnel) { delete self._pending; + if (e) return self.emit('tunnel', e); + self._tunnel = tunnel; - if (tunnel) { - var streamProto = Object.create(ProxiedSocket.prototype); - streamProto._tunnel = tunnel; - tunnel._streamProto = streamProto; - } - self.emit('tunnel', e, tunnel); + tunnel.once('inactive', function () { + self._tunnel = null; + }); + var streamProto = Object.create(ProxiedSocket.prototype); + streamProto._tunnel = tunnel; + tunnel._streamProto = streamProto; + self.emit('tunnel', null, tunnel); }); this._pending = true; - this.on('tunnel', cb); + this.once('tunnel', cb); }; var local_matchers = PROXY_LOCAL.split(' ').map(function (str) { diff --git a/src/colony/modules/net.js b/src/colony/modules/net.js index 2dc48562..00f66420 100644 --- a/src/colony/modules/net.js +++ b/src/colony/modules/net.js @@ -66,7 +66,7 @@ function Socket(opts) { } util.inherits(Socket, stream.Duplex); -['_read', '_write', 'close', 'zzz-destroy', 'setTimeout', 'setNoDelay'].forEach(function (method) { +['_read', '_write', 'close', 'destroy', 'setTimeout', 'setNoDelay'].forEach(function (method) { Socket.prototype[method] = function () { this._pendingCalls.push({method:method, arguments:arguments}); }; From 196ec3579ab775e3a88a92f14bc7847ae28aa97f Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Fri, 9 Jan 2015 14:18:42 -0800 Subject: [PATCH 22/43] use browserify to build streamplex into colony --- config/common.gypi | 1 + config/libcolony.gyp | 16 ++- package.json | 3 +- src/colony/modules/_net_proxied.js | 8 +- src/colony/modules/_streamplex_module.js | 3 - src/colony/modules/net.js | 5 +- src/colony/modules/net_proxied.js | 148 ----------------------- src/colony/modules/tls.js | 3 +- test/suite/net.js | 4 +- 9 files changed, 26 insertions(+), 165 deletions(-) delete mode 100644 src/colony/modules/_streamplex_module.js delete mode 100644 src/colony/modules/net_proxied.js diff --git a/config/common.gypi b/config/common.gypi index 85489305..16f5c9fa 100644 --- a/config/common.gypi +++ b/config/common.gypi @@ -17,6 +17,7 @@ "miniz_path": "../deps/miniz", "miniz_inc_path": "../deps/miniz-inc", "node_libs_path": "../deps/node-libs", + "npm_bin_path": "../node_modules/.bin", "approxidate_path": "../deps/approxidate", "tools_path": "../tools", 'enable_ssl%': 0, diff --git a/config/libcolony.gyp b/config/libcolony.gyp index aac6c4f5..45f61ec5 100644 --- a/config/libcolony.gyp +++ b/config/libcolony.gyp @@ -126,6 +126,19 @@ '<(SHARED_INTERMEDIATE_DIR)/<(_target_name).c' ], 'actions': [ + { + 'action_name': 'bundle_streamplex', + 'variables': { + 'main_file': ' 1) { - // IPv4 + mask - var bits = +parts[1], - mask = 0xFFFFFFFF << (32-bits) >>> 0, - base = net._ipStrToInt(parts[0]) & mask; // NOTE: left signed to match test below - return function (addr, host) { - return ((addr & mask) === base); - }; - } else if (str[0] === '.') { - // base including subdomains - str = str.slice(1); - return function (addr, host) { - var idx = host.lastIndexOf(str); - return (~idx && idx + str.length === host.length); - }; - } else return function (addr, host) { - // exact domain/address - return (host === str); - } -}); - -function protoForConnection(host, port, opts, cb) { // CAUTION: syncronous callback! - var addr = (net.isIPv4(host)) ? net._ipStrToInt(host) : null, - local = local_matchers.some(function (matcher) { return matcher(addr, host); }); - if (local) cb(null, net._CC3KSocket.prototype); - else tunnelKeeper.getTunnel(function (e, tunnel) { - if (e) return cb(e); - cb(null, tunnel._streamProto); - }); -} - -/** - * ProxiedSocket - */ - -function ProxiedSocket(opts) { - if (!(this instanceof ProxiedSocket)) return new ProxiedSocket(opts); - net.Socket.call(this, opts); - this._tunnel = this._opts.tunnel; - this._setup(this._opts); -} -util.inherits(ProxiedSocket, net.Socket); - -ProxiedSocket.prototype._setup = function () { - var type = (this._secure) ? 'tls' : 'net'; - this._transport = this._tunnel.createStream(type); - - var self = this; - // TODO: it'd be great if we is-a substream instead of has-a… - this._transport.on('data', function (d) { - var more = self.push(d); - if (!more) self._transport.pause(); - }); - this._transport.on('end', function () { - self.push(null); - }); - - function reEmit(evt) { - self._transport.on(evt, function test() { - var args = Array.prototype.concat.apply([evt], arguments); - self.emit.apply(self, args); - }); - } - ['connect', 'secureConnect', 'error', 'timeout'].forEach(reEmit); -}; - -ProxiedSocket.prototype._read = function () { - this._transport.resume(); -}; -ProxiedSocket.prototype._write = function (buf, enc, cb) { - this._transport.write(buf, enc, cb); -}; - -ProxiedSocket.prototype._connect = function (port, host) { - this._transport.remoteEmit('_pls_connect', port, host); -}; - -ProxiedSocket.prototype.setTimeout = function (msecs, cb) { - this._transport.remoteEmit('_pls_timeout', msecs); - if (cb) { - if (msecs) this.once('timeout', cb); - else this.removeListener('timeout', cb); - } -}; - -ProxiedSocket.prototype.destroy = function () { - this._transport.destroy(); -}; - -exports._protoForConnection = protoForConnection; diff --git a/src/colony/modules/tls.js b/src/colony/modules/tls.js index 76056ce8..c37b4044 100644 --- a/src/colony/modules/tls.js +++ b/src/colony/modules/tls.js @@ -7,8 +7,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -//var net = require('net'); -var net = require("./net.js"); +var net = require('net'); var util = require('util'); function NotImplementedException () { diff --git a/test/suite/net.js b/test/suite/net.js index 53df9f13..e8fd4f75 100644 --- a/test/suite/net.js +++ b/test/suite/net.js @@ -1,7 +1,7 @@ // NOTE: see https://github.com/tcr/tinytap/issues/4 — not all tests get applied?! var test = require('tinytap'), - //net = require('net'); - net = require("../../src/colony/modules/net.js"); + net = require('net'); +// net = require("../../src/colony/modules/net.js"); test.count(74); From 0f6fc6cf22003664dbfe9b53ad999a0852a8f140 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Fri, 9 Jan 2015 15:21:51 -0800 Subject: [PATCH 23/43] fix broken net error test (by upgrading to fixed streamplex) and clean up test suite includes --- package.json | 2 +- test/suite/net.js | 1 - test/suite/tls.js | 3 +-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 260a23f0..fe08585e 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "colony-compiler": "~0.6.23", "mkdirp": "~0.3.5", "semver": "^4.1.0", - "streamplex": "^0.12.0" + "streamplex": "^0.12.1" }, "devDependencies": { "browserify": "^8.1.0", diff --git a/test/suite/net.js b/test/suite/net.js index e8fd4f75..ebc70b41 100644 --- a/test/suite/net.js +++ b/test/suite/net.js @@ -1,7 +1,6 @@ // NOTE: see https://github.com/tcr/tinytap/issues/4 — not all tests get applied?! var test = require('tinytap'), net = require('net'); -// net = require("../../src/colony/modules/net.js"); test.count(74); diff --git a/test/suite/tls.js b/test/suite/tls.js index 4b75a835..50a554d7 100644 --- a/test/suite/tls.js +++ b/test/suite/tls.js @@ -1,5 +1,4 @@ -var //tls = require('tls'), - tls = require("../../src/colony/modules/tls.js"), +var tls = require('tls'), tap = require('../tap'); tap.count(3); From 127c9db043c7b8a3e38844fb4aad3783dfa2feb1 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Fri, 9 Jan 2015 16:20:13 -0800 Subject: [PATCH 24/43] fix up issue with tls/net circular import causing HTTPS agent to create insecure socket (see https://github.com/tessel/runtime/issues/697 for cause of silent fallback) --- src/colony/modules/net.js | 4 ++-- src/colony/modules/tls.js | 1 + test/suite/https.js | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/colony/modules/net.js b/src/colony/modules/net.js index 062beddc..ed910ea9 100644 --- a/src/colony/modules/net.js +++ b/src/colony/modules/net.js @@ -15,7 +15,6 @@ var tm = process.binding('tm'); var util = require('util'); var dns = require('dns'); var stream = require('stream'); -var tls = require('tls'); /** * ip/helpers @@ -60,7 +59,7 @@ function Socket(opts) { stream.Duplex.call(this, opts); this._opts = opts; - this._secure = opts._secure; + this._secure = opts._secure; this._pendingCalls = []; } util.inherits(Socket, stream.Duplex); @@ -315,6 +314,7 @@ TCPSocket.prototype._connect = function (port, host) { } }; + var tls = require('tls'); // NOTE: imported here to avoid circular dependency! if (self._ssl_checkCerts && !tls.checkServerIdentity(host, self._ssl_cert)) { return self.emit('error', new Error('Hostname/IP doesn\'t match certificate\'s altnames')); } diff --git a/src/colony/modules/tls.js b/src/colony/modules/tls.js index c37b4044..4f67d2d5 100644 --- a/src/colony/modules/tls.js +++ b/src/colony/modules/tls.js @@ -14,6 +14,7 @@ function NotImplementedException () { throw new Error('Not yet implemented.'); } +if (!net._secureConnect) throw Error("Circular import detected!"); exports.connect = net._secureConnect; function checkServerIdentity (host, cert) { diff --git a/test/suite/https.js b/test/suite/https.js index 6d4419d7..a02c2d2d 100644 --- a/test/suite/https.js +++ b/test/suite/https.js @@ -3,7 +3,6 @@ var test = require('tinytap'), test('https', function (t) { var https = require('https') - console.log('imported'); https.get("https://tessel-httpbin.herokuapp.com/", function (res) { t.equal(res.statusCode, 200, 'https status code is 200') t.equal(res.connection.remotePort, 443, 'remote port is 443 for https') From d3466c8fd970782368bf9f9c6906ddc090813dc3 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Fri, 9 Jan 2015 16:23:50 -0800 Subject: [PATCH 25/43] finish getting HTTPS test to pass --- src/colony/modules/_net_proxied.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index 3b7ba240..ef93a883 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -129,6 +129,8 @@ ProxiedSocket.prototype._write = function (buf, enc, cb) { }; ProxiedSocket.prototype._connect = function (port, host) { + this.remotePort = port; + this.remoteAddress = host; this._transport.remoteEmit('_pls_connect', port, host); }; From e0c804b1a38317718481cb4cb7a3cdd6bb58318b Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Fri, 9 Jan 2015 17:20:40 -0800 Subject: [PATCH 26/43] destroy tunnel when proxySocket has an error --- package.json | 2 +- src/colony/modules/_net_proxied.js | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index fe08585e..141e2224 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "colony-compiler": "~0.6.23", "mkdirp": "~0.3.5", "semver": "^4.1.0", - "streamplex": "^0.12.1" + "streamplex": "^0.13.0" }, "devDependencies": { "browserify": "^8.1.0", diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index ef93a883..105de969 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -14,10 +14,12 @@ var PROXY_TOKEN = "DEV-CRED", function createTunnel(proxyServer, cb) { net.connect(proxyServer, function () { - // TODO: proxySocket/tunnel error handling var proxySocket = this, tunnel = streamplex(streamplex.B_SIDE); tunnel.pipe(proxySocket).pipe(tunnel); + proxySocket.on('error', function (e) { + tunnel.destroy(e); // substreams will each emit `e`, then go inactive + }); tunnel.once('inactive', function () { proxySocket.destroy(); }); From dd56bcf9435bfbba8d82a63092d724cbb5919515 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Tue, 20 Jan 2015 13:41:48 -0800 Subject: [PATCH 27/43] pull out proxy server host/port config --- src/colony/modules/_net_proxied.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index 105de969..520f1f7a 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -4,7 +4,9 @@ var util = require('util'), streamplex = require('_streamplex'); -var PROXY_TOKEN = "DEV-CRED", +var PROXY_HOST = "localhost", + PROXY_PORT = 5005, + PROXY_TOKEN = "DEV-CRED", // see also https://tools.ietf.org/html/rfc5735#section-4 PROXY_LOCAL = "10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 127.0.0.0/8 localhost"; @@ -12,8 +14,8 @@ var PROXY_TOKEN = "DEV-CRED", * Temporary tunnel globals */ -function createTunnel(proxyServer, cb) { - net.connect(proxyServer, function () { +function createTunnel(cb) { + net.connect({host:PROXY_HOST, port:PROXY_PORT}, function () { var proxySocket = this, tunnel = streamplex(streamplex.B_SIDE); tunnel.pipe(proxySocket).pipe(tunnel); @@ -38,7 +40,7 @@ tunnelKeeper.getTunnel = function (cb) { // CAUTION: syncronous callback! if (this._tunnel) return cb(null, this._tunnel); var self = this; - if (!this._pending) createTunnel({port:5005}, function (e, tunnel) { + if (!this._pending) createTunnel(function (e, tunnel) { delete self._pending; if (e) return self.emit('tunnel', e); From 1698c33a1323ded7e64da206b3d9f4f7ce8688fd Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Thu, 22 Jan 2015 13:23:04 -0800 Subject: [PATCH 28/43] get user-configuration stuff mostly into place --- src/colony/modules/_net_proxied.js | 31 ++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index 520f1f7a..bab2ffd5 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -3,12 +3,30 @@ var util = require('util'), net = require('net'), streamplex = require('_streamplex'); +// NOTE: this list may not be exhaustive, see also https://tools.ietf.org/html/rfc5735#section-4 +var _PROXY_LOCAL = "10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 127.0.0.0/8 localhost"; -var PROXY_HOST = "localhost", - PROXY_PORT = 5005, - PROXY_TOKEN = "DEV-CRED", - // see also https://tools.ietf.org/html/rfc5735#section-4 - PROXY_LOCAL = "10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 127.0.0.0/8 localhost"; +var PROXY_HOST = process.env.PROXY_HOST || "localhost", + PROXY_PORT = process.env.PROXY_PORT || 5005, + PROXY_TOKEN = process.env.PROXY_TOKEN || process.env.TM_API_KEY || "DEV-CRED", + PROXY_LOCAL = process.env.PROXY_LOCAL || _PROXY_LOCAL, + PROXY_CERT = process.env.PROXY_CERT || [ + "-----BEGIN CERTIFICATE-----", + "MIICTTCCAbYCCQCYeAQlYTG45DANBgkqhkiG9w0BAQUFADBrMQswCQYDVQQGEwJV", + "UzETMBEGA1UECBMKV2FzaGluZ3RvbjEWMBQGA1UEBxMNV2VzdCBSaWNobGFuZDEb", + "MBkGA1UEChMSTmF0aGFuIFZhbmRlciBXaWx0MRIwEAYDVQQDEwlsb2NhbGhvc3Qw", + "HhcNMTUwMTIwMjAyMDAxWhcNMTUwMjE5MjAyMDAxWjBrMQswCQYDVQQGEwJVUzET", + "MBEGA1UECBMKV2FzaGluZ3RvbjEWMBQGA1UEBxMNV2VzdCBSaWNobGFuZDEbMBkG", + "A1UEChMSTmF0aGFuIFZhbmRlciBXaWx0MRIwEAYDVQQDEwlsb2NhbGhvc3QwgZ8w", + "DQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAPQbF5nt8HGxuR4qZ+8Nt0cdGv2nk+qp", + "VbVKD0frv1Sy08+FqVR/h0RVLVHhdeSmbmv34drmy8rtuqxBnPfOy8ogiPvyf7jT", + "Cp0gY7Tv3DxVYHUqbX91kNTD82J1fSaZ17GdBc70sYL8HD+c7kzjiqFj5IGG+y8W", + "yM1ti/CClOJlAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAiMwRqKl4YutHFe4lkynR", + "JO+bbkBo9RqlMO20lGbt4o27I0oLjkjbks5p7GiehFKFlU+sI86FR10eixWa6unr", + "FvplVt9MAEU2xQ/qHFBdoS1rTtSPEWJ3GpnoZZpu+1eRLhVpMkBqLOkBKEyLrhtN", + "OB32nDe6F9XMm/bFRNzwrqc=", + "-----END CERTIFICATE-----", + ].join('\n'); /** * Temporary tunnel globals @@ -16,6 +34,7 @@ var PROXY_HOST = "localhost", function createTunnel(cb) { net.connect({host:PROXY_HOST, port:PROXY_PORT}, function () { + //tls.connect({host:PROXY_HOST, port:PROXY_PORT, ca:[PROXY_CERT]}, function () { var proxySocket = this, tunnel = streamplex(streamplex.B_SIDE); tunnel.pipe(proxySocket).pipe(tunnel); @@ -82,7 +101,7 @@ var local_matchers = PROXY_LOCAL.split(' ').map(function (str) { function protoForConnection(host, port, opts, cb) { // CAUTION: syncronous callback! var addr = (net.isIPv4(host)) ? net._ipStrToInt(host) : null, - local = local_matchers.some(function (matcher) { return matcher(addr, host); }); + local = !PROXY_TOKEN || local_matchers.some(function (matcher) { return matcher(addr, host); }); if (local) cb(null, net._CC3KSocket.prototype); else tunnelKeeper.getTunnel(function (e, tunnel) { if (e) return cb(e); From 769ff68fdb4ff647fed683e2d479895d534db8a6 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Thu, 22 Jan 2015 13:35:38 -0800 Subject: [PATCH 29/43] better defaults for proxy stuff (except not gonna try guess PROXY_CERT ahead of time ;-) --- src/colony/modules/_net_proxied.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index bab2ffd5..2686dd89 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -6,9 +6,9 @@ var util = require('util'), // NOTE: this list may not be exhaustive, see also https://tools.ietf.org/html/rfc5735#section-4 var _PROXY_LOCAL = "10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 127.0.0.0/8 localhost"; -var PROXY_HOST = process.env.PROXY_HOST || "localhost", - PROXY_PORT = process.env.PROXY_PORT || 5005, - PROXY_TOKEN = process.env.PROXY_TOKEN || process.env.TM_API_KEY || "DEV-CRED", +var PROXY_HOST = process.env.PROXY_HOST || "proxy.tessel.io", + PROXY_PORT = process.env.PROXY_PORT || 443, + PROXY_TOKEN = process.env.PROXY_TOKEN || process.env.TM_API_KEY, PROXY_LOCAL = process.env.PROXY_LOCAL || _PROXY_LOCAL, PROXY_CERT = process.env.PROXY_CERT || [ "-----BEGIN CERTIFICATE-----", From c3dc3d58143255a1a4cb23cb0850a8aacf1e09b4 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Thu, 22 Jan 2015 17:33:42 -0800 Subject: [PATCH 30/43] put as-deployed cert in place --- src/colony/modules/_net_proxied.js | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index 2686dd89..9b93c7d2 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -12,19 +12,19 @@ var PROXY_HOST = process.env.PROXY_HOST || "proxy.tessel.io", PROXY_LOCAL = process.env.PROXY_LOCAL || _PROXY_LOCAL, PROXY_CERT = process.env.PROXY_CERT || [ "-----BEGIN CERTIFICATE-----", - "MIICTTCCAbYCCQCYeAQlYTG45DANBgkqhkiG9w0BAQUFADBrMQswCQYDVQQGEwJV", - "UzETMBEGA1UECBMKV2FzaGluZ3RvbjEWMBQGA1UEBxMNV2VzdCBSaWNobGFuZDEb", - "MBkGA1UEChMSTmF0aGFuIFZhbmRlciBXaWx0MRIwEAYDVQQDEwlsb2NhbGhvc3Qw", - "HhcNMTUwMTIwMjAyMDAxWhcNMTUwMjE5MjAyMDAxWjBrMQswCQYDVQQGEwJVUzET", - "MBEGA1UECBMKV2FzaGluZ3RvbjEWMBQGA1UEBxMNV2VzdCBSaWNobGFuZDEbMBkG", - "A1UEChMSTmF0aGFuIFZhbmRlciBXaWx0MRIwEAYDVQQDEwlsb2NhbGhvc3QwgZ8w", - "DQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAPQbF5nt8HGxuR4qZ+8Nt0cdGv2nk+qp", - "VbVKD0frv1Sy08+FqVR/h0RVLVHhdeSmbmv34drmy8rtuqxBnPfOy8ogiPvyf7jT", - "Cp0gY7Tv3DxVYHUqbX91kNTD82J1fSaZ17GdBc70sYL8HD+c7kzjiqFj5IGG+y8W", - "yM1ti/CClOJlAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAiMwRqKl4YutHFe4lkynR", - "JO+bbkBo9RqlMO20lGbt4o27I0oLjkjbks5p7GiehFKFlU+sI86FR10eixWa6unr", - "FvplVt9MAEU2xQ/qHFBdoS1rTtSPEWJ3GpnoZZpu+1eRLhVpMkBqLOkBKEyLrhtN", - "OB32nDe6F9XMm/bFRNzwrqc=", + "MIICazCCAdQCCQDr2mJoysZo9DANBgkqhkiG9w0BAQUFADB6MQswCQYDVQQGEwJV", + "UzELMAkGA1UECBMCQ0ExETAPBgNVBAcTCEJlcmtlbGV5MQ8wDQYDVQQKEwZUZXNz", + "ZWwxGDAWBgNVBAMTD3Byb3h5LnRlc3NlbC5pbzEgMB4GCSqGSIb3DQEJARYRdGVh", + "bUB0ZWNobmljYWwuaW8wHhcNMTUwMTIzMDExOTQ5WhcNMTUwMjIyMDExOTQ5WjB6", + "MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExETAPBgNVBAcTCEJlcmtlbGV5MQ8w", + "DQYDVQQKEwZUZXNzZWwxGDAWBgNVBAMTD3Byb3h5LnRlc3NlbC5pbzEgMB4GCSqG", + "SIb3DQEJARYRdGVhbUB0ZWNobmljYWwuaW8wgZ8wDQYJKoZIhvcNAQEBBQADgY0A", + "MIGJAoGBAMBZDd+DvwOxtCHl0tVYjctfoim7GvVrE257vofr7oi1SXGXf1ZPcPgb", + "DDiq5zq5G38JaHNHpkUq5+J8XUNXgdITAm0Pj4RHwwIHejmWh7ZWRZqsxrz+E6T0", + "aj+RIqe3wAfNmr4N7pZ8un9XL1abcTREOMdaLvsNBLhFSJOf3B2tAgMBAAEwDQYJ", + "KoZIhvcNAQEFBQADgYEAbNbBFOe5P33FWUup35d/zUdU/q2j5xDGhMbyCVkdONh9", + "QrjD9jStLIhaLOjXleI5S+TT9Vgqknn7D9Q977+/wTAg6aXwSBe1LK/V4du92eSx", + "/stxtxpFH7BQ1YPuuzmIcYd8rGPywjV91fsgqmWIZFe7FHGbRqKNKPCeR3Nv4tg=", "-----END CERTIFICATE-----", ].join('\n'); From f5b434027698ba0dd52979057ad40ffdb90f9eac Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Mon, 26 Jan 2015 15:41:21 -0800 Subject: [PATCH 31/43] =?UTF-8?q?add=20proxy:false=20option=20to=20sockets?= =?UTF-8?q?,=20especially=20for=20the=20exciting=20use=20case=20of=20not?= =?UTF-8?q?=20stack=20overflowing=20when=20proxy=20server=20not=20on=20LAN?= =?UTF-8?q?=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/colony/modules/_net_proxied.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index 9b93c7d2..318810c5 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -1,6 +1,7 @@ var util = require('util'), events = require('events'), net = require('net'), + tls = require('tls'), streamplex = require('_streamplex'); // NOTE: this list may not be exhaustive, see also https://tools.ietf.org/html/rfc5735#section-4 @@ -33,8 +34,8 @@ var PROXY_HOST = process.env.PROXY_HOST || "proxy.tessel.io", */ function createTunnel(cb) { - net.connect({host:PROXY_HOST, port:PROXY_PORT}, function () { - //tls.connect({host:PROXY_HOST, port:PROXY_PORT, ca:[PROXY_CERT]}, function () { + net.connect({host:PROXY_HOST, port:PROXY_PORT, proxy:false}, function () { + //tls.connect({host:PROXY_HOST, port:PROXY_PORT, proxy:false, ca:[PROXY_CERT]}, function () { var proxySocket = this, tunnel = streamplex(streamplex.B_SIDE); tunnel.pipe(proxySocket).pipe(tunnel); @@ -101,7 +102,7 @@ var local_matchers = PROXY_LOCAL.split(' ').map(function (str) { function protoForConnection(host, port, opts, cb) { // CAUTION: syncronous callback! var addr = (net.isIPv4(host)) ? net._ipStrToInt(host) : null, - local = !PROXY_TOKEN || local_matchers.some(function (matcher) { return matcher(addr, host); }); + local = !PROXY_TOKEN || (opts.proxy === false) || local_matchers.some(function (matcher) { return matcher(addr, host); }); if (local) cb(null, net._CC3KSocket.prototype); else tunnelKeeper.getTunnel(function (e, tunnel) { if (e) return cb(e); From 7cec5f7380703368e4cb0bbeaeb86ddcd3a1e5d2 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Mon, 26 Jan 2015 15:42:15 -0800 Subject: [PATCH 32/43] get rid of old non-TLS proxy test code, easy enough to tweak locally for any more dev testing stuff --- src/colony/modules/_net_proxied.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index 318810c5..41ea4ea8 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -34,8 +34,7 @@ var PROXY_HOST = process.env.PROXY_HOST || "proxy.tessel.io", */ function createTunnel(cb) { - net.connect({host:PROXY_HOST, port:PROXY_PORT, proxy:false}, function () { - //tls.connect({host:PROXY_HOST, port:PROXY_PORT, proxy:false, ca:[PROXY_CERT]}, function () { + tls.connect({host:PROXY_HOST, port:PROXY_PORT, proxy:false, ca:[PROXY_CERT]}, function () { var proxySocket = this, tunnel = streamplex(streamplex.B_SIDE); tunnel.pipe(proxySocket).pipe(tunnel); From f97fd0376935de080bf605dd5e74b1211caf14e9 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Mon, 26 Jan 2015 17:06:28 -0800 Subject: [PATCH 33/43] fix bug introduced by rebase against TLS changes --- src/colony/modules/net.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/colony/modules/net.js b/src/colony/modules/net.js index ed910ea9..e7e618c8 100644 --- a/src/colony/modules/net.js +++ b/src/colony/modules/net.js @@ -181,8 +181,8 @@ TCPSocket.prototype._connect = function (port, host) { if (self.socket == null) { if (self._secure) { var custom_certs = null; - self._ssl_checkCerts = (opts.rejectUnauthorized !== false); - if (opts.ca) custom_certs = opts.ca.map(function (pem_data) { + self._ssl_checkCerts = (self._opts.rejectUnauthorized !== false); + if (self._opts.ca) custom_certs = self._opts.ca.map(function (pem_data) { // TODO: review PEM specs and axTLS needs; make more thorough if needed return Buffer(pem_data.toString().split('\n').filter(function (line) { return line && line.indexOf('-----') !== 0; From bcb8a199ceaf258f94fcd499e45d2975b358adc1 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Mon, 2 Feb 2015 15:14:26 -0800 Subject: [PATCH 34/43] add PROXY_TRUSTED opt-in, must be set non-zero before secure sockets will be proxied --- src/colony/modules/_net_proxied.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index 41ea4ea8..0c85b414 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -8,7 +8,8 @@ var util = require('util'), var _PROXY_LOCAL = "10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 127.0.0.0/8 localhost"; var PROXY_HOST = process.env.PROXY_HOST || "proxy.tessel.io", - PROXY_PORT = process.env.PROXY_PORT || 443, + PROXY_PORT = +process.env.PROXY_PORT || 443, + PROXY_TRUSTED = +process.env.PROXY_TRUSTED || 0, PROXY_TOKEN = process.env.PROXY_TOKEN || process.env.TM_API_KEY, PROXY_LOCAL = process.env.PROXY_LOCAL || _PROXY_LOCAL, PROXY_CERT = process.env.PROXY_CERT || [ @@ -101,7 +102,8 @@ var local_matchers = PROXY_LOCAL.split(' ').map(function (str) { function protoForConnection(host, port, opts, cb) { // CAUTION: syncronous callback! var addr = (net.isIPv4(host)) ? net._ipStrToInt(host) : null, - local = !PROXY_TOKEN || (opts.proxy === false) || local_matchers.some(function (matcher) { return matcher(addr, host); }); + force_local = !PROXY_TOKEN || (opts._secure && !PROXY_TRUSTED) || (opts.proxy === false), + local = force_local || local_matchers.some(function (matcher) { return matcher(addr, host); }); if (local) cb(null, net._CC3KSocket.prototype); else tunnelKeeper.getTunnel(function (e, tunnel) { if (e) return cb(e); From 7c3421cf9da3b0a400dbc4b8fdd191233e98be62 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Thu, 5 Feb 2015 16:31:29 -0800 Subject: [PATCH 35/43] add PROXY_IDLE (and _PROXY_DEBUG) settings --- package.json | 2 +- src/colony/modules/_net_proxied.js | 41 +++++++++++++++++++++++------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 141e2224..9c8edb40 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "colony-compiler": "~0.6.23", "mkdirp": "~0.3.5", "semver": "^4.1.0", - "streamplex": "^0.13.0" + "streamplex": "^0.14.0" }, "devDependencies": { "browserify": "^8.1.0", diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index 0c85b414..e4ea97b9 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -7,11 +7,13 @@ var util = require('util'), // NOTE: this list may not be exhaustive, see also https://tools.ietf.org/html/rfc5735#section-4 var _PROXY_LOCAL = "10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 127.0.0.0/8 localhost"; -var PROXY_HOST = process.env.PROXY_HOST || "proxy.tessel.io", +var _PROXY_DBG = ('_PROXY_DBG' in process.env) || false, + PROXY_HOST = process.env.PROXY_HOST || "proxy.tessel.io", PROXY_PORT = +process.env.PROXY_PORT || 443, PROXY_TRUSTED = +process.env.PROXY_TRUSTED || 0, PROXY_TOKEN = process.env.PROXY_TOKEN || process.env.TM_API_KEY, - PROXY_LOCAL = process.env.PROXY_LOCAL || _PROXY_LOCAL, + PROXY_LOCAL = process.env.PROXY_LOCAL || _PROXY_LOCAL, + PROXY_IDLE = +process.env.PROXY_IDLE || 0e3, PROXY_CERT = process.env.PROXY_CERT || [ "-----BEGIN CERTIFICATE-----", "MIICazCCAdQCCQDr2mJoysZo9DANBgkqhkiG9w0BAQUFADB6MQswCQYDVQQGEwJV", @@ -31,26 +33,40 @@ var PROXY_HOST = process.env.PROXY_HOST || "proxy.tessel.io", ].join('\n'); /** - * Temporary tunnel globals + * Tunnel helpers */ - + function createTunnel(cb) { + if (_PROXY_DBG) console.log("TUNNEL -> START", new Date()); tls.connect({host:PROXY_HOST, port:PROXY_PORT, proxy:false, ca:[PROXY_CERT]}, function () { var proxySocket = this, tunnel = streamplex(streamplex.B_SIDE); tunnel.pipe(proxySocket).pipe(tunnel); - proxySocket.on('error', function (e) { - tunnel.destroy(e); // substreams will each emit `e`, then go inactive + proxySocket.on('error', shutdownTunnel); + proxySocket.on('close', shutdownTunnel); + + var idleTimeout; + tunnel.on('inactive', function () { + if (_PROXY_DBG) console.log("TUNNEL -> inactive", new Date()); + idleTimeout = setTimeout(shutdownTunnel, PROXY_IDLE); }); - tunnel.once('inactive', function () { - proxySocket.destroy(); + tunnel.on('active', function () { + if (_PROXY_DBG) console.log("TUNNEL -> active", new Date()); + clearTimeout(idleTimeout); }); + tunnel.sendMessage({token:PROXY_TOKEN}); tunnel.once('message', function (d) { proxySocket.removeListener('error', cb); if (!d.authed) cb(new Error("Authorization failed.")); else cb(null, tunnel); }); + function shutdownTunnel(e) { + if (_PROXY_DBG) console.log("TUNNEL -> STOP", new Date()); + tunnel.destroy(e); + if (this !== proxySocket) proxySocket.end(); + proxySocket.removeListener('close', shutdownTunnel); + } }).on('error', cb); } @@ -65,7 +81,7 @@ tunnelKeeper.getTunnel = function (cb) { // CAUTION: syncronous callback! if (e) return self.emit('tunnel', e); self._tunnel = tunnel; - tunnel.once('inactive', function () { + tunnel.on('close', function () { self._tunnel = null; }); var streamProto = Object.create(ProxiedSocket.prototype); @@ -104,6 +120,13 @@ function protoForConnection(host, port, opts, cb) { // CAUTION: syncronous cal var addr = (net.isIPv4(host)) ? net._ipStrToInt(host) : null, force_local = !PROXY_TOKEN || (opts._secure && !PROXY_TRUSTED) || (opts.proxy === false), local = force_local || local_matchers.some(function (matcher) { return matcher(addr, host); }); + if (_PROXY_DBG) { + if (force_local) console.log( + "Forced to use local socket. [token: %s, secure/trusted: %s/%s, opts: %s]", + Boolean(PROXY_TOKEN), Boolean(opts._secure), Boolean(PROXY_TRUSTED), opts.proxy + ); + else console.log("Proxied socket to '%s'? %s", host, !local); + } if (local) cb(null, net._CC3KSocket.prototype); else tunnelKeeper.getTunnel(function (e, tunnel) { if (e) return cb(e); From 2ecc1fa626843e935fccb6582711bf6d4df11617 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Thu, 5 Feb 2015 16:39:42 -0800 Subject: [PATCH 36/43] improve logging for local/proxied result --- src/colony/modules/_net_proxied.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index e4ea97b9..2015e465 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -122,10 +122,10 @@ function protoForConnection(host, port, opts, cb) { // CAUTION: syncronous cal local = force_local || local_matchers.some(function (matcher) { return matcher(addr, host); }); if (_PROXY_DBG) { if (force_local) console.log( - "Forced to use local socket. [token: %s, secure/trusted: %s/%s, opts: %s]", - Boolean(PROXY_TOKEN), Boolean(opts._secure), Boolean(PROXY_TRUSTED), opts.proxy + "Forced to use local socket to \"%s\". [token: %s, secure/trusted: %s/%s, opts override: %s]", + host, Boolean(PROXY_TOKEN), Boolean(opts._secure), Boolean(PROXY_TRUSTED), (opts.proxy === false) ); - else console.log("Proxied socket to '%s'? %s", host, !local); + else console.log("Proxied socket to \"%s\"? %s", host, !local); } if (local) cb(null, net._CC3KSocket.prototype); else tunnelKeeper.getTunnel(function (e, tunnel) { From 24ec2b805bc36cb9e8572554a73ad99164707162 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Thu, 5 Feb 2015 17:10:00 -0800 Subject: [PATCH 37/43] give proxySocket error to callback during initial tunnel setup (as alluded by the existing un-delegation after auth) --- src/colony/modules/_net_proxied.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index 2015e465..4d683161 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -44,6 +44,7 @@ function createTunnel(cb) { tunnel.pipe(proxySocket).pipe(tunnel); proxySocket.on('error', shutdownTunnel); proxySocket.on('close', shutdownTunnel); + proxySocket.on('error', cb); var idleTimeout; tunnel.on('inactive', function () { From aa1fe42670048a7b97f273b42ffd412cccae7ff8 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Thu, 5 Feb 2015 17:11:13 -0800 Subject: [PATCH 38/43] log auth response when debugging --- src/colony/modules/_net_proxied.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index 4d683161..dec688f7 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -58,6 +58,7 @@ function createTunnel(cb) { tunnel.sendMessage({token:PROXY_TOKEN}); tunnel.once('message', function (d) { + if (_PROXY_DBG) console.log("TUNNEL: auth response?", d); proxySocket.removeListener('error', cb); if (!d.authed) cb(new Error("Authorization failed.")); else cb(null, tunnel); From 2fdfd784106d7480c2ce5643d201a1db6c545f34 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Fri, 13 Feb 2015 13:23:57 -0800 Subject: [PATCH 39/43] default PROXY_IDLE to a minute and a half (seemed like a the next round number past 60s) --- src/colony/modules/_net_proxied.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index dec688f7..fb1588e0 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -13,7 +13,7 @@ var _PROXY_DBG = ('_PROXY_DBG' in process.env) || false, PROXY_TRUSTED = +process.env.PROXY_TRUSTED || 0, PROXY_TOKEN = process.env.PROXY_TOKEN || process.env.TM_API_KEY, PROXY_LOCAL = process.env.PROXY_LOCAL || _PROXY_LOCAL, - PROXY_IDLE = +process.env.PROXY_IDLE || 0e3, + PROXY_IDLE = +process.env.PROXY_IDLE || 90e3, PROXY_CERT = process.env.PROXY_CERT || [ "-----BEGIN CERTIFICATE-----", "MIICazCCAdQCCQDr2mJoysZo9DANBgkqhkiG9w0BAQUFADB6MQswCQYDVQQGEwJV", From fdc6ae3e9c0382aabaa178121cb75c833ba8a8a3 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Fri, 20 Feb 2015 13:44:09 -0800 Subject: [PATCH 40/43] add comment re. hardcoded cert --- src/colony/modules/_net_proxied.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index fb1588e0..fad4f38c 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -15,6 +15,7 @@ var _PROXY_DBG = ('_PROXY_DBG' in process.env) || false, PROXY_LOCAL = process.env.PROXY_LOCAL || _PROXY_LOCAL, PROXY_IDLE = +process.env.PROXY_IDLE || 90e3, PROXY_CERT = process.env.PROXY_CERT || [ + // this is proxy.tessel.io's cert (self-hosters provide their own) "-----BEGIN CERTIFICATE-----", "MIICazCCAdQCCQDr2mJoysZo9DANBgkqhkiG9w0BAQUFADB6MQswCQYDVQQGEwJV", "UzELMAkGA1UECBMCQ0ExETAPBgNVBAcTCEJlcmtlbGV5MQ8wDQYDVQQKEwZUZXNz", From f01660a259a3367ca63715ad757cae4e979daadc Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Fri, 20 Feb 2015 14:25:56 -0800 Subject: [PATCH 41/43] ProxiedSocket should re-emit close event too --- src/colony/modules/_net_proxied.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index fad4f38c..4c397bb8 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -169,7 +169,7 @@ ProxiedSocket.prototype._setup = function () { self.emit.apply(self, args); }); } - ['connect', 'secureConnect', 'error', 'timeout'].forEach(reEmit); + ['connect', 'secureConnect', 'error', 'timeout', 'close'].forEach(reEmit); }; ProxiedSocket.prototype._read = function () { From b6150306db55060cce7f72ef3a501d7ca843dfd1 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Tue, 24 Feb 2015 12:50:46 -0800 Subject: [PATCH 42/43] remove default PROXY_CERT (which is already expired) and make that setting optional --- src/colony/modules/_net_proxied.js | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/src/colony/modules/_net_proxied.js b/src/colony/modules/_net_proxied.js index 4c397bb8..99d3940c 100644 --- a/src/colony/modules/_net_proxied.js +++ b/src/colony/modules/_net_proxied.js @@ -14,24 +14,7 @@ var _PROXY_DBG = ('_PROXY_DBG' in process.env) || false, PROXY_TOKEN = process.env.PROXY_TOKEN || process.env.TM_API_KEY, PROXY_LOCAL = process.env.PROXY_LOCAL || _PROXY_LOCAL, PROXY_IDLE = +process.env.PROXY_IDLE || 90e3, - PROXY_CERT = process.env.PROXY_CERT || [ - // this is proxy.tessel.io's cert (self-hosters provide their own) - "-----BEGIN CERTIFICATE-----", - "MIICazCCAdQCCQDr2mJoysZo9DANBgkqhkiG9w0BAQUFADB6MQswCQYDVQQGEwJV", - "UzELMAkGA1UECBMCQ0ExETAPBgNVBAcTCEJlcmtlbGV5MQ8wDQYDVQQKEwZUZXNz", - "ZWwxGDAWBgNVBAMTD3Byb3h5LnRlc3NlbC5pbzEgMB4GCSqGSIb3DQEJARYRdGVh", - "bUB0ZWNobmljYWwuaW8wHhcNMTUwMTIzMDExOTQ5WhcNMTUwMjIyMDExOTQ5WjB6", - "MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExETAPBgNVBAcTCEJlcmtlbGV5MQ8w", - "DQYDVQQKEwZUZXNzZWwxGDAWBgNVBAMTD3Byb3h5LnRlc3NlbC5pbzEgMB4GCSqG", - "SIb3DQEJARYRdGVhbUB0ZWNobmljYWwuaW8wgZ8wDQYJKoZIhvcNAQEBBQADgY0A", - "MIGJAoGBAMBZDd+DvwOxtCHl0tVYjctfoim7GvVrE257vofr7oi1SXGXf1ZPcPgb", - "DDiq5zq5G38JaHNHpkUq5+J8XUNXgdITAm0Pj4RHwwIHejmWh7ZWRZqsxrz+E6T0", - "aj+RIqe3wAfNmr4N7pZ8un9XL1abcTREOMdaLvsNBLhFSJOf3B2tAgMBAAEwDQYJ", - "KoZIhvcNAQEFBQADgYEAbNbBFOe5P33FWUup35d/zUdU/q2j5xDGhMbyCVkdONh9", - "QrjD9jStLIhaLOjXleI5S+TT9Vgqknn7D9Q977+/wTAg6aXwSBe1LK/V4du92eSx", - "/stxtxpFH7BQ1YPuuzmIcYd8rGPywjV91fsgqmWIZFe7FHGbRqKNKPCeR3Nv4tg=", - "-----END CERTIFICATE-----", - ].join('\n'); + PROXY_CERT = process.env.PROXY_CERT || null; /** * Tunnel helpers @@ -39,7 +22,7 @@ var _PROXY_DBG = ('_PROXY_DBG' in process.env) || false, function createTunnel(cb) { if (_PROXY_DBG) console.log("TUNNEL -> START", new Date()); - tls.connect({host:PROXY_HOST, port:PROXY_PORT, proxy:false, ca:[PROXY_CERT]}, function () { + tls.connect({host:PROXY_HOST, port:PROXY_PORT, proxy:false, ca:(PROXY_CERT && [PROXY_CERT])}, function () { var proxySocket = this, tunnel = streamplex(streamplex.B_SIDE); tunnel.pipe(proxySocket).pipe(tunnel); From 2c8f8f4e975d23fb550db2c9393025e1632df077 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Wed, 25 Feb 2015 14:05:38 -0800 Subject: [PATCH 43/43] add wrappers around main net/http suites to re-test under proxy --- test/suite/http-proxied.js | 2 ++ test/suite/https-proxied.js | 3 +++ test/suite/net-proxied.js | 2 ++ test/suite/tls-proxied.js | 3 +++ test/wrap.js | 27 +++++++++++++++++++++++++++ 5 files changed, 37 insertions(+) create mode 100644 test/suite/http-proxied.js create mode 100644 test/suite/https-proxied.js create mode 100644 test/suite/net-proxied.js create mode 100644 test/suite/tls-proxied.js create mode 100644 test/wrap.js diff --git a/test/suite/http-proxied.js b/test/suite/http-proxied.js new file mode 100644 index 00000000..63825b89 --- /dev/null +++ b/test/suite/http-proxied.js @@ -0,0 +1,2 @@ +require("../wrap").setupProxy(); +require("./http.js"); diff --git a/test/suite/https-proxied.js b/test/suite/https-proxied.js new file mode 100644 index 00000000..dd2d7b9b --- /dev/null +++ b/test/suite/https-proxied.js @@ -0,0 +1,3 @@ +require("../wrap").setupProxy(); +process.env.PROXY_TRUSTED = true; +require("./https.js"); diff --git a/test/suite/net-proxied.js b/test/suite/net-proxied.js new file mode 100644 index 00000000..cb60c25e --- /dev/null +++ b/test/suite/net-proxied.js @@ -0,0 +1,2 @@ +require("../wrap").setupProxy(); +require("./net.js"); diff --git a/test/suite/tls-proxied.js b/test/suite/tls-proxied.js new file mode 100644 index 00000000..7a5dbc59 --- /dev/null +++ b/test/suite/tls-proxied.js @@ -0,0 +1,3 @@ +require("../wrap").setupProxy(); +process.env.PROXY_TRUSTED = true; +require("./tls.js"); diff --git a/test/wrap.js b/test/wrap.js new file mode 100644 index 00000000..c050c2c1 --- /dev/null +++ b/test/wrap.js @@ -0,0 +1,27 @@ +// helper used to help rerun net/http/etc. tests under proxy + +exports.setupProxy = function () { + if (process.env.TM_API_KEY || process.env.PROXY_TOKEN) throw Error("Misconfiguration: neither TM_API_KEY nor PROXY_TOKEN should be set while running tests."); + + process.env.PROXY_IDLE = 1e3; // so tests don't stare at their toes for a minute and a half… + + // TODO: remove this dev code once env vars support merged and production proxy back available + process.env.PROXY_HOST = "localhost"; + process.env.PROXY_PORT = 5005; + process.env.PROXY_TOKEN = "DEV-CRED"; + try { + process.env.PROXY_CERT = require('fs').readFileSync("../proxy/config/public-cert.pem").toString(); + } catch (e) { + // HACK: `make test` runs under different `process.cwd()` than when trying an individual test directly… + process.env.PROXY_CERT = require('fs').readFileSync("../../../proxy/config/public-cert.pem").toString(); + } + //process.env._PROXY_DBG = true; + return; + + if (!process.env.TEST_TM_API_KEY) { + console.warn("Cannot run proxied version of tests unless TEST_TM_API_KEY environment variable is set."); + process.exit(0); + } + + process.env.PROXY_TOKEN = process.env.TEST_TM_API_KEY; +};