From 3f7df4ec5c79d3d4cef626e1f30758e543de6f97 Mon Sep 17 00:00:00 2001 From: Vitaliy Zhaborovskyy Date: Wed, 23 Oct 2019 13:45:49 +0300 Subject: [PATCH 1/8] 1. direct openssl call via Pipe removed. all ssl magic does via pyopenssl lib api. After this fix proxy works correct in any platform with python and pyopenssl lib without any additional fixes. 2. subjectAltName ssl extension support added. this fixed ERR_CERT_COMMON_NAME_INVALID error --- proxy2.py | 38 ++++++++-- ssl_wrapper.py | 194 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+), 8 deletions(-) create mode 100644 ssl_wrapper.py diff --git a/proxy2.py b/proxy2.py index f502e8c..2534745 100644 --- a/proxy2.py +++ b/proxy2.py @@ -10,7 +10,9 @@ import time import json import re -from subprocess import Popen, PIPE +from ssl_wrapper import * +from string import Template +from OpenSSL import crypto try: import http.client as httplib @@ -70,26 +72,44 @@ def log_error(self, format, *args): self.log_message(format, *args) def do_CONNECT(self): - if os.path.isfile(self.cakey) and os.path.isfile(self.cacert) and os.path.isfile(self.certkey) and os.path.isdir(self.certdir): + if ca_files_exist(): self.connect_intercept() else: self.connect_relay() def connect_intercept(self): hostname = self.path.split(':')[0] - certpath = "{}/{}.crt".format(self.certdir.rstrip('/'), hostname) + ippat = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") + cert_category = "DNS" + if ippat.match(hostname): + cert_category = "IP" + + certpath = "%s/%s.crt" % (cert_dir.rstrip('/'), hostname) with self.lock: if not os.path.isfile(certpath): - epoch = str(int(time.time() * 1000)) - p1 = Popen(["openssl", "req", "-new", "-key", self.certkey, "-subj", "/CN={}".format(hostname)], stdout=PIPE) - p2 = Popen(["openssl", "x509", "-req", "-days", "3650", "-CA", self.cacert, "-CAkey", self.cakey, "-set_serial", epoch, "-out", certpath], stdin=p1.stdout, stderr=PIPE) - p2.communicate() + x509_serial = int("%d" % (time.time() * 1000)) + valid_time_interval = (0, 60 * 60 * 24 * 365) + cert_request = create_cert_request(cert_key_obj, CN=hostname) + cert = create_certificate( + cert_request, (ca_crt_obj, ca_key_obj), x509_serial, + valid_time_interval, + subject_alt_names=[ + Template("${category}:${hostname}").substitute(hostname=hostname, category=cert_category) + ] + ) + with open(certpath, 'wb+') as f: + f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert)) + self.wfile.write("{} {} {}\r\n".format(self.protocol_version, 200, 'Connection Established').encode('latin_1')) self.wfile.write(b'\r\n') - self.connection = ssl.wrap_socket(self.connection, keyfile=self.certkey, certfile=certpath, server_side=True) + self.connection = ssl.wrap_socket(self.connection, + keyfile=cert_key, + certfile=certpath, + server_side=True) + self.rfile = self.connection.makefile("rb", self.rbufsize) self.wfile = self.connection.makefile("wb", self.wbufsize) @@ -271,6 +291,8 @@ def decode_content_body(self, data, encoding): text = zlib.decompress(data) except zlib.error: text = zlib.decompress(data, -zlib.MAX_WBITS) + elif encoding == 'br': #Brotli + return data else: raise Exception("Unknown Content-Encoding: {}".format(encoding)) return text diff --git a/ssl_wrapper.py b/ssl_wrapper.py new file mode 100644 index 0000000..e23e8d7 --- /dev/null +++ b/ssl_wrapper.py @@ -0,0 +1,194 @@ +""" +Provides x509 certificates and paths. +""" + +from os import mkdir +from os.path import abspath, dirname, isdir, isfile, join +import OpenSSL.crypto as crypto + + +proxy_CN = 'proxy2 CA' + +# TODO: do this on package-install-time after move to pyopenssl +dir_name = join(dirname(abspath(__file__)), 'ssl-data') + +ca_key = join(dir_name, 'ca.key') +ca_crt = join(dir_name, 'ca.crt') +cert_key = join(dir_name, 'cert.key') +cert_dir = join(dir_name, 'certs') + +def generate_key_pair(key_type, bits): + """ + Creates key pair + :param key_type: one of crypto.TYPE_RSA or crypto.TYPE_DSA + :param bits: key length + :return: key pair in a PKey object + :return type: instance of crypto.PKey + """ + pkey = crypto.PKey() + pkey.generate_key(key_type, bits) + return pkey + + +def create_cert_request(p_key, digest="sha256", **subject_kwargs): + """ + Creates certificate request + :param p_key: key to associate with the request + :param digest: signing method + :param subject_kwargs: subject of request + valuable args are: (took from RFC 5280) + C: country + ST: state or province name + L: Locality name + O: organization + OU: organizational unit + CN: common name (e.g., "Susan Housley") + emailAddress: e-mail + :return: certificate request + """ + req = crypto.X509Req() + subj = req.get_subject() + + for key, value in subject_kwargs.items(): + setattr(subj, key, value) + + req.set_pubkey(p_key) + req.sign(p_key, digest) + return req + + +def create_certificate( + req, cert_key_pair, serial, begin_end_validity, digest="sha256", + self_signed_x509v3=False, subject_alt_names=[]): + """ + Create certificate by certificate request. + :param req: certificate request + :param cert_key_pair: tuple with issuer certificate and private key + :param serial: serial number + :param begin_end_validity: tuple with seconds certificate validity. + 0 means now. Example for set one year valid certificate from now: + begin_end_validity=(0, 60*60*24*365) + :param digest: signing method + :param self_signed_x509v3: generate self signed x509v3 CA certificate, add + extensions similar to these: + X509v3 extensions: + X509v3 Subject Key Identifier: + 88:31:6A:B7:8C:B3:F0:1D:5F:CD:9F:F8:70:F7:D4:7C:E5:5E:D2:A1 + X509v3 Authority Key Identifier: + keyid:88:31:6A:B7:8C:B3:F0:1D:5F:CD:9F:F8:70:F7:D4:7C:E5:5E:D2:A1 + X509v3 Basic Constraints: + CA:TRUE + :param subject_alt_names: subject alt names e.g. IP:192.168.7.1 or DNS:my.domain + :return: signed certificate + :return type: crypto.X509 + """ + i_cert, i_key = cert_key_pair + not_before, not_after = begin_end_validity + ret_x509_obj = crypto.X509() + ret_x509_obj.set_serial_number(serial) + + ret_x509_obj.gmtime_adj_notAfter(not_after) + ret_x509_obj.gmtime_adj_notBefore(not_before) + if i_cert == '__self_signed': + i_cert = ret_x509_obj + ret_x509_obj.set_issuer(i_cert.get_subject()) + ret_x509_obj.set_subject(req.get_subject()) + ret_x509_obj.set_pubkey(req.get_pubkey()) + if self_signed_x509v3: + ret_x509_obj.set_version(2) + ret_x509_obj.add_extensions([ + crypto.X509Extension("subjectKeyIdentifier", False, "hash", + subject=ret_x509_obj), + crypto.X509Extension("basicConstraints", False, "CA:TRUE"), + ]) + ret_x509_obj.add_extensions([ + crypto.X509Extension("authorityKeyIdentifier", False, + "keyid:always", issuer=ret_x509_obj), + ]) + if len(subject_alt_names) != 0: + ret_x509_obj.set_version(2) # 0x3 + ret_x509_obj.add_extensions([ + crypto.X509Extension( + type_name=b'subjectAltName', + critical=False, + value=", ".join(subject_alt_names).encode()) + ]) + + ret_x509_obj.sign(i_key, digest) + return ret_x509_obj + + +def ca_files_exist(): + return all( + [map(isfile, [ca_key, ca_crt, cert_key]), isdir(dir_name)]) + +if not ca_files_exist(): + # TODO: move this code to pyopenssl library + try: + if not isdir(dir_name): + mkdir(dir_name) + ca_key_o = generate_key_pair(crypto.TYPE_RSA, 2048) + cert_key_o = generate_key_pair(crypto.TYPE_RSA, 2048) + cert_req_temp = create_cert_request(ca_key_o, CN=proxy_CN) + ca_crt_o = create_certificate( + cert_req_temp, ('__self_signed', ca_key_o), 1509982490957715, + (0, 60 * 60 * 24 * 30), self_signed_x509v3=True + ) + with open(ca_key, 'w+') as f: + f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, ca_key_o)) + with open(cert_key, 'w+') as f: + f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, cert_key_o)) + with open(ca_crt, 'w+') as f: + f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, ca_crt_o)) + + if not isdir(cert_dir): + mkdir(cert_dir) + + except StandardError as e: + logger.exception(e) + + +def _load_crypto_obj(path, crypto_method): + with open(path, 'r') as key_fp: + return crypto_method(crypto.FILETYPE_PEM, key_fp.read()) + +cert_key_obj = _load_crypto_obj(cert_key, crypto.load_privatekey) +ca_key_obj = _load_crypto_obj(ca_key, crypto.load_privatekey) +ca_crt_obj = _load_crypto_obj(ca_crt, crypto.load_certificate) + + +__all__ = [ + 'proxy_CN', + 'dir_name', + 'ca_key', + 'ca_crt', + 'cert_key', + 'cert_dir', + 'ca_files_exist', + 'cert_key_obj', + 'ca_key_obj', + 'ca_crt_obj', + 'generate_key_pair', + 'create_cert_request', + 'create_certificate', +] + + +if __name__ == '__main__': + def _load_crypto_obj(path, crypto_method): + with open(path, 'r') as key_fp: + return crypto_method(crypto.FILETYPE_PEM, key_fp.read()) + + cert_key_obj = _load_crypto_obj(cert_key, crypto.load_privatekey) + ca_key_obj = _load_crypto_obj(ca_key, crypto.load_privatekey) + ca_crt_obj = _load_crypto_obj(ca_crt, crypto.load_certificate) + cert_req = create_cert_request(ca_key_obj, CN=proxy_CN) + signed_req = create_certificate( + cert_req, (ca_crt_obj, ca_key_obj), 1509982490957715, + (0, 60 * 60 * 24 * 30), self_signed_x509v3=True + ) + + + print(crypto.dump_certificate(crypto.FILETYPE_PEM, signed_req)) + 'openssl x509 -req -days 3650 -CA ca.crt -CAkey ca.key -set_serial 1509982490957715' + 'https://github.com/pyca/pyopenssl/blob/master/examples/certgen.py' From bb21533addd704e15a0bab9627ac55b95ae2d472 Mon Sep 17 00:00:00 2001 From: Vitaliy Zhaborovskyy Date: Wed, 23 Oct 2019 15:38:02 +0300 Subject: [PATCH 2/8] tried to fix for correct work with python3. doesn't effect: ssl.SSLEOFError: EOF occurred in violation of protocol --- proxy2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proxy2.py b/proxy2.py index 2534745..657730e 100644 --- a/proxy2.py +++ b/proxy2.py @@ -75,6 +75,7 @@ def do_CONNECT(self): if ca_files_exist(): self.connect_intercept() else: + print("can't encode ssl traffic, just relay it") self.connect_relay() def connect_intercept(self): @@ -206,7 +207,7 @@ def do_GET(self): return content_encoding = res.headers.get('Content-Encoding', 'identity') - res_body_plain = self.decode_content_body(res_body, content_encoding) + res_body_plain = self.decode_content_body(res_body.encode('latin_1'), content_encoding) res_body_modified = self.response_handler(req, req_body, res, res_body_plain) if res_body_modified is False: From 356dac1fd01d6e40a97051d0e9dc8f60db4fc58d Mon Sep 17 00:00:00 2001 From: Vitaliy Zhaborovskyy Date: Sun, 17 May 2020 15:53:44 +0300 Subject: [PATCH 3/8] full python3 support. fixed some bug and tested with 3.8 version --- https_trasparent.py | 2 +- proxy2.py | 32 ++++++++++++++++---------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/https_trasparent.py b/https_trasparent.py index f5150b7..48bd689 100644 --- a/https_trasparent.py +++ b/https_trasparent.py @@ -32,7 +32,7 @@ def test(HandlerClass=ProxyRequestHandler, ServerClass=ThreadingHTTPSServer, pro httpd = ServerClass(server_address, HandlerClass) sa = httpd.socket.getsockname() - print "Serving HTTPS Proxy on", sa[0], "port", sa[1], "..." + print("Serving HTTPS Proxy on", sa[0], "port", sa[1], "...") httpd.serve_forever() diff --git a/proxy2.py b/proxy2.py index 657730e..b255687 100644 --- a/proxy2.py +++ b/proxy2.py @@ -13,21 +13,19 @@ from ssl_wrapper import * from string import Template from OpenSSL import crypto - +import html try: import http.client as httplib import urllib.parse as urlparse from http.server import HTTPServer, BaseHTTPRequestHandler from socketserver import ThreadingMixIn - from io import StringIO - from html.parser import HTMLParser + from io import StringIO, BytesIO except ImportError: import httplib import urlparse from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler from SocketServer import ThreadingMixIn - from cStringIO import StringIO - from HTMLParser import HTMLParser + from cStringIO import StringIO, BytesIO def print_color(c, s): @@ -216,6 +214,7 @@ def do_GET(self): elif res_body_modified is not None: res_body_plain = res_body_modified res_body = self.encode_content_body(res_body_plain, content_encoding) + del res.headers['Content-Length'] res.headers['Content-Length'] = str(len(res_body)) setattr(res, 'headers', self.filter_headers(res.headers)) @@ -224,16 +223,18 @@ def do_GET(self): for k, v in res.headers.items(): self.send_header(k, v) self.end_headers() - self.wfile.write(res_body.encode('latin_1')) + if res_body: + self.wfile.write(res_body.encode('latin_1')) self.wfile.flush() with self.lock: self.save_handler(req, req_body, res, res_body_plain) def relay_streaming(self, res): - self.wfile.write("%s %d %s\r\n" % (self.protocol_version, res.status, res.reason)) - for line in res.headers.headers: - self.wfile.write(line) + self.wfile.write("{} {} {}\r\n".format(self.protocol_version, res.status, res.reason) + .encode('latin-1', 'strinct')) + for k, v in res.headers.items(): + self.send_header(k, v) self.end_headers() try: while True: @@ -270,7 +271,7 @@ def encode_content_body(self, text, encoding): if encoding == 'identity': data = text elif encoding in ('gzip', 'x-gzip'): - io = StringIO() + io = BytesIO() with gzip.GzipFile(fileobj=io, mode='wb') as f: f.write(text) data = io.getvalue() @@ -284,7 +285,7 @@ def decode_content_body(self, data, encoding): if encoding == 'identity': text = data elif encoding in ('gzip', 'x-gzip'): - io = StringIO(data) + io = BytesIO(data) with gzip.GzipFile(fileobj=io) as f: text = f.read() elif encoding == 'deflate': @@ -338,7 +339,7 @@ def parse_qsl(s): content_type = req.headers.get('Content-Type', '') if content_type.startswith('application/x-www-form-urlencoded'): - req_body_text = parse_qsl(req_body) + req_body_text = parse_qsl(req_body.decode('latin-1')) elif content_type.startswith('application/json'): try: json_obj = json.loads(req_body) @@ -382,10 +383,9 @@ def parse_qsl(s): except ValueError: res_body_text = res_body elif content_type.startswith('text/html'): - m = re.search(r']*>\s*([^<]+?)\s*', res_body, re.I) + m = re.search(r']*>\s*([^<]+?)\s*', res_body.decode('latin-1'), re.I) if m: - h = HTMLParser() - print_color(32, "==== HTML TITLE ====\n{}\n".format(h.unescape(m.group(1)))) + print_color(32, "==== HTML TITLE ====\n{}\n".format(html.unescape(m.group(1)))) elif content_type.startswith('text/') and len(res_body) < 1024: res_body_text = res_body @@ -406,7 +406,7 @@ def test(HandlerClass=ProxyRequestHandler, ServerClass=ThreadingHTTPServer, prot if sys.argv[1:]: port = int(sys.argv[1]) else: - port = 8080 + port = 8082 server_address = ('::1', port) HandlerClass.protocol_version = protocol From b3d74e7ce0d82b21136db0429f2f577bafd64401 Mon Sep 17 00:00:00 2001 From: Vitaliy Zhaborovskiy Date: Thu, 3 Sep 2020 14:46:48 +0300 Subject: [PATCH 4/8] fixed bugs in ssl wrapper. now bytes array is uses --- .idea/vcs.xml | 6 ++++++ ssl_wrapper.py | 16 ++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 .idea/vcs.xml diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/ssl_wrapper.py b/ssl_wrapper.py index e23e8d7..37a2036 100644 --- a/ssl_wrapper.py +++ b/ssl_wrapper.py @@ -97,13 +97,13 @@ def create_certificate( if self_signed_x509v3: ret_x509_obj.set_version(2) ret_x509_obj.add_extensions([ - crypto.X509Extension("subjectKeyIdentifier", False, "hash", + crypto.X509Extension(b'subjectKeyIdentifier', False, b'hash', subject=ret_x509_obj), - crypto.X509Extension("basicConstraints", False, "CA:TRUE"), + crypto.X509Extension(b'basicConstraints', False, b'CA:TRUE'), ]) ret_x509_obj.add_extensions([ - crypto.X509Extension("authorityKeyIdentifier", False, - "keyid:always", issuer=ret_x509_obj), + crypto.X509Extension(b'authorityKeyIdentifier', False, + b'keyid:always', issuer=ret_x509_obj), ]) if len(subject_alt_names) != 0: ret_x509_obj.set_version(2) # 0x3 @@ -120,7 +120,7 @@ def create_certificate( def ca_files_exist(): return all( - [map(isfile, [ca_key, ca_crt, cert_key]), isdir(dir_name)]) + list(map(isfile, [ca_key, ca_crt, cert_key])) + [isdir(dir_name)]) if not ca_files_exist(): # TODO: move this code to pyopenssl library @@ -134,11 +134,11 @@ def ca_files_exist(): cert_req_temp, ('__self_signed', ca_key_o), 1509982490957715, (0, 60 * 60 * 24 * 30), self_signed_x509v3=True ) - with open(ca_key, 'w+') as f: + with open(ca_key, 'wb+') as f: f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, ca_key_o)) - with open(cert_key, 'w+') as f: + with open(cert_key, 'wb+') as f: f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, cert_key_o)) - with open(ca_crt, 'w+') as f: + with open(ca_crt, 'wb+') as f: f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, ca_crt_o)) if not isdir(cert_dir): From 3c4a7aa64ae557f7213762c7e2d744a396041afd Mon Sep 17 00:00:00 2001 From: Vitaliy Zhaborovskiy Date: Fri, 4 Sep 2020 11:30:56 +0300 Subject: [PATCH 5/8] debug commit --- proxy2.py | 91 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 71 insertions(+), 20 deletions(-) diff --git a/proxy2.py b/proxy2.py index b255687..096b2d7 100644 --- a/proxy2.py +++ b/proxy2.py @@ -42,6 +42,7 @@ class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): def handle_error(self, request, client_address): # surpress socket/ssl related errors cls, e = sys.exc_info()[:2] + print("error is here: ", cls, e) if cls is socket.error or cls is ssl.SSLError: pass else: @@ -101,7 +102,7 @@ def connect_intercept(self): f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert)) - self.wfile.write("{} {} {}\r\n".format(self.protocol_version, 200, 'Connection Established').encode('latin_1')) + self.wfile.write("{} {} {}\r\n".format(self.protocol_version, 200, 'Connection Established').encode('latin-1')) self.wfile.write(b'\r\n') self.connection = ssl.wrap_socket(self.connection, @@ -117,6 +118,7 @@ def connect_intercept(self): self.close_connection = 0 else: self.close_connection = 1 + print("CONNECTION CLOSED 0") def connect_relay(self): address = self.path.split(':', 1) @@ -140,6 +142,7 @@ def connect_relay(self): data = r.recv(8192) if not data: self.close_connection = 1 + print("CONNECTION CLOSED 2") break other.sendall(data) @@ -164,18 +167,22 @@ def do_GET(self): return elif req_body_modified is not None: req_body = req_body_modified + if 'Content-Length' in req.headers: + del req.headers['Content-Length'] req.headers['Content-length'] = str(len(req_body)) u = urlparse.urlsplit(req.path) scheme, netloc, path = u.scheme, u.netloc, (u.path + '?' + u.query if u.query else u.path) assert scheme in ('http', 'https') if netloc: + if 'Host' in req.headers: + del req.headers['Host'] req.headers['Host'] = netloc setattr(req, 'headers', self.filter_headers(req.headers)) try: origin = (scheme, netloc) - if not origin in self.tls.conns: + if origin not in self.tls.conns: if scheme == 'https': self.tls.conns[origin] = httplib.HTTPSConnection(netloc, timeout=self.timeout) else: @@ -189,15 +196,15 @@ def do_GET(self): setattr(res, 'response_version', version_table[res.version]) # support streaming - if not 'Content-Length' in res.headers and 'no-store' in res.headers.get('Cache-Control', ''): + if 'Content-Length' not in res.headers and 'no-store' in res.headers.get('Cache-Control', ''): self.response_handler(req, req_body, res, '') setattr(res, 'headers', self.filter_headers(res.headers)) self.relay_streaming(res) - with self.lock: - self.save_handler(req, req_body, res, '') + #with self.lock: + # self.save_handler(req, req_body, res, '') return - res_body = res.read().decode('latin_1') + res_body = res.read().decode('latin-1') except Exception as e: if origin in self.tls.conns: del self.tls.conns[origin] @@ -205,7 +212,7 @@ def do_GET(self): return content_encoding = res.headers.get('Content-Encoding', 'identity') - res_body_plain = self.decode_content_body(res_body.encode('latin_1'), content_encoding) + res_body_plain = self.decode_content_body(res_body.encode('latin-1'), content_encoding) res_body_modified = self.response_handler(req, req_body, res, res_body_plain) if res_body_modified is False: @@ -214,21 +221,55 @@ def do_GET(self): elif res_body_modified is not None: res_body_plain = res_body_modified res_body = self.encode_content_body(res_body_plain, content_encoding) - del res.headers['Content-Length'] + if 'Content-Length' in res.headers: + del res.headers['Content-Length'] res.headers['Content-Length'] = str(len(res_body)) - setattr(res, 'headers', self.filter_headers(res.headers)) + if 'Content-Length' not in res.headers: + res.headers['Content-Length'] = str(len(res_body)) - self.wfile.write("{} {} {}\r\n".format(self.protocol_version, res.status, res.reason).encode('latin_1')) +# + headers_before_filtering = "" for k, v in res.headers.items(): - self.send_header(k, v) - self.end_headers() - if res_body: - self.wfile.write(res_body.encode('latin_1')) - self.wfile.flush() + headers_before_filtering = headers_before_filtering + k + ": " + v + "\n" +# + setattr(res, 'headers', self.filter_headers(res.headers)) - with self.lock: - self.save_handler(req, req_body, res, res_body_plain) + try: + self.wfile.write("{} {} {}\r\n".format(self.protocol_version, res.status, res.reason).encode('latin-1')) + #print("{} {} {}\r\n".format(self.protocol_version, res.status, res.reason).encode('latin-1')) + + for k, v in res.headers.items(): + self.send_header(k, v) + + #print(b"".join(self._headers_buffer)) + self.end_headers() + if res_body: + self.wfile.write(res_body.encode('latin-1')) + self.wfile.flush() + + except Exception as e: + print("------------\n", + self.command, self.path, self.protocol_version, res.status, res.reason, "\n", + res.headers, "\n", + '=========', "\n", + headers_before_filtering) + #print(b"".join(self._headers_buffer)) + #print("{} {} {}\r\n".format(self.protocol_version, res.status, res.reason).encode('latin-1')) + #print("------------") + if res_body: + if res.headers['Content-Encoding'] in ('gzip', 'x-gzip'): + io = BytesIO(res_body.encode('latin-1')) + with gzip.GzipFile(fileobj=io) as f: + data = f.read() + print(data) + raise e + self.close_connection = 1 + return + + + #with self.lock: + # self.save_handler(req, req_body, res, res_body_plain) def relay_streaming(self, res): self.wfile.write("{} {} {}\r\n".format(self.protocol_version, res.status, res.reason) @@ -255,7 +296,16 @@ def relay_streaming(self, res): def filter_headers(self, headers): # http://tools.ietf.org/html/rfc2616#section-13.5.1 - hop_by_hop = ('connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', 'upgrade') + hop_by_hop = ( + 'connection', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'te', + 'trailers', + 'transfer-encoding', + 'upgrade' + ) for k in hop_by_hop: del headers[k] @@ -263,6 +313,7 @@ def filter_headers(self, headers): if 'Accept-Encoding' in headers: ae = headers['Accept-Encoding'] filtered_encodings = [x for x in re.split(r',\s*', ae) if x in ('identity', 'gzip', 'x-gzip', 'deflate')] + del headers['Accept-Encoding'] headers['Accept-Encoding'] = ', '.join(filtered_encodings) return headers @@ -303,7 +354,7 @@ def send_cacert(self): with open(self.cacert, 'rb') as f: data = f.read() - self.wfile.write("{} {} {}\r\n".format(self.protocol_version, 200, 'OK').encode('latin_1')) + self.wfile.write("{} {} {}\r\n".format(self.protocol_version, 200, 'OK').encode('latin-1')) self.send_header('Content-Type', 'application/x-x509-ca-cert') self.send_header('Content-Length', len(data)) self.send_header('Connection', 'close') @@ -406,7 +457,7 @@ def test(HandlerClass=ProxyRequestHandler, ServerClass=ThreadingHTTPServer, prot if sys.argv[1:]: port = int(sys.argv[1]) else: - port = 8082 + port = 8080 server_address = ('::1', port) HandlerClass.protocol_version = protocol From 99aa857c7ebe9e1232532549769a66d4ec922f79 Mon Sep 17 00:00:00 2001 From: Vitaliy Zhaborovskiy Date: Fri, 4 Sep 2020 18:12:22 +0300 Subject: [PATCH 6/8] Small bugs with new headers values fixed: the headers here isnot dictionary and old values has to be deleted for its value update. The "BrokenPipeError" catches at handle_error function. In Python3 it is standalone exception --- proxy2.py | 57 ++++++++++++++++--------------------------------------- 1 file changed, 16 insertions(+), 41 deletions(-) diff --git a/proxy2.py b/proxy2.py index 096b2d7..0b3c4c4 100644 --- a/proxy2.py +++ b/proxy2.py @@ -42,8 +42,13 @@ class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): def handle_error(self, request, client_address): # surpress socket/ssl related errors cls, e = sys.exc_info()[:2] - print("error is here: ", cls, e) - if cls is socket.error or cls is ssl.SSLError: + if cls is socket.error or cls is BrokenPipeError or cls is ssl.SSLError: + # BrokenPipeError is socket.error in Python2 and standalone error in Python3. + # This is most frequently raised error here. + # I don't understand why it raises here + # looks like it is caused by some errors in the proxy logic: for some + # reasons a client closes connection + # I thinks the keep-alive logic should be checked. pass else: return HTTPServer.handle_error(self, request, client_address) @@ -228,48 +233,18 @@ def do_GET(self): if 'Content-Length' not in res.headers: res.headers['Content-Length'] = str(len(res_body)) -# - headers_before_filtering = "" - for k, v in res.headers.items(): - headers_before_filtering = headers_before_filtering + k + ": " + v + "\n" -# setattr(res, 'headers', self.filter_headers(res.headers)) - try: - self.wfile.write("{} {} {}\r\n".format(self.protocol_version, res.status, res.reason).encode('latin-1')) - #print("{} {} {}\r\n".format(self.protocol_version, res.status, res.reason).encode('latin-1')) - - for k, v in res.headers.items(): - self.send_header(k, v) - - #print(b"".join(self._headers_buffer)) - self.end_headers() - if res_body: - self.wfile.write(res_body.encode('latin-1')) - self.wfile.flush() - - except Exception as e: - print("------------\n", - self.command, self.path, self.protocol_version, res.status, res.reason, "\n", - res.headers, "\n", - '=========', "\n", - headers_before_filtering) - #print(b"".join(self._headers_buffer)) - #print("{} {} {}\r\n".format(self.protocol_version, res.status, res.reason).encode('latin-1')) - #print("------------") - if res_body: - if res.headers['Content-Encoding'] in ('gzip', 'x-gzip'): - io = BytesIO(res_body.encode('latin-1')) - with gzip.GzipFile(fileobj=io) as f: - data = f.read() - print(data) - raise e - self.close_connection = 1 - return - + self.wfile.write("{} {} {}\r\n".format(self.protocol_version, res.status, res.reason).encode('latin-1')) + for k, v in res.headers.items(): + self.send_header(k, v) + self.end_headers() + if res_body: + self.wfile.write(res_body.encode('latin-1')) + self.wfile.flush() - #with self.lock: - # self.save_handler(req, req_body, res, res_body_plain) + with self.lock: + self.save_handler(req, req_body, res, res_body_plain) def relay_streaming(self, res): self.wfile.write("{} {} {}\r\n".format(self.protocol_version, res.status, res.reason) From e3832dfbe65aa88dddc6418f7205d7221de0957d Mon Sep 17 00:00:00 2001 From: Vitaliy Zhaborovskiy Date: Fri, 4 Sep 2020 18:21:51 +0300 Subject: [PATCH 7/8] Content-length -> Content-Length --- proxy2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy2.py b/proxy2.py index 0b3c4c4..62746fd 100644 --- a/proxy2.py +++ b/proxy2.py @@ -174,7 +174,7 @@ def do_GET(self): req_body = req_body_modified if 'Content-Length' in req.headers: del req.headers['Content-Length'] - req.headers['Content-length'] = str(len(req_body)) + req.headers['Content-Length'] = str(len(req_body)) u = urlparse.urlsplit(req.path) scheme, netloc, path = u.scheme, u.netloc, (u.path + '?' + u.query if u.query else u.path) From 45a93778155f6bffe5e5a4842ad99bbd3a523dfc Mon Sep 17 00:00:00 2001 From: vitaliyz Date: Mon, 17 Jan 2022 15:25:50 +0200 Subject: [PATCH 8/8] chain proxy support added --- proxy2.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/proxy2.py b/proxy2.py index 62746fd..3b850c2 100644 --- a/proxy2.py +++ b/proxy2.py @@ -59,13 +59,13 @@ class ProxyRequestHandler(BaseHTTPRequestHandler): cacert = join_with_script_dir('ca.crt') certkey = join_with_script_dir('cert.key') certdir = join_with_script_dir('certs/') - timeout = 5 + timeout = 10 + chain_proxy = "" lock = threading.Lock() def __init__(self, *args, **kwargs): self.tls = threading.local() self.tls.conns = {} - BaseHTTPRequestHandler.__init__(self, *args, **kwargs) def log_error(self, format, *args): @@ -106,7 +106,6 @@ def connect_intercept(self): with open(certpath, 'wb+') as f: f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert)) - self.wfile.write("{} {} {}\r\n".format(self.protocol_version, 200, 'Connection Established').encode('latin-1')) self.wfile.write(b'\r\n') @@ -114,7 +113,6 @@ def connect_intercept(self): keyfile=cert_key, certfile=certpath, server_side=True) - self.rfile = self.connection.makefile("rb", self.rbufsize) self.wfile = self.connection.makefile("wb", self.wbufsize) @@ -123,7 +121,6 @@ def connect_intercept(self): self.close_connection = 0 else: self.close_connection = 1 - print("CONNECTION CLOSED 0") def connect_relay(self): address = self.path.split(':', 1) @@ -147,7 +144,7 @@ def connect_relay(self): data = r.recv(8192) if not data: self.close_connection = 1 - print("CONNECTION CLOSED 2") + break other.sendall(data) @@ -188,10 +185,14 @@ def do_GET(self): try: origin = (scheme, netloc) if origin not in self.tls.conns: + connection_host = self.chain_proxy if len(self.chain_proxy) else netloc if scheme == 'https': - self.tls.conns[origin] = httplib.HTTPSConnection(netloc, timeout=self.timeout) + self.tls.conns[origin] = httplib.HTTPSConnection(connection_host, timeout=self.timeout) else: - self.tls.conns[origin] = httplib.HTTPConnection(netloc, timeout=self.timeout) + self.tls.conns[origin] = httplib.HTTPConnection(connection_host, timeout=self.timeout) + if len(self.chain_proxy): + self.tls.conns[origin].set_tunnel(netloc) + conn = self.tls.conns[origin] conn.request(self.command, path, req_body, dict(req.headers)) res = conn.getresponse() @@ -208,7 +209,6 @@ def do_GET(self): #with self.lock: # self.save_handler(req, req_body, res, '') return - res_body = res.read().decode('latin-1') except Exception as e: if origin in self.tls.conns: @@ -235,6 +235,7 @@ def do_GET(self): setattr(res, 'headers', self.filter_headers(res.headers)) + self.wfile.write("{} {} {}\r\n".format(self.protocol_version, res.status, res.reason).encode('latin-1')) for k, v in res.headers.items(): self.send_header(k, v) @@ -433,9 +434,10 @@ def test(HandlerClass=ProxyRequestHandler, ServerClass=ThreadingHTTPServer, prot port = int(sys.argv[1]) else: port = 8080 - server_address = ('::1', port) + server_address = ("localhost", port) HandlerClass.protocol_version = protocol + #HandlerClass.chain_proxy = "localhost:9182" httpd = ServerClass(server_address, HandlerClass) sa = httpd.socket.getsockname()