Merge pull request mitmproxy#3045 from mhils/3.0.x

Bump to 3.0.4

.travis.yml

@@ -42,15 +42,15 @@ matrix:
           yarn: true
           directories:
               - web/node_modules
-    - language: python
-      env:
-        - NAME=docs
+    - python: 3.6
+      env: NAME=docs TOXENV=docs
       install:
         - wget https://github.com/gohugoio/hugo/releases/download/v0.37/hugo_0.37_Linux-64bit.deb
         - sudo dpkg -i hugo*.deb
-        - pip install awscli
+        - pip install tox virtualenv setuptools
+        - pyenv global system 3.6
       script:
-        - cd docs && ./ci
+        - tox
       after_success:
         - echo done
 

CHANGELOG

@@ -1,6 +1,18 @@
-23 February 2018: mitmproxy 3.0
+05 April 2018: mitmproxy 3.0.4
 
-  * Fix a quote-related issue affecting the mitmproxy console command prompt
+  * Fix an issue that caused mitmproxy to not retry HTTP requests on timeout.
+
+  * Various other fixes (@kira0204, @fenilgandhi, @tran-tien-dat, @smonami, 
+  @luzpaz, @fristonio, @kajojify, @Oliver-Fish, @hcbarry, @jplochocki, @MikeShi42,
+  @ghillu, @emilstahl)
+
+25 February 2018: mitmproxy 3.0.3
+
+  * Fix an issue that caused mitmproxy to lose keyboard control after spawning an external editor.
+
+23 February 2018: mitmproxy 3.0.1
+
+  * Fix a quote-related issue affecting the mitmproxy console command prompt.
 
 22 February 2018: mitmproxy 3.0
 

README.rst

@@ -26,9 +26,9 @@ and pathod websites.
 
 |mitmproxy_site|
 
-The latest documentation for mitmproxy is available on our website.
+The documentation for mitmproxy is available on our website:
 
-|mitmproxy_docs|
+|mitmproxy_docs_stable| |mitmproxy_docs_master| 
 
 
 Join our discussion forum on Discourse to ask questions, help
@@ -45,7 +45,7 @@ Join our developer chat on Slack if you would like to contribute to mitmproxy it
 Installation
 ------------
 
-The installation instructions are `here <https://mitmproxy.org/docs/latest/overview-installation>`__.
+The installation instructions are `here <https://docs.mitmproxy.org/stable/overview-installation>`__.
 If you want to contribute changes, keep on reading.
 
 Contributing
@@ -150,9 +150,13 @@ with the following command:
     :target: https://mitmproxy.org/
     :alt: mitmproxy.org
 
-.. |mitmproxy_docs| image:: https://shields.mitmproxy.org/api/docs-latest-brightgreen.svg
-    :target: https://mitmproxy.org/docs/latest/
-    :alt: mitmproxy documentation
+.. |mitmproxy_docs_stable| image:: https://shields.mitmproxy.org/api/docs-stable-brightgreen.svg
+    :target: https://docs.mitmproxy.org/stable/
+    :alt: mitmproxy documentation stable
+
+.. |mitmproxy_docs_master| image:: https://shields.mitmproxy.org/api/docs-master-brightgreen.svg
+    :target: https://docs.mitmproxy.org/master/
+    :alt: mitmproxy documentation master
 
 .. |mitmproxy_discourse| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-discourse.mitmproxy.org-orange.svg
     :target: https://discourse.mitmproxy.org
@@ -182,7 +186,7 @@ with the following command:
     :target: https://pypi.python.org/pypi/mitmproxy
     :alt: Supported Python versions
 
-.. _`advanced installation`: https://mitmproxy.org/docs/latest/overview-installation/#advanced-installation
+.. _`advanced installation`: https://docs.mitmproxy.org/stable/overview-installation/#advanced-installation
 .. _virtualenv: https://virtualenv.pypa.io/
 .. _`pytest`: http://pytest.org/
 .. _tox: https://tox.readthedocs.io/

docs/.gitignore

@@ -1,3 +1,4 @@
+generated/
 src/public
 node_modules
 public

docs/build-current

@@ -1,5 +1,12 @@
-#!/bin/sh
+#!/usr/bin/env bash
+
 set -e
 
+for script in scripts/* ; do
+  echo "Generating output for $script ..."
+  output="${script##*/}"
+  "$script" > "src/generated/${output%.*}.html"
+done
+
 cd src
 hugo

docs/scripts/filters.py

@@ -1,9 +1,9 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from mitmproxy import flowfilter
 
 
 print("<table class=\"table filtertable\"><tbody>")
 for i in flowfilter.help:
     print("<tr><th>%s</th><td>%s</td></tr>" % i)
-print("</tbody></table>")
+print("</tbody></table>")

docs/scripts/options.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+from mitmproxy import options, optmanager
+from mitmproxy.tools import dump, console, web
+
+masters = {
+    "mitmproxy": console.master.ConsoleMaster,
+    "mitmdump": dump.DumpMaster,
+    "mitmweb": web.master.WebMaster
+}
+
+unified_options = {}
+
+for tool_name, master in masters.items():
+    opts = options.Options()
+    inst = master(opts)
+    for key, option in optmanager.dump_dicts(opts).items():
+        if key in unified_options:
+            unified_options[key]['tools'].append(tool_name)
+        else:
+            unified_options[key] = option
+            unified_options[key]['tools'] = [tool_name]
+
+print("""
+      <table class=\"table optiontable\">
+        <thead>
+          <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+        </tr>
+        </thead>
+        <tbody>
+      """.strip())
+for key, option in sorted(unified_options.items(), key=lambda t: t[0]):
+    print("""
+          <tr>
+          <th>{}<br/>{}</th>
+          <td>{}</td>
+          <td>{}<br/>
+            Default: {}
+            {}
+          </td>
+          </tr>
+          """.strip().format(
+              key,
+              ' '.join(["<span class='badge'>{}</span>".format(t) for t in option['tools']]),
+              option['type'],
+              option['help'],
+              option['default'],
+              "<br/>Choices: {}".format(', '.join(option['choices'])) if option['choices'] else "",
+    ))
+print("</tbody></table>")

docs/src/content/concepts-filters.md

@@ -10,37 +10,7 @@ menu:
 Many commands in the mitmproxy tool make use of filter expressions. Filter
 expressions consist of the following operators:
 
-
-<table class="table filtertable"><tbody>
-<tr><th>~a</th><td>Match asset in response: CSS, Javascript, Flash, images.</td></tr>
-<tr><th>~b regex</th><td>Body</td></tr>
-<tr><th>~bq regex</th><td>Request body</td></tr>
-<tr><th>~bs regex</th><td>Response body</td></tr>
-<tr><th>~c int</th><td>HTTP response code</td></tr>
-<tr><th>~d regex</th><td>Domain</td></tr>
-<tr><th>~dst regex</th><td>Match destination address</td></tr>
-<tr><th>~e</th><td>Match error</td></tr>
-<tr><th>~h regex</th><td>Header</td></tr>
-<tr><th>~hq regex</th><td>Request header</td></tr>
-<tr><th>~hs regex</th><td>Response header</td></tr>
-<tr><th>~http</th><td>Match HTTP flows</td></tr>
-<tr><th>~m regex</th><td>Method</td></tr>
-<tr><th>~marked</th><td>Match marked flows</td></tr>
-<tr><th>~q</th><td>Match request with no response</td></tr>
-<tr><th>~s</th><td>Match response</td></tr>
-<tr><th>~src regex</th><td>Match source address</td></tr>
-<tr><th>~t regex</th><td>Content-type header</td></tr>
-<tr><th>~tcp</th><td>Match TCP flows</td></tr>
-<tr><th>~tq regex</th><td>Request Content-Type header</td></tr>
-<tr><th>~ts regex</th><td>Response Content-Type header</td></tr>
-<tr><th>~u regex</th><td>URL</td></tr>
-<tr><th>~websocket</th><td>Match WebSocket flows</td></tr>
-<tr><th>!</th><td>unary not</td></tr>
-<tr><th>&</th><td>and</td></tr>
-<tr><th>|</th><td>or</td></tr>
-<tr><th>(...)</th><td>grouping</td></tr>
-</tbody></table>
-
+{{< readfile file="/generated/filters.html" >}}
 
 - Regexes are Python-style
 - Regexes can be specified as quoted strings

docs/src/content/concepts-options.md

@@ -34,33 +34,14 @@ interactively have immediate effect in the running instance, and can be made
 persistent by saving the settings out to a YAML configuration file (please see
 the specific tool's interactive help for details on how to do this).
 
-For all tools, options can be set directly by name using the `--set` command-line
-option. Please see the command-line help (`--help`) for usage.
+For all tools, options can be set directly by name using the `--set`
+command-line option. Please see the command-line help (`--help`) for usage.
 
 
-## Example
-
-Here is an excerpt showing the first few lines of the complete `--options`
-output:
-
-{{< highlight yaml  >}}
-
-# Add all certificates of the upstream server to the certificate chain
-# that will be served to the proxy client, as extras. Type bool.
-add_upstream_certs_to_client_chain: false
-
-# Allow remote clients to connect to proxy. If set to false, client will
-# not be able to connect to proxy unless it is on the same network or
-# the proxyauth option is set Type bool.
-allow_remote: false
-
-# Strip out request headers that might cause the server to return
-# 304-not-modified. Type bool.
-anticache: false
-
-# Try to convince servers to send us un-compressed data. Type bool.
-anticomp: false
-
-{{< /highlight >}}
+## Available Options
 
+This list might not reflect what is actually available in your current mitmproxy
+environment. For an up-to-date list please use the `--options` flag for each of
+the mitmproxy tools.
 
+{{< readfile file="/generated/options.html" >}}

docs/src/content/howto-transparent.md

@@ -172,7 +172,7 @@ rdr on en0 inet proto tcp to any port {80, 443} -> 127.0.0.1 port 8080
 
 These rules tell pf to redirect all traffic destined for port 80 or 443
 to the local mitmproxy instance running on port 8080. You should replace
-`en2` with the interface on which your test device will appear.
+`en0` with the interface on which your test device will appear.
 
 ### 3. Configure pf with the rules.
 

docs/src/layouts/shortcodes/readfile.html

@@ -0,0 +1,6 @@
+{{$file := .Get "file"}}
+{{- if eq (.Get "markdown") "true" -}}
+{{- $file  | readFile | markdownify -}}
+{{- else -}}
+{{ $file  | readFile | safeHTML }}
+{{- end -}}

docs/src/themes/mitmproxydocs/static/css/style.css

@@ -250,6 +250,21 @@
 .chroma .gu {
   color: #75715e; }
 
+.badge {
+  color: #fff;
+  background-color: #6c757d;
+  display: inline-block;
+  padding: .25em .4em;
+  font-size: 75%;
+  font-weight: 1;
+  line-height: 1;
+  text-align: center;
+  white-space: nowrap;
+  vertical-align: baseline;
+  border-radius: 0.25rem; }
+  .badge:empty {
+    display: none; }
+
 @keyframes spinAround {
   from {
     transform: rotate(0deg); }

docs/style/badge.scss

@@ -0,0 +1,18 @@
+.badge {
+  color: #fff;
+  background-color: #6c757d;
+  display: inline-block;
+  padding: .25em .4em;
+  font-size: 75%;
+  font-weight: 1;
+  line-height: 1;
+  text-align: center;
+  white-space: nowrap;
+  vertical-align: baseline;
+  border-radius: 0.25rem;
+
+  // Empty badges collapse automatically
+  &:empty {
+    display: none;
+  }
+}

docs/style/style.scss

@@ -1,4 +1,5 @@
 @import "./syntax";
+@import "./badge";
 
 $primary: #C93312;
 $warning-invert: #FFFFFF;

mitmproxy/connections.py

@@ -1,18 +1,18 @@
-import time
-
 import os
+import time
 import typing
 import uuid
 
-from mitmproxy import stateobject, exceptions
 from mitmproxy import certs
+from mitmproxy import exceptions
+from mitmproxy import stateobject
 from mitmproxy.net import tcp
 from mitmproxy.net import tls
+from mitmproxy.utils import human
 from mitmproxy.utils import strutils
 
 
 class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
-
     """
     A client connection
 
@@ -72,11 +72,10 @@ def __repr__(self):
         else:
             alpn = ""
 
-        return "<ClientConnection: {tls}{alpn}{host}:{port}>".format(
+        return "<ClientConnection: {tls}{alpn}{address}>".format(
             tls=tls,
             alpn=alpn,
-            host=self.address[0],
-            port=self.address[1],
+            address=human.format_address(self.address),
         )
 
     def __eq__(self, other):
@@ -161,7 +160,6 @@ def finish(self):
 
 
 class ServerConnection(tcp.TCPClient, stateobject.StateObject):
-
     """
     A server connection
 
@@ -209,11 +207,10 @@ def __repr__(self):
             )
         else:
             alpn = ""
-        return "<ServerConnection: {tls}{alpn}{host}:{port}>".format(
+        return "<ServerConnection: {tls}{alpn}{address}>".format(
             tls=tls,
             alpn=alpn,
-            host=self.address[0],
-            port=self.address[1],
+            address=human.format_address(self.address),
         )
 
     def __eq__(self, other):