Minor tweaks to release checlist and binary builds

These are commonly symlinked during dev

.coveragerc

@@ -2,5 +2,5 @@
 branch = True
 
 [report]
-omit = *contrib*, *tnetstring*, *platform*, *console*
+omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py
 include = *libmproxy*

.gitattributes

@@ -0,0 +1,2 @@
+libmproxy/web/static/**/* -diff
+web/src/js/filt/filt.js -diff

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 MANIFEST
 /build
 /dist
@@ -9,9 +10,18 @@ MANIFEST
 *.py[cdo]
 *.swp
 *.swo
+mitmproxy.egg-info/
 mitmproxyc
 mitmdumpc
 .coverage
 .idea
 netlib
+pathod
 libpathod
+
+# UI
+
+node_modules
+bower_components
+*.compiled.js
+*.map

.travis.yml

@@ -1,12 +1,11 @@
 language: python
+sudo: false
 python:
   - "2.7"
+  - pypy
 # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
-install: 
-  - "pip install --upgrade git+https://github.com/mitmproxy/netlib.git"
-  - "pip install --upgrade git+https://github.com/mitmproxy/pathod.git"
-  - "pip install -r requirements.txt --use-mirrors"
-  - "pip install -r test/requirements.txt --use-mirrors"
+install:
+  - "pip install --upgrade --src . -r requirements.txt"
 # command to run tests, e.g. python setup.py test
 script: 
   - "nosetests --with-cov --cov-report term-missing"
@@ -18,4 +17,7 @@ notifications:
       - "irc.oftc.net#mitmproxy"
     on_success: change
     on_failure: always
-
+cache:
+  directories:
+  - /home/travis/virtualenv/python2.7.9/lib/python2.7/site-packages
+  - /home/travis/virtualenv/pypy-2.4.0/site-packages

CHANGELOG

@@ -1,3 +1,60 @@
+29 Dec 2014: mitmproxy 0.11.2:
+
+    * Configuration files - mitmproxy.conf, mitmdump.conf, common.conf in the
+      .mitmproxy directory.
+    * Better handling of servers that reject connections that are not SNI.
+    * Many other small bugfixes and improvements.
+
+
+15 November 2014: mitmproxy 0.11.1:
+
+    * Bug fixes: connection leaks some crashes 
+
+
+7 November 2014: mitmproxy 0.11:
+    
+    * Performance improvements for mitmproxy console
+
+    * SOCKS5 proxy mode allows mitmproxy to act as a SOCKS5 proxy server
+
+    * Data streaming for response bodies exceeding a threshold
+      (bradpeabody@gmail.com)
+
+    * Ignore hosts or IP addresses, forwarding both HTTP and HTTPS traffic
+      untouched
+
+    * Finer-grained control of traffic replay, including options to ignore
+      contents or parameters when matching flows (marcelo.glezer@gmail.com)
+
+    * Pass arguments to inline scripts 
+
+    * Configurable size limit on HTTP request and response bodies
+
+    * Per-domain specification of interception certificates and keys (see
+      --cert option)
+
+    * Certificate forwarding, relaying upstream SSL certificates verbatim (see
+      --cert-forward)
+
+    * Search and highlighting for HTTP request and response bodies in
+      mitmproxy console (pedro@worcel.com)
+
+    * Transparent proxy support on Windows
+
+    * Improved error messages and logging
+
+    * Support for FreeBSD in transparent mode, using pf (zbrdge@gmail.com)
+
+    * Content view mode for WBXML (davidshaw835@air-watch.com)
+
+    * Better documentation, with a new section on proxy modes
+
+    * Generic TCP proxy mode
+
+    * Countless bugfixes and other small improvements
+
+
+
 28 January 2014: mitmproxy 0.10:
 	
     * Support for multiple scripts and multiple script arguments

CONTRIBUTORS

@@ -1,51 +1,65 @@
-   854	Aldo Cortesi
-    64	Maximilian Hils
+   902	Aldo Cortesi
+   323	Maximilian Hils
     18	Henrik Nordstrom
     13	Thomas Roth
+    12	Pedro Worcel
     11	Stephen Altamirano
     10	András Veres-Szentkirályi
-     8	Jason A. Novak
      8	Rouli
+     8	Jason A. Novak
      7	Alexis Hildebrandt
-     6	Pedro Worcel
      5	Tomaz Muraus
+     5	Brad Peabody
      5	Matthias Urlichs
      4	root
-     4	Bryan Bishop
      4	Marc Liyanage
      4	Valtteri Virtanen
-     3	Kyle Manna
+     4	Bryan Bishop
      3	Chris Neasbitt
-     2	alts
-     2	Heikki Hannikainen
-     2	Jim Lloyd
+     3	Zack B
+     3	Eli Shvartsman
+     3	Kyle Manna
      2	Michael Frister
+     2	Bennett Blodinger
+     2	Jim Lloyd
      2	Rob Wills
-     2	Jaime Soriano Pastor
      2	israel
+     2	Jaime Soriano Pastor
+     2	Heikki Hannikainen
      2	Mark E. Haase
+     2	alts
+     1	davidpshaw
+     1	deployable
+     1	joebowbeer
+     1	meeee
+     1	phil plante
+     1	Michael Bisbjerg
+     1	Andy Smith
+     1	Dan Wilbraham
+     1	David Shaw
+     1	Eric Entzel
+     1	Felix Wolfsteller
+     1	Henrik Nordström
+     1	Ivaylo Popov
+     1	JC
+     1	Jakub Nawalaniec
+     1	James Billingham
+     1	Jean Regisser
+     1	Kit Randel
+     1	Marcelo Glezer
+     1	Mathieu Mitchell
+     1	Mikhail Korobov
+     1	Nicolas Esteves
+     1	Oleksandr Sheremet
      1	Paul
      1	Rich Somerfield
      1	Rory McCann
-     1	Felix Wolfsteller
      1	Rune Halvorsen
      1	Sahn Lam
-     1	Eric Entzel
-     1	Dan Wilbraham
+     1	Seppo Yli-Olli
+     1	Sergey Chipiga
+     1	Steven Van Acker
      1	Ulrich Petri
-     1	Andy Smith
+     1	Vyacheslav Bakhmutov
      1	Yuangxuan Wang
      1	capt8bit
-     1	joebowbeer
-     1	meeee
-     1	James Billingham
-     1	Jakub Nawalaniec
-     1	JC
-     1	Kit Randel
-     1	phil plante
-     1	Mathieu Mitchell
-     1	Ivaylo Popov
-     1	Henrik Nordström
-     1	Michael Bisbjerg
-     1	Nicolas Esteves
-     1	Oleksandr Sheremet

MANIFEST.in

@@ -1,11 +1,10 @@
-include LICENSE
-include CHANGELOG
-include CONTRIBUTORS
-include README.txt
-include setup.py
+include mitmproxy mitmdump
+include LICENSE CHANGELOG CONTRIBUTORS README.txt
 exclude README.mkd
 recursive-include examples *
 recursive-include doc *
 recursive-include test *
-recursive-include libmproxy/resources *
-recursive-exclude test *.swo *.swp *.pyc
+recursive-include libmproxy *
+recursive-exclude * *.pyc *.pyo *.swo *.swp
+recursive-exclude netlib *
+recursive-exclude libpathod *

README.mkd

@@ -13,6 +13,9 @@ mitmproxy.org website:
 [mitmproxy.org](http://mitmproxy.org).
 
 
+You can find complete directions for installing mitmproxy [here](http://mitmproxy.org/doc/install.html). 
+
+
 Features
 --------
 
@@ -26,9 +29,17 @@ Features
 - SSL certificates for interception are generated on the fly.
 - And much, much more.
 
+__mitmproxy__ is tested and developed on OSX, Linux and OpenBSD. On Windows,
+only mitmdump is supported, which does not have a graphical user interface.
+
+
+
+Hacking
+-------
+
+
+### Requirements
 
-Requirements
-------------
 
 * [Python](http://www.python.org) 2.7.x.
 * [netlib](http://pypi.python.org/pypi/netlib), version matching mitmproxy.
@@ -40,22 +51,36 @@ Optional packages for extended content decoding:
 * [protobuf](https://code.google.com/p/protobuf/) version 2.5.0 or newer.
 * [cssutils](http://cthedot.de/cssutils/) version 1.0 or newer.
 
-__mitmproxy__ is tested and developed on OSX, Linux and OpenBSD. Windows is not
-officially supported at the moment.
+For convenience, all optional dependencies can be installed with 
+
+`pip install "mitmproxy[contentviews]"`
+
+### Setting up a dev environment
+
+The following procedure is recommended to set up your dev environment:
+
+```
+$ git clone https://github.com/mitmproxy/mitmproxy.git
+$ cd mitmproxy
+$ pip install --src . -r requirements.txt
+```
+
+This installs the latest GitHub versions of mitmproxy, netlib and pathod into `mitmproxy/`. All other development dependencies save countershape are installed into their usual locations. 
 
 
-Hacking
--------
+### Testing
 
-The following components are needed if you plan to hack on mitmproxy:
+The test suite requires the `dev` extra requirements listed in [setup.py](https://github.com/mitmproxy/mitmproxy/blob/master/setup.py) and [pathod](http://pathod.net), version matching mitmproxy. Install these with:
 
-* The test suite uses the [nose](http://readthedocs.org/docs/nose/en/latest/) unit testing
-  framework and requires [pathod](http://pathod.org) and [flask](http://flask.pocoo.org/).
-* Rendering the documentation requires [countershape](http://github.com/cortesi/countershape).
+`pip install "mitmproxy[dev]"`
 
-For convenience, all dependencies save countershape can be installed from pypi
-to a virtualenv with 'pip install -r requirements.txt'.
 
 Please ensure that all patches are accompanied by matching changes in the test
 suite. The project maintains 100% test coverage.
 
+
+### Docs
+
+Rendering the documentation requires [countershape](http://github.com/cortesi/countershape). After installation, you can render the documentation to the doc like this:
+
+`cshape doc-src doc`

doc-src/_nav.html

@@ -2,24 +2,32 @@
     $!nav(idxpath, this, state)!$
     $!nav("install.html", this, state)!$
     $!nav("howmitmproxy.html", this, state)!$
+    $!nav("modes.html", this, state)!$
 
     <li class="nav-header">Tools</li>
         $!nav("mitmproxy.html", this, state)!$
         $!nav("mitmdump.html", this, state)!$
+        $!nav("config.html", this, state)!$
 
     <li class="nav-header">Features</li>
         $!nav("anticache.html", this, state)!$
-        $!nav("clientreplay.html", this, state)!$
+
         $!nav("filters.html", this, state)!$
-        $!nav("forwardproxy.html", this, state)!$
-        $!nav("proxyauth.html", this, state)!$
         $!nav("replacements.html", this, state)!$
+        $!nav("clientreplay.html", this, state)!$
         $!nav("serverreplay.html", this, state)!$
         $!nav("setheaders.html", this, state)!$
-        $!nav("sticky.html", this, state)!$
+        $!nav("passthrough.html", this, state)!$
+        $!nav("proxyauth.html", this, state)!$
         $!nav("reverseproxy.html", this, state)!$
+        $!nav("responsestreaming.html", this, state)!$
+        $!nav("socksproxy.html", this, state)!$
+        $!nav("sticky.html", this, state)!$
+        $!nav("tcpproxy.html", this, state)!$
+        $!nav("upstreamproxy.html", this, state)!$
         $!nav("upstreamcerts.html", this, state)!$
 
+
     <li class="nav-header">Installing Certificates</li>
         $!nav("ssl.html", this, state)!$
         $!nav("certinstall/webapp.html", this, state)!$
@@ -46,5 +54,6 @@
         $!nav("tutorials/transparent-dhcp.html", this, state)!$
 
     <li class="nav-header">Hacking</li>
+        $!nav("dev/architecture.html", this, state)!$
         $!nav("dev/testing.html", this, state)!$
 </ul>

doc-src/certinstall/firefox.html

@@ -7,7 +7,7 @@ app](@!urlTo("webapp.html")!@). If this fails, do the following:
 <ol class="tlist">
     <li> If needed, copy the ~/.mitmproxy/mitmproxy-ca-cert.pem file to the target. </li>
 
-    <li>Open preferences, click on "Advanced", then select"Encryption":
+    <li>Open preferences, click on "Advanced", then select"Certificates":
         <img src="@!urlTo('firefox3.jpg')!@"/>
     </li>
 
@@ -21,7 +21,7 @@ app](@!urlTo("webapp.html")!@). If this fails, do the following:
 ## Installing the certificate
 
 <ol class="tlist">
-    <li>Tick "Trust this CS to identify web sites", and click "Ok":
+    <li>Tick "Trust this CA to identify web sites", and click "Ok":
         <img src="@!urlTo('firefox3-trust.jpg')!@"/>
     </li>
 

doc-src/certinstall/index.py

@@ -9,4 +9,5 @@ pages = [
     Page("ios-simulator.html", "IOS Simulator"),
     Page("android.html", "Android"),
     Page("java.html", "Java"),
+    Page("mitm.it-error.html", "Error: No proxy configured"),
 ]

doc-src/certinstall/mitm.it-error.html

@@ -0,0 +1,5 @@
+**Looks like you wanted to install the mitmproxy CA using the web app?**
+
+Unfortunately, there's been no mitmproxy instance on the wire that could have intercepted your request.
+Please configure your client to use mitmproxy and try again.<br>
+The request to <a href="http://mitm.it/">http://mitm.it/</a> must go through your mitmproxy instance.

doc-src/certinstall/webapp.html

@@ -8,3 +8,6 @@ domain **mitm.it**. You should see something like this:
 
 Just click on the relevant icon, and then follow the setup instructions
 for the platform you're on.
+
+Make sure you aren't using a bandwith optimizer (like Google's Data Compression
+Proxy on Chrome for Android) or the page will not load.

doc-src/config.html

@@ -0,0 +1,86 @@
+
+
+Mitmproxy is configured through a set of files in the users ~/.mitmproxy
+directory. 
+
+<table class="table">
+	<tbody>
+		<tr>
+			<th>mitmproxy.conf</th> 
+			<td>Settings for the <b>mitmproxy</b>. This file can contain any options supported by mitmproxy.</td>
+		</tr>
+		<tr>
+			<th>mitmdump.conf</th> 
+			<td>Settings for the <b>mitmdump</b>. This file can contain any options supported by mitmdump.</td>
+		</tr>
+		<tr>
+			<th>common.conf</th>
+
+			<td>Settings shared between all command-line tools. Settings in
+			this file are over-ridden by those in the tool-specific
+			files. Only options shared by mitmproxy and mitmdump should be used in this file. </td>
+		</tr>
+	</tbody>
+</table>
+
+# Syntax
+
+## Comments
+
+<pre>
+# this is a comment
+; this is also a comment (.ini style)
+--- and this is a comment too (yaml style)
+</pre>
+
+## Key/Value pairs
+
+- Keys and values are case-sensitive
+- Whitespace is ignored
+- Lists are comma-delimited, and enclosed in square brackets
+
+<pre>
+name = value   # (.ini style) 
+name: value    # (yaml style)
+--name value   # (command-line option style)
+
+fruit = [apple, orange, lemon]
+indexes = [1, 12, 35 , 40]
+</pre>
+
+## Flags
+
+These are boolean options that take no value but true/false.
+
+<pre>
+name = true    # (.ini style)
+name
+--name 	 	   # (command-line option style)
+</pre>
+
+# Options
+
+The options available in the config files are precisely those available as
+command-line flags, with the key being the option's long name. To get a
+complete list of these, use the __--help__ option on each of the tools. Be
+careful to only specify common options in the __common.conf__ file -
+unsupported options in this file will be detected as an error on startup.
+
+# Examples
+
+## common.conf
+
+Note that __port__ is an option supported by all tools. 
+
+<pre class="code">
+port = 8080
+</pre>
+
+
+## mitmproxy.conf
+
+<pre class="code">
+palette = light
+</pre>
+
+

doc-src/custom-routing.txt

@@ -0,0 +1,24 @@
+# Adapted from http://tldp.org/HOWTO/TransparentProxy-6.html (6.2 Second method)
+# Note that the choice of firewall mark (3) and routing table (2) was fairly arbitrary.
+# If you are already using policy routing or firewall marking for some other purpose,
+# make sure you choose unique numbers here. Otherwise, don't worry about it.
+
+
+
+# On the router, run
+
+PROXY_IP=192.168.1.100
+TARGET_IP=192.168.1.110
+
+iptables -t mangle -A PREROUTING -j ACCEPT -p tcp -m multiport --dports 80,443 -s ! $TARGET_IP
+# Alternative to MITM the whole network:
+# iptables -t mangle -A PREROUTING -j ACCEPT -p tcp -m multiport --dports 80,443 -s  $PROXY_IP
+iptables -t mangle -A PREROUTING -j MARK --set-mark 3 -p tcp -m multiport --dports 80,443
+ip rule add fwmark 3 table 2
+ip route add default via $PROXY_IP dev br0 table 2
+
+
+
+# On the proxy machine, run
+
+iptables -A PREROUTING -t nat -i eth0 -p tcp -m multiport --dports 80,443 -j REDIRECT --to-port 8080

doc-src/dev/architecture.html

@@ -0,0 +1,8 @@
+To give you a better understanding of how mitmproxy works, mitmproxy's high-level architecture is detailed
+in the following graphic:
+
+<img src="@!urlTo('schematics/architecture.png')!@">
+
+<a href="@!urlTo('schematics/architecture.pdf')!@">(architecture.pdf)</a>
+<p>Please don't refrain from asking any further
+questions on the mailing list, the IRC channel or the GitHub issue tracker.</p>

doc-src/dev/index.py

@@ -2,5 +2,6 @@ from countershape import Page
 
 pages = [
      Page("testing.html", "Testing"),
+     Page("architecture.html", "Architecture"),
 #    Page("addingviews.html", "Writing Content Views"),
 ]

doc-src/features/forwardproxy.html

@@ -1,16 +0,0 @@
-
-In this mode, mitmproxy accepts proxy requests and unconditionally forwards all
-requests to a specified upstream server. This is in contrast to <a
-href="@!urlTo("reverseproxy.html")!@">reverse proxy mode</a>, in which
-mitmproxy forwards ordinary HTTP requests to an upstream server.
-
-<table class="table">
-    <tbody>
-        <tr>
-            <th width="20%">command-line</th> <td>-F http[s]://hostname[:port]</td>
-        </tr>
-        <tr>
-            <th>mitmproxy shortcut</th> <td><b>F</b></td>
-        </tr>
-    </tbody>
-</table>

doc-src/features/index.py

@@ -4,12 +4,16 @@ pages = [
     Page("anticache.html", "Anticache"),
     Page("clientreplay.html", "Client-side replay"),
     Page("filters.html", "Filter expressions"),
-    Page("forwardproxy.html", "Forward proxy mode"),
+    Page("passthrough.html", "Ignore Domains"),
+    Page("proxyauth.html", "Proxy Authentication"),
+    Page("replacements.html", "Replacements"),
+    Page("responsestreaming.html", "Response Streaming"),
+    Page("reverseproxy.html", "Reverse proxy mode"),
+    Page("socksproxy.html", "SOCKS Mode"),
     Page("setheaders.html", "Set Headers"),
     Page("serverreplay.html", "Server-side replay"),
     Page("sticky.html", "Sticky cookies and auth"),
-    Page("proxyauth.html", "Proxy Authentication"),
-    Page("replacements.html", "Replacements"),
-    Page("reverseproxy.html", "Reverse proxy mode"),
+    Page("tcpproxy.html", "TCP Proxy"),
     Page("upstreamcerts.html", "Upstream Certs"),
-]
+    Page("upstreamproxy.html", "Upstream proxy mode"),
+]

doc-src/features/passthrough.html

@@ -0,0 +1,81 @@
+There are two main reasons why you may want to exempt some traffic from mitmproxy's interception mechanism:
+
+- **Certificate pinning:** Some traffic is is protected using
+  [certificate pinning](https://security.stackexchange.com/questions/29988/what-is-certificate-pinning) and mitmproxy's
+  interception leads to errors. For example, Windows Update or the Apple App Store fail to work if mitmproxy is active.
+- **Convenience:** You really don't care about some parts of the traffic and just want them to go away.
+
+If you want to peek into (SSL-protected) non-HTTP connections, check out the [tcp proxy](@!urlTo("tcpproxy.html")!@) feature.
+If you want to ignore traffic from mitmproxy's processing because of large response bodies, take a look at the
+[response streaming](@!urlTo("responsestreaming.html")!@) feature.
+
+## How it works
+
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>--ignore regex</td>
+        </tr>
+        <tr>
+            <th>mitmproxy shortcut</th> <td><b>I</b></td>
+        </tr>
+    </tbody>
+</table>
+
+
+mitmproxy allows you to specify a regex which is matched against a <code>host:port</code> string (e.g. "example.com:443")
+to determine hosts that should be excluded. 
+
+There are two important quirks to consider:
+
+- **In transparent mode, the ignore pattern is matched against the IP.** While we usually infer the hostname from the 
+  Host header if the --host argument is passed to mitmproxy, we do not have access to this information before the SSL
+  handshake.
+- In regular mode, explicit HTTP requests are never ignored.[^explicithttp] The ignore pattern is applied on CONNECT
+  requests, which initiate HTTPS or clear-text WebSocket connections.
+
+
+### Tutorial
+
+If you just want to ignore one specific domain, there's usually a bulletproof method to do so:
+
+1. Run mitmproxy or mitmdump in verbose mode (-v) and observe the host:port information in the serverconnect 
+messages. mitmproxy will filter on these.
+2. Take the host:port string, surround it with ^ and $, escape all dots (. becomes \\.) 
+and use this as your ignore pattern:
+
+<pre class="terminal">
+$ mitmdump -v
+127.0.0.1:50588: clientconnect
+127.0.0.1:50588: request
+  -> CONNECT example.com:443 HTTP/1.1
+127.0.0.1:50588: Set new server address: example.com:443
+<span style="color: white">127.0.0.1:50588: serverconnect
+  -> example.com:443</span>
+^C
+$ <span style="color: white">mitmproxy --ignore ^example\.com:443$</span>    
+</pre>
+
+Here are some other examples for ignore patterns:
+<pre>
+# Exempt traffic from the iOS App Store (usually just works):
+--ignore apple.com:443  
+# "Correct" version without false-positives:
+--ignore ^(.+\.)?apple\.com:443$ 
+    
+# Ignore example.com on all ports, but no subdomains:
+--ignore ^example.com:
+    
+# Transparent mode:
+--ignore 17\.178\.96\.59:443
+# IP address range:
+--ignore 17\.178\.\d+\.\d+:443
+</pre>
+
+### See Also
+
+- [TCP Proxy](@!urlTo("tcpproxy.html")!@)
+- [Response Streaming](@!urlTo("responsestreaming.html")!@)
+
+[^explicithttp]: This stems from an limitation of explicit HTTP proxying: A single connection can be re-used for multiple target domains - a <code>GET http://example.com/</code> request may be followed by a <code>GET http://evil.com/</code> request on the same connection. If we start to ignore the connection after the first request, we would miss the relevant second one.

doc-src/features/responsestreaming.html

@@ -0,0 +1,54 @@
+By using mitmproxy's streaming feature, response contents can be passed to the client incrementally before they have been fully received by the proxy.
+This is especially useful for large binary files such as videos, where buffering the whole file slows down the client's browser.
+
+By default, mitmproxy will read the entire response, perform any indicated
+manipulations on it and then send the (possibly modified) response to
+the client. In some cases this is undesirable and you may wish to "stream"
+the reponse back to the client. When streaming is enabled, the response is
+not buffered on the proxy but directly sent back to the client instead.
+
+<h2>On the command-line</h2>
+
+Streaming can be enabled on the command line for all response bodies exceeding a certain size.  The SIZE argument understands
+k/m/g suffixes, e.g. 3m for 3 megabytes.
+
+<table class="table">
+    <tbody>
+    <tr>
+        <th width="20%">command-line</th>
+        <td>
+            --stream SIZE
+        </td>
+    </tr>
+    </tbody>
+</table>
+
+
+<h2>Caveats</h2>
+
+When response streaming is enabled, <strong>streamed response contents will not be
+    recorded or preserved in any way.</strong>
+
+When response streaming is enabled, the response body cannot be modified.
+
+<h2>Customizing Response Streaming</h2>
+
+You can also use an <a href="@!urlTo("scripting/inlinescripts.html")!@">inline script</a> to customize exactly
+which responses are streamed.
+
+Responses that should be tagged for streaming by setting their respective .stream attribute to True:
+
+$!example("examples/stream.py")!$
+
+
+<h2>Implementation Details</h2>
+
+When response streaming is enabled, portions of the code which would have otherwise performed changes
+on the response body will see an empty response body instead (<code>libmproxy.protocol.http.CONTENT_MISSING</code>). Any modifications will be ignored.
+
+Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a <code>Transfer-Encoding:
+    chunked</code> header, the response will be streamed one chunk at a time.
+
+### See Also
+
+- [Ignore Domains](@!urlTo("passthrough.html")!@)

doc-src/features/reverseproxy.html

@@ -1,19 +1,52 @@
 
 In reverse proxy mode, mitmproxy accepts standard HTTP requests and forwards
-them to the specified upstream server. This is in contrast to <a
-href="@!urlTo("forwardproxy.html")!@">forward proxy mode</a>, in which
-mitmproxy forwards HTTP proxy requests to an upstream server.
-
-Note that the displayed URL for flows in this mode will use the value of the
-__Host__ header field from the request, not the reverse proxy server. 
+them to the specified upstream server. This is in contrast to
+<a href="@!urlTo("upstreamproxy.html")!@">upstream proxy mode</a>, in which
+mitmproxy forwards HTTP proxy requests to an upstream proxy server.
 
 <table class="table">
     <tbody>
         <tr>
-            <th width="20%">command-line</th> <td>-P http[s]://hostname[:port]</td>
-        </tr>
-        <tr>
-            <th>mitmproxy shortcut</th> <td><b>P</b></td>
+            <th width="20%">command-line</th> <td>-R <i>schema</i>://hostname[:port]</td>
         </tr>
     </tbody>
 </table>
+
+Here, **schema** is one of http, https, http2https or https2http. The latter
+two extended schema specifications control the use of HTTP and HTTPS on
+mitmproxy and the upstream server. You can indicate that mitmproxy should use
+HTTP, and the upstream server uses HTTPS like this:
+
+	http2https://hostname:port
+
+And you can indicate that mitmproxy should use HTTPS while the upstream
+service uses HTTP like this:
+
+	https2http://hostname:port
+
+
+### Host Header
+
+In reverse proxy mode, mitmproxy does not rewrite the host header. While often useful, this
+may lead to issues with public web servers. For example, consider the following scenario:
+
+    $ python mitmdump -d -R http://example.com/ &
+    $ curl http://localhost:8080/
+
+    >> GET https://example.com/
+        Host: localhost:8080
+        User-Agent: curl/7.35.0
+        [...]
+
+    << 404 Not Found 345B
+
+Since the Host header doesn't match <samp>example.com</samp>, an error is returned.<br>
+There are two ways to solve this:
+<ol>
+    <li>Modify the hosts file of your OS so that example.com resolves to 127.0.0.1.</li>
+    <li>
+        Instruct mitmproxy to rewrite the host header by passing <kbd>&#8209;&#8209;setheader&nbsp;:~q:Host:example.com</kbd>.
+        However, keep in mind that absolute URLs within the returned document or HTTP redirects will cause the client application
+        to bypass the proxy.
+    </li>
+</ol>

doc-src/features/socksproxy.html

@@ -0,0 +1,10 @@
+
+In this mode, mitmproxy acts as a SOCKS5 proxy server.
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>--socks</td>
+        </tr>
+    </tbody>
+</table>

doc-src/features/tcpproxy.html

@@ -0,0 +1,30 @@
+WebSockets or other non-HTTP protocols are not supported by mitmproxy yet. However, you can exempt hostnames from
+processing, so that mitmproxy acts as a generic TCP forwarder. This feature is closely related to the
+[ignore domains](@!urlTo("passthrough.html")!@) functionality, but differs in two important aspects:
+
+- The raw TCP messages are printed to the event log.
+- SSL connections will be intercepted.
+
+Please note that message interception or modification are not possible yet.
+If you are not interested in the raw TCP messages, you should use the ignore domains feature.
+
+## How it works
+
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>--tcp HOST</td>
+        </tr>
+        <tr>
+            <th>mitmproxy shortcut</th> <td><b>T</b></td>
+        </tr>
+    </tbody>
+</table>
+
+For a detailed description on the structure of the hostname pattern, please refer to the [Ignore Domains](@!urlTo("passthrough.html")!@) feature.
+
+### See Also
+
+- [Ignore Domains](@!urlTo("passthrough.html")!@)
+- [Response Streaming](@!urlTo("responsestreaming.html")!@)

doc-src/features/upstreamproxy.html

@@ -0,0 +1,27 @@
+
+In this mode, mitmproxy accepts proxy requests and unconditionally forwards all
+requests to a specified upstream proxy server. This is in contrast to <a
+href="@!urlTo("reverseproxy.html")!@">reverse proxy mode</a>, in which
+mitmproxy forwards ordinary HTTP requests to an upstream server.
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>-U http://hostname[:port]</td>
+        </tr>
+    </tbody>
+</table>
+
+Here, **schema** is one of http, https, http2https or https2http. The latter
+two extended schema specifications control the use of HTTP and HTTPS on
+mitmproxy and the upstream server. You can indicate that mitmproxy should use
+HTTP, and the upstream server uses HTTPS like this:
+
+	http2https://hostname:port
+
+And you can indicate that mitmproxy should use HTTPS while the upstream
+service uses HTTP like this:
+
+	https2http://hostname:port
+
+

doc-src/index.html

@@ -1,4 +1,27 @@
 
-@!index_contents!@
+__mitmproxy__ is an interactive, SSL-capable man-in-the-middle proxy for HTTP
+with a console interface.
+
+__mitmdump__ is the command-line version of mitmproxy. Think tcpdump for HTTP.
+
+__libmproxy__ is the library that mitmproxy and mitmdump are built on.
+
+Documentation, tutorials and distribution packages can be found on the
+mitmproxy.org website:
+
+[mitmproxy.org](http://mitmproxy.org).
 
 
+Features
+--------
+
+- Intercept HTTP requests and responses and modify them on the fly.
+- Save complete HTTP conversations for later replay and analysis.
+- Replay the client-side of an HTTP conversations.
+- Replay HTTP responses of a previously recorded server.
+- Reverse proxy mode to forward traffic to a specified server.
+- Transparent proxy mode on OSX and Linux.
+- Make scripted changes to HTTP traffic using Python.
+- SSL certificates for interception are generated on the fly.
+- And much, much more.
+

doc-src/index.py

@@ -1,11 +1,16 @@
-import os, sys, datetime
+import os
+import sys
+import datetime
 import countershape
-from countershape import Page, Directory, PythonModule, markup, model
+from countershape import Page, Directory, markup, model
 import countershape.template
-sys.path.insert(0, "..")
+
+MITMPROXY_SRC = os.path.abspath(
+    os.path.expanduser(os.environ.get("MITMPROXY_SRC", ".."))
+)
+sys.path.insert(0, MITMPROXY_SRC)
 from libmproxy import filt, version
 
-MITMPROXY_SRC = os.environ.get("MITMPROXY_SRC", os.path.abspath(".."))
 ns.VERSION = version.VERSION
 
 if ns.options.website:
@@ -23,44 +28,22 @@ ns.docMaintainer = "Aldo Cortesi"
 ns.docMaintainerEmail = "aldo@corte.si"
 ns.copyright = u"\u00a9 mitmproxy project, %s" % datetime.date.today().year
 
+
 def mpath(p):
     p = os.path.join(MITMPROXY_SRC, p)
     return os.path.expanduser(p)
 
-with open(mpath("README.mkd")) as f:
-        readme = f.read()
-        ns.index_contents = readme.split("\n", 1)[1] #remove first line (contains build status)
 
 def example(s):
     d = file(mpath(s)).read().rstrip()
     extemp = """<div class="example">%s<div class="example_legend">(%s)</div></div>"""
     return extemp%(countershape.template.Syntax("py")(d), s)
+
+
 ns.example = example
 
 
-filt_help = []
-for i in filt.filt_unary:
-    filt_help.append(
-        ("~%s"%i.code, i.help)
-    )
-for i in filt.filt_rex:
-    filt_help.append(
-        ("~%s regex"%i.code, i.help)
-    )
-for i in filt.filt_int:
-    filt_help.append(
-        ("~%s int"%i.code, i.help)
-    )
-filt_help.sort()
-filt_help.extend(
-    [
-        ("!", "unary not"),
-        ("&", "and"),
-        ("|", "or"),
-        ("(...)", "grouping"),
-    ]
-)
-ns.filt_help = filt_help
+ns.filt_help = filt.help
 
 
 def nav(page, current, state):
@@ -73,12 +56,16 @@ def nav(page, current, state):
 ns.nav = nav
 ns.navbar = countershape.template.File(None, "_nav.html")
 
+
 pages = [
     Page("index.html", "Introduction"),
     Page("install.html", "Installation"),
+    Page("howmitmproxy.html", "How mitmproxy works"),
+    Page("modes.html", "Modes of Operation"),
+
     Page("mitmproxy.html", "mitmproxy"),
     Page("mitmdump.html", "mitmdump"),
-    Page("howmitmproxy.html", "How mitmproxy works"),
+    Page("config.html", "configuration"),
 
     Page("ssl.html", "Overview"),
     Directory("certinstall"),

doc-src/install.html

@@ -1,40 +1,33 @@
 
+## Installing from source
+
 The preferred way to install mitmproxy - whether you're installing the latest
 release or from source - is to use [pip](http://www.pip-installer.org/). If you
 don't already have pip on your system, you can find installation instructions
 [here](http://www.pip-installer.org/en/latest/installing.html).
 
-
-## Installing the latest release
-
-A single command will download and install the latest release of mitmproxy,
-along with all its dependencies:
-
 <pre class="terminal">
 pip install mitmproxy
 </pre>
 
+If you also want to install the optional packages AMF, protobuf and CSS
+content views, do this:
 
-## Installing from source
-
-When installing from source, the easiest method is still to use pip. In this
-case run:
-    
 <pre class="terminal">
-pip install /path/to/source
+pip install "mitmproxy[contentviews]"
 </pre>
 
-Note that if you're installing current git master, you will also have to
-install the current git master of [netlib](http://github.com/mitmproxy/netlib) by
-hand.
 
 ## OSX
 
+The easiest way to get up and running on OSX is to download the pre-built
+binary packages from [mitmproxy.org](http://mitmproxy.org). If you still want
+to install using pip, there are a few things to keep in mind:
+
 - If you're running a Python interpreter installed with homebrew (or similar),
 you may have to install some dependencies by hand. 
 - Make sure that XCode is installed from the App Store, and that the
 command-line tools have been downloaded (XCode/Preferences/Downloads).
-- Now use __pip__ to do the installation, as above.
 
 There are a few bits of customization you might want to do to make mitmproxy
 comfortable to use on OSX. The default color scheme is optimized for a dark
@@ -50,3 +43,17 @@ image/*; /usr/bin/open -Wn %s
 video/*; /usr/bin/open -Wn %s
 </pre>
 
+
+## Ubuntu
+
+On Ubuntu, you will need the following native packages to install mitmproxy
+from source:
+
+- build-essential
+- python-dev
+- libffi-dev
+- libssl-dev
+- libxml2-dev
+- libxslt1-dev
+
+

doc-src/modes.html

@@ -0,0 +1,222 @@
+
+Mitmproxy has four modes of operation that allow you to use mitmproxy in a
+variety of scenarios:
+
+- **Regular** (the default)
+- **Transparent**
+- **Reverse Proxy**
+- **Upstream Proxy**
+
+Now, which one should you pick? Use this flow chart:
+
+<img src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"/>
+
+<div class="page-header">
+    <h1>Regular Proxy</h1>
+</div>
+
+Mitmproxy's regular mode is the simplest and the easiest to set up.
+
+1. Start mitmproxy.
+2. Configure your client to use mitmproxy. For instance on IOS, the settings might look like <a href="@!urlTo('screenshots/ios-manual.png')!@">this</a>.
+3. Quick Check: You should already be able to visit an unencrypted HTTP site
+through the proxy.
+4. Open the magic domain <strong>mitm.it</strong> and install the certificate for your device.
+
+<div class="well">
+<strong>Heads Up:</strong> Unfortunately, some applications bypass the
+system HTTP proxy settings - Android applications are a common example. In
+these cases, you need to use mitmproxy's transparent mode.
+</div>
+
+If you are proxying an external device, your network will probably look like this:
+
+<img src="@!urlTo('schematics/proxy-modes-regular.png')!@">
+
+The square brackets signify the source and destination IP addresses. Your
+client explicitly connects to mitmproxy and mitmproxy explicitly connects
+to the target server.
+
+<div class="page-header">
+    <h1>Transparent Proxy</h1>
+</div>
+
+In transparent mode, traffic is directed into a proxy at the network layer,
+without any client configuration required. This makes transparent proxying
+ideal for situations where you can't change client behaviour. In the graphic
+below, a machine running mitmproxy has been inserted between the router and
+the internet:
+
+<a href="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
+</a>
+
+The square brackets signify the source and destination IP addresses. Round
+brackets mark the next hop on the *Ethernet/data link* layer. This distinction
+is important: when the packet arrives at the mitmproxy machine, it must still
+be addressed to the target server. This means that Network Address Translation
+should not be applied before the traffic reaches mitmproxy, since this would
+remove the target information, leaving mitmproxy unable to determine the real
+destination.
+
+<a href="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@"></a>
+
+<h2>Common Configurations</h2>
+
+There are many ways to configure your network for transparent proxying. We'll
+look at three common scenarios:
+
+1. Configuring the client to use a custom gateway/router/"next hop"
+2. Implementing custom routing on the router
+
+In most cases, the first option is recommended due to its ease of use.
+
+<h3>(a) Custom Gateway</h3>
+
+One simple way to get traffic to the mitmproxy machine with the destination IP
+intact, is to simply configure the client with the mitmproxy box as the
+default gateway.
+
+<a href="@!urlTo('schematics/proxy-modes-transparent-2.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-transparent-2.png')!@"></a>
+
+In this scenario, we would:
+
+- Configure the proxy machine for transparent mode. You can find instructions
+in the <em>Transparent Proxying</em> section of the mitmproxy docs.
+
+- Configure the client to use the proxy machine's IP as the default gateway.
+<a href="@!urlTo('screenshots/ios-gateway.png')!@">Here</a> is what this would
+look like on IOS.
+
+- Quick Check: At this point, you should already be able to visit an
+unencrypted HTTP site over the proxy.
+
+- Open the magic domain <strong>mitm.it</strong> and install the certificate
+for your device.
+
+Setting the custom gateway on clients can be automated by serving the settings
+out to clients over DHCP. This lets set up an interception network where all
+clients are proxied automatically, which can save time and effort. 
+
+
+<div class="well">
+    <strong style="text-align: center; display: block">Troubleshooting Transparent Mode</strong>
+
+    <p>Incorrect transparent mode configurations are a frequent source of
+    error. If it doesn't work for you, try the following things:</p>
+
+    <ul>
+        <li>
+            Open mitmproxy's event log (press `e`) - do you see clientconnect
+            messages? If not, the packets are not arriving at the proxy. One common
+            cause is the occurrence of ICMP redirects, which means that your
+            machine is telling the client that there's a faster way to the
+            internet by contacting your router directly (see the
+            <em>Transparent Proxying</em> section on how to disable them). If in
+            doubt, <a href="https://wireshark.org/">Wireshark</a> may help you
+            to see whether something arrives at your machine or not.
+        </li>
+        <li>
+            Make sure you have not explicitly configured an HTTP proxy on the
+            client. This is not needed in transparent mode.
+        </li>
+        <li>
+            Re-check the instructions in the <em>Transparent Proxying</em> section. Anything you missed?
+        </li>
+    </ul>
+
+    If you encounter any other pitfalls that should be listed here, please let us know!
+
+</div>
+
+<h3>(b) Custom Routing</h3>
+
+In some cases, you may need more fine-grained control of which traffic reaches
+the mitmproxy instance, and which doesn't. You may, for instance, choose only
+to divert traffic to some hosts into the transparent proxy. There are a huge
+number of ways to accomplish this, and much will depend on the router or
+packet filter you're using. In most cases, the configuration will look like
+this:
+
+<a href="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
+</a>
+
+
+<div class="page-header">
+    <h1>Reverse Proxy</h1>
+</div>
+
+Mitmproxy is usually used with a client that uses the proxy to access the
+Internet. Using reverse proxy mode, you can use mitmproxy to act like a normal
+HTTP server:
+
+<a href="@!urlTo('schematics/proxy-modes-reverse.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-reverse.png')!@">
+</a>
+
+There are various use-cases:
+
+- Say you have an internal API running at http://example.local/. You could now
+set up mitmproxy in reverse proxy mode at http://debug.example.local/ and
+dynamically point clients to this new API endpoint, which provides clients
+with the same data and you with debug information. Similarly, you could move
+your real server to a different IP/port and set up mitmproxy at the original
+place to debug all sessions.
+
+- Say you're a web developer working on example.com (with a development
+version running on localhost:8000). You can modify your hosts file so that
+example.com points to 127.0.0.1 and then run mitmproxy in reverse proxy mode
+on port 80. You can test your app on the example.com domain and get all
+requests recorded in mitmproxy.
+
+- Say you have some toy project that should get SSL support. Simply set up
+mitmproxy with SSL termination and you're done (<code>mitmdump -p 443 -R
+https2http://localhost:80/</code>). There are better tools for this specific
+task, but mitmproxy is very quick and simple way to set up an SSL-speaking
+server.
+
+- Want to add a non-SSL-capable compression proxy in front of your server? You
+could even spawn a mitmproxy instance that terminates SSL (https2http://...),
+point it to the compression proxy and let the compression proxy point to a
+SSL-initiating mitmproxy (http2https://...), which then points to the real
+server. As you see, it's a fairly flexible thing.
+
+Note that mitmproxy supports either an HTTP or an HTTPS upstream server, not
+both at the same time. You can work around this by spawning a second mitmproxy
+instance.
+
+<div class="well">
+    <strong style="text-align: center; display: block">Caveat: Interactive Use</strong>
+
+
+One caveat is that reverse proxy mode is often not sufficient for interactive
+browsing. Consider trying to clone Google by using:
+
+<code>mitmproxy -R http://google.com/</code>
+
+This works for the initial request, but the HTML served to the client remains
+unchanged. As soon as the user clicks on an non-relative URL (or downloads a
+non-relative image resource), traffic no longer passes through mitmproxy, and
+the client connects to Google directly again.
+
+</div>
+
+
+
+<div class="page-header">
+    <h1>Upstream Proxy</h1>
+</div>
+
+If you want to chain proxies by adding mitmproxy in front of a different proxy
+appliance, you can use mitmproxy's upstream mode. In upstream mode, all
+requests are unconditionally transferred to an upstream proxy of your choice.
+
+<a href="@!urlTo('schematics/proxy-modes-upstream.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-upstream.png')!@"></a>
+
+mitmproxy supports both explicit HTTP and explicit HTTPS in upstream proxy
+mode. You could in theory chain multiple mitmproxy instances in a row, but
+that doesn't make any sense in practice (i.e. outside of our tests). 

doc-src/scripting/inlinescripts.html

@@ -1,5 +1,5 @@
 __mitmproxy__ has a powerful scripting API that allows you to modify flows
-on-the-fly or rewrite previously saved flows locally. 
+on-the-fly or rewrite previously saved flows locally.
 
 The mitmproxy scripting API is event driven - a script is simply a Python
 module that exposes a set of event methods. Here's a complete mitmproxy script
@@ -21,6 +21,11 @@ We can now run this script using mitmdump or mitmproxy as follows:
 
 The new header will be added to all responses passing through the proxy.
 
+## Example Scripts
+
+mitmproxy comes with a variety of example inline scripts, which demonstrate
+many basic tasks. We encourage you to either browse them locally or in our
+[GitHub repo](https://github.com/mitmproxy/mitmproxy/tree/master/examples).
 
 
 ## Events
@@ -30,38 +35,45 @@ The new header will be added to all responses passing through the proxy.
 Called once on startup, before any other events.
 
 
-### clientconnect(ScriptContext, ClientConnect)
+### clientconnect(ScriptContext, ConnectionHandler)
 
 Called when a client initiates a connection to the proxy. Note that
 a connection can correspond to multiple HTTP requests.
 
-
-### serverconnect(ScriptContext, ServerConnection)
+### serverconnect(ScriptContext, ConnectionHandler)
 
 Called when the proxy initiates a connection to the target server. Note that
 a connection can correspond to multiple HTTP requests.
 
-### request(ScriptContext, Flow)
+### request(ScriptContext, HTTPFlow)
 
-Called when a client request has been received. The __Flow__ object is
+Called when a client request has been received. The __HTTPFlow__ object is
 guaranteed to have a non-None __request__ attribute.
 
+### responseheaders(ScriptContext, HTTPFlow)
 
-### response(ScriptContext, Flow)
+Called when the headers of a server response have been received.
+This will always be called before the response hook.
+The __HTTPFlow__ object is guaranteed to have non-None __request__ and
+__response__ attributes.  __response.content__ will be None,
+as the response body has not been read yet.
 
-Called when a server response has been received. The __Flow__ object is
+### response(ScriptContext, HTTPFlow)
+
+Called when a server response has been received. The __HTTPFlow__ object is
 guaranteed to have non-None __request__ and __response__ attributes.
+Note that if response streaming is enabled for this response,
+__response.content__ will not contain the response body.
 
-
-### error(ScriptContext, Flow)
+### error(ScriptContext, HTTPFlow)
 
 Called when a flow error has occurred, e.g. invalid server responses, or
 interrupted connections. This is distinct from a valid server HTTP error
-response, which is simply a response with an HTTP error code. The __Flow__
+response, which is simply a response with an HTTP error code. The __HTTPFlow__
 object is guaranteed to have non-None __request__ and __error__ attributes.
 
 
-### clientdisconnect(ScriptContext, ClientDisconnect)
+### clientdisconnect(ScriptContext, ConnectionHandler)
 
 Called when a client disconnects from the proxy.
 
@@ -76,43 +88,46 @@ The main classes you will deal with in writing mitmproxy scripts are:
 
 <table class="table">
     <tr>
-        <th>libmproxy.flow.ClientConnection</th>
+        <th>libmproxy.proxy.server.ConnectionHandler</th>
+        <td>Describes a proxy client connection session. Always has a client_conn attribute, might have a server_conn
+            attribute.
+        </td>
+    </tr>
+    <tr>
+        <th>libmproxy.proxy.connection.ClientConnection</th>
         <td>Describes a client connection.</td>
     </tr>
     <tr>
-        <th>libmproxy.flow.ClientDisconnection</th>
-        <td>Describes a client disconnection.</td>
+        <th>libmproxy.proxy.connection.ServerConnection</th>
+        <td>Describes a server connection.</td>
     </tr>
     <tr>
-        <th>libmproxy.flow.Error</th>
-        <td>A communications error.</td>
-    </tr>
-    <tr>
-        <th>libmproxy.flow.Flow</th>
+        <th>libmproxy.protocol.http.HTTPFlow</th>
         <td>A collection of objects representing a single HTTP transaction.</td>
     </tr>
     <tr>
-        <th>libmproxy.flow.Headers</th>
-        <td>HTTP headers for a request or response.</td>
+        <th>libmproxy.protocol.http.HTTPResponse</th>
+        <td>An HTTP response.</td>
+    </tr>
+    <tr>
+        <th>libmproxy.protocol.http.HTTPRequest</th>
+        <td>An HTTP request.</td>
+    </tr>
+    <tr>
+        <th>libmproxy.protocol.primitives.Error</th>
+        <td>A communications error.</td>
+    </tr>
+    <tr>
+        <th>libmproxy.script.ScriptContext</th>
+        <td> A handle for interacting with mitmproxy's from within scripts.</td>
     </tr>
     <tr>
         <th>libmproxy.flow.ODict</th>
 
         <td>A dictionary-like object for managing sets of key/value data. There
-        is also a variant called CaselessODict that ignores key case for some
-        calls (used mainly for headers).</td>
-    </tr>
-    <tr>
-        <th>libmproxy.flow.Response</th>
-        <td>An HTTP response.</td>
-    </tr>
-    <tr>
-        <th>libmproxy.flow.Request</th>
-        <td>An HTTP request.</td>
-    </tr>
-    <tr>
-        <th>libmproxy.flow.ScriptContext</th>
-        <td> A handle for interacting with mitmproxy's from within scripts.  </td>
+            is also a variant called CaselessODict that ignores key case for some
+            calls (used mainly for headers).
+        </td>
     </tr>
     <tr>
         <th>libmproxy.certutils.SSLCert</th>
@@ -120,11 +135,12 @@ The main classes you will deal with in writing mitmproxy scripts are:
     </tr>
 </table>
 
-The canonical API documentation is the code. You can view the API documentation
-using pydoc (which is installed with Python by default), like this:
+The canonical API documentation is the code, which you can browse locally or in our
+[GitHub repo](https://github.com/mitmproxy/mitmproxy).
+You can view the API documentation using pydoc (which is installed with Python by default), like this:
 
 <pre class="terminal">
-> pydoc libmproxy.flow.Request
+> pydoc libmproxy.protocol.http.HTTPRequest
 </pre>
 
 
@@ -135,6 +151,13 @@ While that's a very desirable behaviour under some circumstances, scripts can be
 
 $!example("examples/nonblocking.py")!$
 
+## Make scripts configurable with arguments
+
+Sometimes, you want to pass runtime arguments to the inline script. This can be simply done by surrounding the script call with quotes, e.g.
+<code>mitmdump -s "script.py --foo 42"</code>. The arguments are then exposed in the start event:
+
+$!example("examples/modify_response_body.py")!$
+
 ## Running scripts on saved flows
 
 Sometimes, we want to run a script on __Flow__ objects that are already
@@ -144,6 +167,11 @@ flows from a file (see the "scripted data transformation" example on the
 one-shot script on a single flow through the _|_ (pipe) shortcut in mitmproxy.
 
 In this case, there are no client connections, and the events are run in the
-following order: __start__, __request__, __response__, __error__, __done__.  If
+following order: __start__, __request__, __responseheaders__, __response__, __error__, __done__.  If
 the flow doesn't have a __response__ or __error__ associated with it, the
-matching event will be skipped. 
+matching events will be skipped.
+
+## Spaces in the script path
+By default, spaces are interpreted as separator between the inline script and its arguments (e.g. <code>-s "foo.py
+    42"</code>). Consequently, the script path needs to be wrapped in a separate pair of quotes if it contains spaces:
+<code>-s "'./foo bar/baz.py' 42"</code>.

doc-src/scripting/libmproxy.html

@@ -1,3 +1,13 @@
+<div class="well">
+    <strong>Heads up!</strong> We strongly encourage you to use <a href="@!urlTo("scripting/inlinescripts.html")!@">inline scripts</a> rather than libmproxy
+    directly.<br><br>
+    <ul>
+        <li>Inline Scripts are equally powerful and provide an easier syntax.</li>
+        <li>Most examples are written as inline scripts.</li>
+        <li>Multiple inline scripts can be combined and used together.</li>
+        <li>Inline Scripts can either be executed headless with mitmdump or within the mitmproxy UI.</li>
+    </ul>
+</div>
 
 All of mitmproxy's basic functionality is exposed through the __libmproxy__
 library. The example below shows a simple implementation of the "sticky cookie"

doc-src/ssl.html

@@ -41,10 +41,26 @@ The files created by mitmproxy in the .mitmproxy directory are as follows:
 Using a custom certificate
 --------------------------
 
-You can use your own certificate by passing the __--cert__ option to mitmproxy.
+You can use your own certificate by passing the <kbd>--cert</kbd> option to mitmproxy. mitmproxy then uses the provided
+certificate for interception of the specified domains instead of generating a cert signed by its own CA.
 
-The certificate file is expected to be in the PEM format. You can generate
-a certificate in this format using these instructions:
+The certificate file is expected to be in the PEM format.
+You can include intermediary certificates right below your leaf certificate, so that you PEM file roughly looks like
+this:
+
+<pre>
+-----BEGIN PRIVATE KEY-----
+&lt;private key&gt;
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+&lt;cert&gt;
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+&lt;intermediary cert (optional)&gt;
+-----END CERTIFICATE-----
+</pre>
+
+For example, you can generate a certificate in this format using these instructions:
 
 <pre class="terminal">
 > openssl genrsa -out cert.key 8192
@@ -55,6 +71,15 @@ a certificate in this format using these instructions:
 </pre>
 
 
+Using a custom certificate authority
+------------------------------------
+
+By default, mitmproxy will (generate and) use <samp>~/.mitmproxy/mitmproxy-ca.pem</samp> as the default certificate
+authority to generate certificates for all domains for which no custom certificate is provided (see above).
+You can use your own certificate authority by passing the <kbd>--confdir</kbd> option to mitmproxy.
+mitmproxy will then look for <samp>mitmproxy-ca.pem</samp> in the specified directory. If no such file exists,
+it will be generated automatically.
+
 Installing the mitmproxy CA
 ---------------------------
 

doc-src/transparent/linux.html

@@ -15,6 +15,16 @@ achieve transparent mode.
 
     </li>
 
+    <li> If your target machine is on the same physical network and you configured it to use a custom gateway,
+    disable ICMP redirects:
+
+    <pre class="terminal">echo 0 | sudo tee /proc/sys/net/ipv4/conf/*/send_redirects</pre>
+
+    You may also want to consider enabling this permanently in
+    <b>/etc/sysctl.conf</b> as demonstrated <a href="http://unix.stackexchange.com/a/58081">here</a>.
+
+    </li>
+
     <li> Create an iptables ruleset that redirects the desired traffic to the
     mitmproxy port. Details will differ according to your setup, but the
     ruleset should look something like this:

doc-src/tutorials/gamecenter.html

@@ -67,8 +67,8 @@ timestamp. Looks pretty simple to mess with.
 Lets edit the score submission. First, select it in mitmproxy, then press
 __enter__ to view it. Make sure you're viewing the request, not the response -
 you can use __tab__ to flick between the two. Now press __e__ for edit. You'll
-be prompted for the part of the request you want to change - press __b__ for
-body.  Your preferred editor (taken from the EDITOR environment variable) will
+be prompted for the part of the request you want to change - press __r__ for
+raw body.  Your preferred editor (taken from the EDITOR environment variable) will
 now fire up. Lets bump the score up to something a bit more ambitious:
 
 <!--(block|syntax("xml"))-->

examples/README

@@ -1,10 +1,23 @@
-add_header.py           Simple script that just adds a header to every request.
-dup_and_replay.py       Duplicates each request, changes it, and then replays the modified request.
-flowbasic               Basic use of mitmproxy as a library.
-modify_form.py          Modify all form submissions to add a parameter.
-modify_querystring.py   Modify all query strings to add a parameters.
-proxapp                 How to embed a WSGI app in a mitmproxy server
-stub.py                 Script stub with a method definition for every event.
-stickycookies           An example of writing a custom proxy with libmproxy.
-upsidedownternet.py     Rewrites traffic to turn PNGs upside down.
-mitmproxywrapper.py     Bracket mitmproxy run with proxy enable/disable on OS X
+# inline script examples
+add_header.py             Simple script that just adds a header to every request.
+change_upstream_proxy.py  Dynamically change the upstream proxy
+dup_and_replay.py         Duplicates each request, changes it, and then replays the modified request.
+iframe_injector.py        Inject configurable iframe into pages.
+modify_form.py            Modify all form submissions to add a parameter.
+modify_querystring.py     Modify all query strings to add a parameters.
+modify_response_body.py   Replace arbitrary strings in all responses
+nonblocking.py            Demonstrate parallel processing with a blocking script.
+proxapp.py                How to embed a WSGI app in a mitmproxy server
+redirect_requests.py      Redirect requests or directly reply to them.
+stub.py                   Script stub with a method definition for every event.
+upsidedownternet.py       Rewrites traffic to turn images upside down.
+
+
+# libmproxy examples
+flowbasic                 Basic use of mitmproxy as a library.
+stickycookies             An example of writing a custom proxy with libmproxy.
+
+
+# misc
+read_dumpfile             Read a dumpfile generated by mitmproxy.
+mitmproxywrapper.py       Bracket mitmproxy run with proxy enable/disable on OS X

examples/add_header.py

@@ -1,2 +1,2 @@
 def response(context, flow):
-    flow.response.headers["newheader"] = ["foo"]
+    flow.response.headers["newheader"] = ["foo"]

examples/change_upstream_proxy.py

@@ -0,0 +1,21 @@
+# This scripts demonstrates how mitmproxy can switch to a second/different upstream proxy
+# in upstream proxy mode.
+#
+# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s "change_upstream_proxy.py host"
+from libmproxy.protocol.http import send_connect_request
+
+alternative_upstream_proxy = ("localhost", 8082)
+def should_redirect(flow):
+    return flow.request.host == "example.com"
+
+
+def request(context, flow):
+    if flow.live and should_redirect(flow):
+
+        # If you want to change the target server, you should modify flow.request.host and flow.request.port
+        # flow.live.change_server should only be used by inline scripts to change the upstream proxy,
+        # unless you are sure that you know what you are doing.
+        server_changed = flow.live.change_server(alternative_upstream_proxy, persistent_change=True)
+        if flow.request.scheme == "https" and server_changed:
+            send_connect_request(flow.live.c.server_conn, flow.request.host, flow.request.port)
+            flow.live.c.establish_ssl(server=True)

examples/dup_and_replay.py

@@ -1,4 +1,4 @@
-def request(ctx, flow):
-   f = ctx.duplicate_flow(flow)
-   f.request.path = "/changed"
-   ctx.replay_request(f)
+def request(context, flow):
+    f = context.duplicate_flow(flow)
+    f.request.path = "/changed"
+    context.replay_request(f)

examples/flowbasic

@@ -3,11 +3,15 @@
     This example shows how to build a proxy based on mitmproxy's Flow
     primitives.
 
+    Heads Up: In the majority of cases, you want to use inline scripts.
+
     Note that request and response messages are not automatically replied to,
     so we need to implement handlers to do this.
 """
 import os
-from libmproxy import proxy, flow
+from libmproxy import flow, proxy
+from libmproxy.proxy.server import ProxyServer
+
 
 class MyMaster(flow.FlowMaster):
     def run(self):
@@ -16,24 +20,25 @@ class MyMaster(flow.FlowMaster):
         except KeyboardInterrupt:
             self.shutdown()
 
-    def handle_request(self, r):
-        f = flow.FlowMaster.handle_request(self, r)
+    def handle_request(self, f):
+        f = flow.FlowMaster.handle_request(self, f)
         if f:
-            r.reply()
+            f.reply()
         return f
 
-    def handle_response(self, r):
-        f = flow.FlowMaster.handle_response(self, r)
+    def handle_response(self, f):
+        f = flow.FlowMaster.handle_response(self, f)
         if f:
-            r.reply()
+            f.reply()
         print f
         return f
 
 
 config = proxy.ProxyConfig(
-    cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
+    port=8080,
+    cadir="~/.mitmproxy/"  # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file.
 )
 state = flow.State()
-server = proxy.ProxyServer(config, 8080)
+server = ProxyServer(config)
 m = MyMaster(server, state)
 m.run()

examples/har_extractor.py

@@ -0,0 +1,212 @@
+"""
+    This inline script utilizes harparser.HAR from https://github.com/JustusW/harparser
+    to generate a HAR log object.
+"""
+try:
+    from harparser import HAR
+    from pytz import UTC
+except ImportError as e:
+    import sys
+    print >> sys.stderr, "\r\nMissing dependencies: please run `pip install mitmproxy[examples]`.\r\n"
+    raise
+
+from datetime import datetime, timedelta, tzinfo
+
+
+class _HARLog(HAR.log):
+    # The attributes need to be registered here for them to actually be available later via self. This is
+    # due to HAREncodable linking __getattr__ to __getitem__. Anything that is set only in __init__ will
+    # just be added as key/value pair to self.__classes__.
+    __page_list__ = []
+    __page_count__ = 0
+    __page_ref__ = {}
+
+    def __init__(self, page_list):
+        self.__page_list__ = page_list
+        self.__page_count__ = 0
+        self.__page_ref__ = {}
+
+        HAR.log.__init__(self, {"version": "1.2",
+                                "creator": {"name": "MITMPROXY HARExtractor",
+                                            "version": "0.1",
+                                            "comment": ""},
+                                "pages": [],
+                                "entries": []})
+
+    def reset(self):
+        self.__init__(self.__page_list__)
+
+    def add(self, obj):
+        if isinstance(obj, HAR.pages):
+            self['pages'].append(obj)
+        if isinstance(obj, HAR.entries):
+            self['entries'].append(obj)
+
+    def create_page_id(self):
+        self.__page_count__ += 1
+        return "autopage_%s" % str(self.__page_count__)
+
+    def set_page_ref(self, page, ref):
+        self.__page_ref__[page] = ref
+
+    def get_page_ref(self, page):
+        return self.__page_ref__.get(page, None)
+
+    def get_page_list(self):
+        return self.__page_list__
+
+
+def start(context, argv):
+    """
+        On start we create a HARLog instance. You will have to adapt this to suit your actual needs
+        of HAR generation. As it will probably be necessary to cluster logs by IPs or reset them
+        from time to time.
+    """
+    context.dump_file = None
+    if len(argv) > 1:
+        context.dump_file = argv[1]
+    else:
+        raise ValueError('Usage: -s "har_extractor.py filename" '
+                         '(- will output to stdout, filenames ending with .zhar will result in compressed har)')
+    context.HARLog = _HARLog(['https://github.com'])
+    context.seen_server = set()
+
+
+def response(context, flow):
+    """
+       Called when a server response has been received. At the time of this message both
+       a request and a response are present and completely done.
+    """
+    # Values are converted from float seconds to int milliseconds later.
+    ssl_time = -.001
+    connect_time = -.001
+    if flow.server_conn not in context.seen_server:
+        # Calculate the connect_time for this server_conn. Afterwards add it to seen list, in
+        # order to avoid the connect_time being present in entries that use an existing connection.
+        connect_time = flow.server_conn.timestamp_tcp_setup - flow.server_conn.timestamp_start
+        context.seen_server.add(flow.server_conn)
+
+        if flow.server_conn.timestamp_ssl_setup is not None:
+            # Get the ssl_time for this server_conn as the difference between the start of the successful
+            # tcp setup and the successful ssl setup. If  no ssl setup has been made it is left as -1 since
+            # it doesn't apply to this connection.
+            ssl_time = flow.server_conn.timestamp_ssl_setup - flow.server_conn.timestamp_tcp_setup
+
+    # Calculate the raw timings from the different timestamps present in the request and response object.
+    # For lack of a way to measure it dns timings can not be calculated. The same goes for HAR blocked:
+    # MITMProxy will open a server connection as soon as it receives the host and port from the client
+    # connection. So the time spent waiting is actually spent waiting between request.timestamp_end and
+    # response.timestamp_start thus it correlates to HAR wait instead.
+    timings_raw = {'send': flow.request.timestamp_end - flow.request.timestamp_start,
+                   'wait': flow.response.timestamp_start - flow.request.timestamp_end,
+                   'receive': flow.response.timestamp_end - flow.response.timestamp_start,
+                   'connect': connect_time,
+                   'ssl': ssl_time}
+
+    # HAR timings are integers in ms, so we have to re-encode the raw timings to that format.
+    timings = dict([(key, int(1000 * value)) for key, value in timings_raw.iteritems()])
+
+    # The full_time is the sum of all timings. Timings set to -1 will be ignored as per spec.
+    full_time = 0
+    for item in timings.values():
+        if item > -1:
+            full_time += item
+
+    started_date_time = datetime.fromtimestamp(flow.request.timestamp_start, tz=utc).isoformat()
+
+    request_query_string = [{"name": k, "value": v} for k, v in flow.request.get_query()]
+    request_http_version = ".".join([str(v) for v in flow.request.httpversion])
+    # Cookies are shaped as tuples by MITMProxy.
+    request_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.request.get_cookies() or {}).iteritems()]
+    request_headers = [{"name": k, "value": v} for k, v in flow.request.headers]
+    request_headers_size = len(str(flow.request.headers))
+    request_body_size = len(flow.request.content)
+
+    response_http_version = ".".join([str(v) for v in flow.response.httpversion])
+    # Cookies are shaped as tuples by MITMProxy.
+    response_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.response.get_cookies() or {}).iteritems()]
+    response_headers = [{"name": k, "value": v} for k, v in flow.response.headers]
+    response_headers_size = len(str(flow.response.headers))
+    response_body_size = len(flow.response.content)
+    response_body_decoded_size = len(flow.response.get_decoded_content())
+    response_body_compression = response_body_decoded_size - response_body_size
+    response_mime_type = flow.response.headers.get_first('Content-Type', '')
+    response_redirect_url = flow.response.headers.get_first('Location', '')
+
+    entry = HAR.entries({"startedDateTime": started_date_time,
+                         "time": full_time,
+                         "request": {"method": flow.request.method,
+                                     "url": flow.request.url,
+                                     "httpVersion": request_http_version,
+                                     "cookies": request_cookies,
+                                     "headers": request_headers,
+                                     "queryString": request_query_string,
+                                     "headersSize": request_headers_size,
+                                     "bodySize": request_body_size, },
+                         "response": {"status": flow.response.code,
+                                      "statusText": flow.response.msg,
+                                      "httpVersion": response_http_version,
+                                      "cookies": response_cookies,
+                                      "headers": response_headers,
+                                      "content": {"size": response_body_size,
+                                                  "compression": response_body_compression,
+                                                  "mimeType": response_mime_type},
+                                      "redirectURL": response_redirect_url,
+                                      "headersSize": response_headers_size,
+                                      "bodySize": response_body_size, },
+                         "cache": {},
+                         "timings": timings, })
+
+    # If the current url is in the page list of context.HARLog or does not have a referrer we add it as a new
+    # pages object.
+    if flow.request.url in context.HARLog.get_page_list() or flow.request.headers.get('Referer', None) is None:
+        page_id = context.HARLog.create_page_id()
+        context.HARLog.add(HAR.pages({"startedDateTime": entry['startedDateTime'],
+                                      "id": page_id,
+                                      "title": flow.request.url, }))
+        context.HARLog.set_page_ref(flow.request.url, page_id)
+        entry['pageref'] = page_id
+
+    # Lookup the referer in the page_ref of context.HARLog to point this entries pageref attribute to the right
+    # pages object, then set it as a new reference to build a reference tree.
+    elif context.HARLog.get_page_ref(flow.request.headers.get('Referer', (None, ))[0]) is not None:
+        entry['pageref'] = context.HARLog.get_page_ref(flow.request.headers['Referer'][0])
+        context.HARLog.set_page_ref(flow.request.headers['Referer'][0], entry['pageref'])
+
+    context.HARLog.add(entry)
+
+
+def done(context):
+    """
+        Called once on script shutdown, after any other events.
+    """
+    from pprint import pprint
+    import json
+
+    json_dump = context.HARLog.json()
+    compressed_json_dump = context.HARLog.compress()
+
+    print "=" * 100
+    if context.dump_file == '-':
+        pprint(json.loads(json_dump))
+    elif context.dump_file.endswith('.zhar'):
+        file(context.dump_file, "w").write(compressed_json_dump)
+    else:
+        file(context.dump_file, "w").write(json_dump)
+    print "=" * 100
+    print "HAR log finished with %s bytes (%s bytes compressed)" % (len(json_dump), len(compressed_json_dump))
+    print "Compression rate is %s%%" % str(100. * len(compressed_json_dump) / len(json_dump))
+    print "=" * 100
+
+
+def print_attributes(obj, filter_string=None, hide_privates=False):
+    """
+        Useful helper method to quickly get all attributes of an object and its values.
+    """
+    for attr in dir(obj):
+        if hide_privates and "__" in attr:
+            continue
+        if filter_string is not None and filter_string not in attr:
+            continue
+        value = getattr(obj, attr)
+        print "%s.%s" % ('obj', attr), value, type(value)

examples/iframe_injector

@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-"""
-    Zap encoding in requests and inject iframe after body tag in html responses.
-    Usage:
-        iframe_injector http://someurl/somefile.html
-"""
-from libmproxy import controller, proxy
-import os
-import sys
-
-
-class InjectingMaster(controller.Master):
-    def __init__(self, server, iframe_url):
-        controller.Master.__init__(self, server)
-        self._iframe_url = iframe_url
-
-    def run(self):
-        try:
-            return controller.Master.run(self)
-        except KeyboardInterrupt:
-            self.shutdown()
-
-    def handle_request(self, msg):
-        if 'Accept-Encoding' in msg.headers:
-            msg.headers["Accept-Encoding"] = 'none'
-        msg.reply()
-
-    def handle_response(self, msg):
-        if msg.content:
-            c = msg.replace('<body>', '<body><iframe src="%s" frameborder="0" height="0" width="0"></iframe>' % self._iframe_url)
-            if c > 0:
-                print 'Iframe injected!'
-        msg.reply()
-
-
-def main(argv):
-    if len(argv) != 2:
-        print "Usage: %s IFRAME_URL" % argv[0]
-        sys.exit(1)
-    iframe_url = argv[1]
-    config = proxy.ProxyConfig(
-        cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
-    )
-    server = proxy.ProxyServer(config, 8080)
-    print 'Starting proxy...'
-    m = InjectingMaster(server, iframe_url)
-    m.run()
-
-if __name__ == '__main__':
-    main(sys.argv)

examples/iframe_injector.py

@@ -0,0 +1,22 @@
+# Usage: mitmdump -s "iframe_injector.py url"
+# (this script works best with --anticache)
+from bs4 import BeautifulSoup
+from libmproxy.protocol.http import decoded
+
+
+def start(context, argv):
+    if len(argv) != 2:
+        raise ValueError('Usage: -s "iframe_injector.py url"')
+    context.iframe_url = argv[1]
+
+
+def response(context, flow):
+    if flow.request.host in context.iframe_url:
+        return
+    with decoded(flow.response):  # Remove content encoding (gzip, ...)
+        html = BeautifulSoup(flow.response.content)
+        if html.body:
+            iframe = html.new_tag("iframe", src=context.iframe_url, frameborder=0, height=0, width=0)
+            html.body.insert(0, iframe)
+            flow.response.content = str(html)
+            context.log("Iframe inserted.")

examples/ignore_websocket.py

@@ -0,0 +1,34 @@
+# This script makes mitmproxy switch to passthrough mode for all HTTP
+# responses with "Connection: Upgrade" header. This is useful to make
+# WebSockets work in untrusted environments.
+#
+# Note: Chrome (and possibly other browsers), when explicitly configured
+# to use a proxy (i.e. mitmproxy's regular mode), send a CONNECT request
+# to the proxy before they initiate the websocket connection.
+# To make WebSockets work in these cases, supply
+# `--ignore :80$` as an additional parameter.
+# (see http://mitmproxy.org/doc/features/passthrough.html)
+
+from libmproxy.protocol.http import HTTPRequest
+from libmproxy.protocol.tcp import TCPHandler
+from libmproxy.protocol import KILL
+from libmproxy.script import concurrent
+
+
+def start(context, argv):
+    HTTPRequest._headers_to_strip_off.remove("Connection")
+    HTTPRequest._headers_to_strip_off.remove("Upgrade")
+
+
+def done(context):
+    HTTPRequest._headers_to_strip_off.append("Connection")
+    HTTPRequest._headers_to_strip_off.append("Upgrade")
+
+@concurrent
+def response(context, flow):
+    if flow.response.headers.get_first("Connection", None) == "Upgrade":
+        # We need to send the response manually now...
+        flow.client_conn.send(flow.response.assemble())
+        # ...and then delegate to tcp passthrough.
+        TCPHandler(flow.live.c, log=False).handle_messages()
+        flow.reply(KILL)

examples/mitmproxywrapper.py

@@ -86,7 +86,6 @@ class Wrapper(object):
             cmd = ['mitmproxy', '-p', str(self.port)]
             if self.extra_arguments:
                 cmd.extend(self.extra_arguments)
-            cmd.extend(['--palette', 'light'])
             subprocess.check_call(cmd)
 
     def wrap_honeyproxy(self):

examples/modify_form.py

@@ -1,8 +1,6 @@
 
 def request(context, flow):
     if "application/x-www-form-urlencoded" in flow.request.headers["content-type"]:
-        frm = flow.request.get_form_urlencoded()
-        frm["mitmproxy"] = ["rocks"]
-        flow.request.set_form_urlencoded(frm)
-
-
+        form = flow.request.get_form_urlencoded()
+        form["mitmproxy"] = ["rocks"]
+        flow.request.set_form_urlencoded(form)

examples/modify_querystring.py

@@ -3,5 +3,4 @@ def request(context, flow):
     q = flow.request.get_query()
     if q:
         q["mitmproxy"] = ["rocks"]
-        flow.request.set_query(q)
-
+        flow.request.set_query(q)

examples/modify_response_body.py

@@ -0,0 +1,15 @@
+# Usage: mitmdump -s "modify_response_body.py mitmproxy bananas"
+# (this script works best with --anticache)
+from libmproxy.protocol.http import decoded
+
+
+def start(context, argv):
+    if len(argv) != 3:
+        raise ValueError('Usage: -s "modify-response-body.py old new"')
+    # You may want to use Python's argparse for more sophisticated argument parsing.
+    context.old, context.new = argv[1], argv[2]
+
+
+def response(context, flow):
+    with decoded(flow.response):  # automatically decode gzipped responses.
+        flow.response.content = flow.response.content.replace(context.old, context.new)

examples/nonblocking.py

@@ -1,8 +1,9 @@
 import time
 from libmproxy.script import concurrent
 
-@concurrent
+
+@concurrent  # Remove this and see what happens
 def request(context, flow):
     print "handle request: %s%s" % (flow.request.host, flow.request.path)
     time.sleep(5)
-    print "start  request: %s%s" % (flow.request.host, flow.request.path)
+    print "start  request: %s%s" % (flow.request.host, flow.request.path)

examples/proxapp

@@ -1,47 +0,0 @@
-#!/usr/bin/env python
-"""
-This example shows how to graft a WSGI app onto mitmproxy. In this
-instance, we're using the Bottle framework (http://bottlepy.org/) to expose
-a single simplest-possible page.
-"""
-import bottle
-import os
-from libmproxy import proxy, flow
-
-@bottle.route('/')
-def index():
-    return 'Hi!'
-
-
-class MyMaster(flow.FlowMaster):
-    def run(self):
-        try:
-            flow.FlowMaster.run(self)
-        except KeyboardInterrupt:
-            self.shutdown()
-
-    def handle_request(self, r):
-        f = flow.FlowMaster.handle_request(self, r)
-        if f:
-            r.reply()
-        return f
-
-    def handle_response(self, r):
-        f = flow.FlowMaster.handle_response(self, r)
-        if f:
-            r.reply()
-        print f
-        return f
-
-
-config = proxy.ProxyConfig(
-    cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
-)
-state = flow.State()
-server = proxy.ProxyServer(config, 8080)
-# Register the app using the magic domain "proxapp" on port 80. Requests to
-# this domain and port combination will now be routed to the WSGI app instance.
-server.apps.add(bottle.app(), "proxapp", 80)
-m = MyMaster(server, state)
-m.run()
-

examples/proxapp.py

@@ -0,0 +1,24 @@
+"""
+This example shows how to graft a WSGI app onto mitmproxy. In this
+instance, we're using the Flask framework (http://flask.pocoo.org/) to expose
+a single simplest-possible page.
+"""
+from flask import Flask
+
+app = Flask("proxapp")
+
+
+@app.route('/')
+def hello_world():
+    return 'Hello World!'
+
+
+# Register the app using the magic domain "proxapp" on port 80. Requests to
+# this domain and port combination will now be routed to the WSGI app instance.
+def start(context, argv):
+    context.app_registry.add(app, "proxapp", 80)
+
+    # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design.
+    # mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set)
+    # but won't send any data.
+    context.app_registry.add(app, "example.com", 443)

examples/read_dumpfile

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+#
+# Simple script showing how to read a mitmproxy dump file
+#
+
+from libmproxy import flow
+import json, sys
+
+with open("logfile", "rb") as logfile:
+    freader = flow.FlowReader(logfile)
+    try:
+        for f in freader.stream():
+            print(f)
+            print(f.request.host)
+            json.dump(f.get_state(), sys.stdout, indent=4)
+            print ""
+    except flow.FlowReadError, v:
+        print "Flow file corrupted. Stopped loading."

examples/redirect_requests.py

@@ -1,19 +1,24 @@
-from libmproxy.flow import Response
+from libmproxy.protocol.http import HTTPResponse
 from netlib.odict import ODictCaseless
 
 """
 This example shows two ways to redirect flows to other destinations.
 """
 
+
 def request(context, flow):
-    if flow.request.host.endswith("example.com"):
-        resp = Response(flow.request,
-                        [1,1],
-                        200, "OK",
-                        ODictCaseless([["Content-Type","text/html"]]),
-                        "helloworld",
-                        None)
-        flow.request.reply(resp)
-    if flow.request.host.endswith("example.org"):
+    # pretty_host(hostheader=True) takes the Host: header of the request into account,
+    # which is useful in transparent mode where we usually only have the IP otherwise.
+
+    # Method 1: Answer with a locally generated response
+    if flow.request.pretty_host(hostheader=True).endswith("example.com"):
+        resp = HTTPResponse(
+            [1, 1], 200, "OK",
+            ODictCaseless([["Content-Type", "text/html"]]),
+            "helloworld")
+        flow.reply(resp)
+
+    # Method 2: Redirect the request to a different server
+    if flow.request.pretty_host(hostheader=True).endswith("example.org"):
         flow.request.host = "mitmproxy.org"
-        flow.request.headers["Host"] = ["mitmproxy.org"]
+        flow.request.update_host_header()

examples/stickycookies

@@ -1,12 +1,14 @@
 #!/usr/bin/env python
 """
 This example builds on mitmproxy's base proxying infrastructure to
-implement functionality similar to the "sticky cookies" option. This is at
-a lower level than the Flow mechanism, so we're dealing directly with
-request and response objects.
+implement functionality similar to the "sticky cookies" option.
+
+Heads Up: In the majority of cases, you want to use inline scripts.
 """
-from libmproxy import controller, proxy
 import os
+from libmproxy import controller, proxy
+from libmproxy.proxy.server import ProxyServer
+
 
 class StickyMaster(controller.Master):
     def __init__(self, server):
@@ -19,24 +21,22 @@ class StickyMaster(controller.Master):
         except KeyboardInterrupt:
             self.shutdown()
 
-    def handle_request(self, msg):
-        hid = (msg.host, msg.port)
-        if msg.headers["cookie"]:
-            self.stickyhosts[hid] = msg.headers["cookie"]
+    def handle_request(self, flow):
+        hid = (flow.request.host, flow.request.port)
+        if flow.request.headers["cookie"]:
+            self.stickyhosts[hid] = flow.request.headers["cookie"]
         elif hid in self.stickyhosts:
-            msg.headers["cookie"] = self.stickyhosts[hid]
-        msg.reply()
+            flow.request.headers["cookie"] = self.stickyhosts[hid]
+        flow.reply()
 
-    def handle_response(self, msg):
-        hid = (msg.request.host, msg.request.port)
-        if msg.headers["set-cookie"]:
-            self.stickyhosts[hid] = msg.headers["set-cookie"]
-        msg.reply()
+    def handle_response(self, flow):
+        hid = (flow.request.host, flow.request.port)
+        if flow.response.headers["set-cookie"]:
+            self.stickyhosts[hid] = flow.response.headers["set-cookie"]
+        flow.reply()
 
 
-config = proxy.ProxyConfig(
-    cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
-)
-server = proxy.ProxyServer(config, 8080)
+config = proxy.ProxyConfig(port=8080)
+server = ProxyServer(config)
 m = StickyMaster(server)
 m.run()

examples/stream.py

@@ -0,0 +1,5 @@
+def responseheaders(context, flow):
+    """
+    Enables streaming for all responses.
+    """
+    flow.response.stream = True

examples/stub.py

@@ -1,54 +1,63 @@
 """
     This is a script stub, with definitions for all events.
 """
-def start(ctx, argv):
+def start(context, argv):
     """
         Called once on script startup, before any other events.
     """
-    ctx.log("start")
+    context.log("start")
 
-def clientconnect(ctx, client_connect):
+def clientconnect(context, conn_handler):
     """
         Called when a client initiates a connection to the proxy. Note that a
         connection can correspond to multiple HTTP requests
     """
-    ctx.log("clientconnect")
+    context.log("clientconnect")
 
-def serverconnect(ctx, server_connection):
+def serverconnect(context, conn_handler):
     """
         Called when the proxy initiates a connection to the target server. Note that a
         connection can correspond to multiple HTTP requests
     """
-    ctx.log("serverconnect")
+    context.log("serverconnect")
 
-def request(ctx, flow):
+def request(context, flow):
     """
         Called when a client request has been received.
     """
-    ctx.log("request")
+    context.log("request")
 
-def response(ctx, flow):
+
+def responseheaders(context, flow):
+    """
+        Called when the response headers for a server response have been received,
+        but the response body has not been processed yet. Can be used to tell mitmproxy
+        to stream the response.
+    """
+    context.log("responseheaders")
+
+def response(context, flow):
     """
        Called when a server response has been received.
     """
-    ctx.log("response")
+    context.log("response")
 
-def error(ctx, flow):
+def error(context, flow):
     """
         Called when a flow error has occured, e.g. invalid server responses, or
         interrupted connections. This is distinct from a valid server HTTP error
         response, which is simply a response with an HTTP error code.
     """
-    ctx.log("error")
+    context.log("error")
 
-def clientdisconnect(ctx, client_disconnect):
+def clientdisconnect(context, conn_handler):
     """
         Called when a client disconnects from the proxy.
     """
-    ctx.log("clientdisconnect")
+    context.log("clientdisconnect")
 
-def done(ctx):
+def done(context):
     """
         Called once on script shutdown, after any other events.
     """
-    ctx.log("done")
+    context.log("done")

examples/upsidedownternet.py

@@ -1,10 +1,16 @@
 import cStringIO
 from PIL import Image
+from libmproxy.protocol.http import decoded
 
 def response(context, flow):
-    if flow.response.headers["content-type"] == ["image/png"]:
-        s = cStringIO.StringIO(flow.response.content)
-        img = Image.open(s).rotate(180)
-        s2 = cStringIO.StringIO()
-        img.save(s2, "png")
-        flow.response.content = s2.getvalue()
+    if flow.response.headers.get_first("content-type", "").startswith("image"):
+        with decoded(flow.response):  # automatically decode gzipped responses.
+            try:
+                s = cStringIO.StringIO(flow.response.content)
+                img = Image.open(s).rotate(180)
+                s2 = cStringIO.StringIO()
+                img.save(s2, "png")
+                flow.response.content = s2.getvalue()
+                flow.response.headers["content-type"] = ["image/png"]
+            except:  # Unknown image types etc.
+                pass

libmproxy/app.py

@@ -1,27 +0,0 @@
-import flask
-import os.path
-
-mapp = flask.Flask(__name__)
-mapp.debug = True
-
-def master():
-    return flask.request.environ["mitmproxy.master"]
-
-@mapp.route("/")
-def index():
-    return flask.render_template("index.html", section="home")
-
-
-@mapp.route("/cert/pem")
-def certs_pem():
-    capath = master().server.config.cacert
-    p = os.path.splitext(capath)[0] + "-cert.pem"
-    return flask.Response(open(p).read(), mimetype='application/x-x509-ca-cert')
-
-
-@mapp.route("/cert/p12")
-def certs_p12():
-    capath = master().server.config.cacert
-    p = os.path.splitext(capath)[0] + "-cert.p12"
-    return flask.Response(open(p).read(), mimetype='application/x-pkcs12')
-

libmproxy/cmdline.py

@@ -1,12 +1,18 @@
-import proxy
-import re, filt
-import argparse
+from __future__ import absolute_import
+import os
+import re
+import configargparse
+from netlib import http
+from . import filt, utils, version
+from .proxy import config
 
 APP_HOST = "mitm.it"
 APP_PORT = 80
 
-class ParseException(Exception): pass
-class OptionException(Exception): pass
+
+class ParseException(Exception):
+    pass
+
 
 def _parse_hook(s):
     sep, rem = s[0], s[1:]
@@ -17,13 +23,15 @@ def _parse_hook(s):
     elif len(parts) == 3:
         patt, a, b = parts
     else:
-        raise ParseException("Malformed hook specifier - too few clauses: %s"%s)
+        raise ParseException(
+            "Malformed hook specifier - too few clauses: %s" % s
+        )
 
     if not a:
-        raise ParseException("Empty clause: %s"%str(patt))
+        raise ParseException("Empty clause: %s" % str(patt))
 
     if not filt.parse(patt):
-        raise ParseException("Malformed filter pattern: %s"%patt)
+        raise ParseException("Malformed filter pattern: %s" % patt)
 
     return patt, a, b
 
@@ -58,7 +66,7 @@ def parse_replace_hook(s):
     try:
         re.compile(regex)
     except re.error, e:
-        raise ParseException("Malformed replacement regex: %s"%str(e.message))
+        raise ParseException("Malformed replacement regex: %s" % str(e.message))
     return patt, regex, replacement
 
 
@@ -91,6 +99,27 @@ def parse_setheader(s):
     return _parse_hook(s)
 
 
+def parse_server_spec(url):
+    normalized_url = re.sub("^https?2", "", url)
+
+    p = http.parse_url(normalized_url)
+    if not p or not p[1]:
+        raise configargparse.ArgumentTypeError(
+            "Invalid server specification: %s" % url
+        )
+
+    if url.lower().startswith("https2http"):
+        ssl = [True, False]
+    elif url.lower().startswith("http2https"):
+        ssl = [False, True]
+    elif url.lower().startswith("https"):
+        ssl = [True, True]
+    else:
+        ssl = [False, False]
+
+    return ssl + list(p[1:3])
+
+
 def get_common_options(options):
     stickycookie, stickyauth = None, None
     if options.stickycookie_filt:
@@ -99,206 +128,300 @@ def get_common_options(options):
     if options.stickyauth_filt:
         stickyauth = options.stickyauth_filt
 
+    stream_large_bodies = utils.parse_size(options.stream_large_bodies)
+
     reps = []
     for i in options.replace:
         try:
             p = parse_replace_hook(i)
         except ParseException, e:
-            raise OptionException(e.message)
+            raise configargparse.ArgumentTypeError(e.message)
         reps.append(p)
     for i in options.replace_file:
         try:
             patt, rex, path = parse_replace_hook(i)
         except ParseException, e:
-            raise OptionException(e.message)
+            raise configargparse.ArgumentTypeError(e.message)
         try:
             v = open(path, "rb").read()
         except IOError, e:
-            raise OptionException("Could not read replace file: %s"%path)
+            raise configargparse.ArgumentTypeError(
+                "Could not read replace file: %s" % path
+            )
         reps.append((patt, rex, v))
 
-
     setheaders = []
     for i in options.setheader:
         try:
             p = parse_setheader(i)
         except ParseException, e:
-            raise OptionException(e.message)
+            raise configargparse.ArgumentTypeError(e.message)
         setheaders.append(p)
 
     return dict(
-        app = options.app,
-        app_host = options.app_host,
-        app_port = options.app_port,
-        app_external = options.app_external,
+        app=options.app,
+        app_host=options.app_host,
+        app_port=options.app_port,
 
-        anticache = options.anticache,
-        anticomp = options.anticomp,
-        client_replay = options.client_replay,
-        eventlog = options.eventlog,
-        kill = options.kill,
-        no_server = options.no_server,
-        refresh_server_playback = not options.norefresh,
-        rheaders = options.rheaders,
-        rfile = options.rfile,
-        replacements = reps,
-        setheaders = setheaders,
-        server_replay = options.server_replay,
-        scripts = options.scripts,
-        stickycookie = stickycookie,
-        stickyauth = stickyauth,
-        showhost = options.showhost,
-        wfile = options.wfile,
-        verbosity = options.verbose,
-        nopop = options.nopop,
+        anticache=options.anticache,
+        anticomp=options.anticomp,
+        client_replay=options.client_replay,
+        kill=options.kill,
+        no_server=options.no_server,
+        refresh_server_playback=not options.norefresh,
+        rheaders=options.rheaders,
+        rfile=options.rfile,
+        replacements=reps,
+        setheaders=setheaders,
+        server_replay=options.server_replay,
+        scripts=options.scripts,
+        stickycookie=stickycookie,
+        stickyauth=stickyauth,
+        stream_large_bodies=stream_large_bodies,
+        showhost=options.showhost,
+        outfile=options.outfile,
+        verbosity=options.verbose,
+        nopop=options.nopop,
+        replay_ignore_content = options.replay_ignore_content,
+        replay_ignore_params = options.replay_ignore_params,
+        replay_ignore_payload_params = options.replay_ignore_payload_params
     )
 
 
 def common_options(parser):
     parser.add_argument(
-        "-b",
-        action="store", type = str, dest="addr", default='',
-        help = "Address to bind proxy to (defaults to all interfaces)"
+        '--version',
+        action= 'version',
+        version= "%(prog)s" + " " + version.VERSION
+    )
+    parser.add_argument(
+        '--shortversion',
+        action= 'version',
+        help = "show program's short version number and exit",
+        version = version.VERSION
     )
     parser.add_argument(
         "--anticache",
         action="store_true", dest="anticache", default=False,
-        help="Strip out request headers that might cause the server to return 304-not-modified."
+
+        help="""
+            Strip out request headers that might cause the server to return
+            304-not-modified.
+        """
     )
     parser.add_argument(
-        "--confdir",
-        action="store", type = str, dest="confdir", default='~/.mitmproxy',
-        help = "Configuration directory. (~/.mitmproxy)"
-    )
-    parser.add_argument(
-        "-e",
-        action="store_true", dest="eventlog",
-        help="Show event log."
-    )
-    parser.add_argument(
-        "-n",
-        action="store_true", dest="no_server",
-        help="Don't start a proxy server."
-    )
-    parser.add_argument(
-        "-p",
-        action="store", type = int, dest="port", default=8080,
-        help = "Proxy service port."
-    )
-    parser.add_argument(
-        "-P",
-        action="store", dest="reverse_proxy", default=None,
-        help="Reverse proxy to upstream server: http[s]://host[:port]"
-    )
-    parser.add_argument(
-        "-F",
-        action="store", dest="forward_proxy", default=None,
-        help="Proxy to unconditionally forward to: http[s]://host[:port]"
-    )
-    parser.add_argument(
-        "-q",
-        action="store_true", dest="quiet",
-        help="Quiet."
-    )
-    parser.add_argument(
-        "-r",
-        action="store", dest="rfile", default=None,
-        help="Read flows from file."
-    )
-    parser.add_argument(
-        "-s",
-        action="append", type=str, dest="scripts", default=[],
-        metavar='"script.py --bar"',
-        help="Run a script. Surround with quotes to pass script arguments. Can be passed multiple times."
-    )
-    parser.add_argument(
-        "-t",
-        action="store", dest="stickycookie_filt", default=None, metavar="FILTER",
-        help="Set sticky cookie filter. Matched against requests."
-    )
-    parser.add_argument(
-        "-T",
-        action="store_true", dest="transparent_proxy", default=False,
-        help="Set transparent proxy mode."
-    )
-    parser.add_argument(
-        "-u",
-        action="store", dest="stickyauth_filt", default=None, metavar="FILTER",
-        help="Set sticky auth filter. Matched against requests."
-    )
-    parser.add_argument(
-        "-v",
-        action="count", dest="verbose", default=1,
-        help="Increase verbosity. Can be passed multiple times."
-    )
-    parser.add_argument(
-        "-w",
-        action="store", dest="wfile", default=None,
-        help="Write flows to file."
-    )
-    parser.add_argument(
-        "-z",
-        action="store_true", dest="anticomp", default=False,
-        help="Try to convince servers to send us un-compressed data."
-    )
-    parser.add_argument(
-        "-Z",
-        action="store", dest="body_size_limit", default=None,
-        metavar="SIZE",
-        help="Byte size limit of HTTP request and response bodies."\
-             " Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
+        "--cadir",
+        action="store", type=str, dest="cadir", default=config.CA_DIR,
+        help="Location of the default mitmproxy CA files. (%s)"%config.CA_DIR
     )
     parser.add_argument(
         "--host",
         action="store_true", dest="showhost", default=False,
         help="Use the Host header to construct URLs for display."
     )
-
     parser.add_argument(
-        "--no-upstream-cert", default=False,
-        action="store_true", dest="no_upstream_cert",
-        help="Don't connect to upstream server to look up certificate details."
+        "-q", "--quiet",
+        action="store_true", dest="quiet",
+        help="Quiet."
+    )
+    parser.add_argument(
+        "-r", "--read-flows",
+        action="store", dest="rfile", default=None,
+        help="Read flows from file."
+    )
+    parser.add_argument(
+        "-s", "--script",
+        action="append", type=str, dest="scripts", default=[],
+        metavar='"script.py --bar"',
+        help="""
+            Run a script. Surround with quotes to pass script arguments. Can be
+            passed multiple times.
+        """
+    )
+    parser.add_argument(
+        "-t", "--stickycookie",
+        action="store",
+        dest="stickycookie_filt",
+        default=None,
+        metavar="FILTER",
+        help="Set sticky cookie filter. Matched against requests."
+    )
+    parser.add_argument(
+        "-u", "--stickyauth",
+        action="store", dest="stickyauth_filt", default=None, metavar="FILTER",
+        help="Set sticky auth filter. Matched against requests."
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="store_const", dest="verbose", default=1, const=2,
+        help="Increase event log verbosity."
+    )
+    outfile = parser.add_mutually_exclusive_group()
+    outfile.add_argument(
+        "-w", "--wfile",
+        action="store", dest="outfile", type=lambda f: (f, "wb"),
+        help="Write flows to file."
+    )
+    outfile.add_argument(
+        "-a", "--afile",
+        action="store", dest="outfile", type=lambda f: (f, "ab"),
+        help="Append flows to file."
+    )
+    parser.add_argument(
+        "-z", "--anticomp",
+        action="store_true", dest="anticomp", default=False,
+        help="Try to convince servers to send us un-compressed data."
+    )
+    parser.add_argument(
+        "-Z", "--body-size-limit",
+        action="store", dest="body_size_limit", default=None,
+        metavar="SIZE",
+        help="Byte size limit of HTTP request and response bodies."
+             " Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
+    )
+    parser.add_argument(
+        "--stream",
+        action="store", dest="stream_large_bodies", default=None,
+        metavar="SIZE",
+        help="""
+            Stream data to the client if response body exceeds the given
+            threshold. If streamed, the body will not be stored in any way.
+            Understands k/m/g suffixes, i.e. 3m for 3 megabytes.
+         """
     )
 
-    group = parser.add_argument_group("Web App")
+    group = parser.add_argument_group("Proxy Options")
+    # We could make a mutually exclusive group out of -R, -U, -T, but we don't
+    # do that because  - --upstream-server should be in that group as well, but
+    # it's already in a different group.  - our own error messages are more
+    # helpful
     group.add_argument(
-        "-a",
+        "-b", "--bind-address",
+        action="store", type=str, dest="addr", default='',
+        help="Address to bind proxy to (defaults to all interfaces)"
+    )
+    group.add_argument(
+        "-I", "--ignore",
+        action="append", type=str, dest="ignore_hosts", default=[],
+        metavar="HOST",
+        help="""
+            Ignore host and forward all traffic without processing it. In
+            transparent mode, it is recommended to use an IP address (range),
+            not the hostname. In regular mode, only SSL traffic is ignored and
+            the hostname should be used. The supplied value is interpreted as a
+            regular expression and matched on the ip or the hostname. Can be
+            passed multiple times.
+        """
+    )
+    group.add_argument(
+        "--tcp",
+        action="append", type=str, dest="tcp_hosts", default=[],
+        metavar="HOST",
+        help="""
+            Generic TCP SSL proxy mode for all hosts that match the pattern.
+            Similar to --ignore, but SSL connections are intercepted. The
+            communication contents are printed to the event log in verbose mode.
+        """
+    )
+    group.add_argument(
+        "-n", "--no-server",
+        action="store_true", dest="no_server",
+        help="Don't start a proxy server."
+    )
+    group.add_argument(
+        "-p", "--port",
+        action="store", type=int, dest="port", default=8080,
+        help="Proxy service port."
+    )
+    group.add_argument(
+        "-R", "--reverse",
+        action="store",
+        type=parse_server_spec,
+        dest="reverse_proxy",
+        default=None,
+        help="""
+            Forward all requests to upstream HTTP server:
+            http[s][2http[s]]://host[:port]
+        """
+    )
+    group.add_argument(
+        "--socks",
+        action="store_true", dest="socks_proxy", default=False,
+        help="Set SOCKS5 proxy mode."
+    )
+    group.add_argument(
+        "-T", "--transparent",
+        action="store_true", dest="transparent_proxy", default=False,
+        help="Set transparent proxy mode."
+    )
+    group.add_argument(
+        "-U", "--upstream",
+        action="store",
+        type=parse_server_spec,
+        dest="upstream_proxy",
+        default=None,
+        help="Forward all requests to upstream proxy server: http://host[:port]"
+    )
+
+    group = parser.add_argument_group(
+        "Advanced Proxy Options",
+        """
+            The following options allow a custom adjustment of the proxy
+            behavior. Normally, you don't want to use these options directly and
+            use the provided wrappers instead (-R, -U, -T).
+        """
+    )
+    group.add_argument(
+        "--http-form-in", dest="http_form_in", default=None,
+        action="store", choices=("relative", "absolute"),
+        help="Override the HTTP request form accepted by the proxy"
+    )
+    group.add_argument(
+        "--http-form-out", dest="http_form_out", default=None,
+        action="store", choices=("relative", "absolute"),
+        help="Override the HTTP request form sent upstream by the proxy"
+    )
+
+    group = parser.add_argument_group("Onboarding App")
+    group.add_argument(
+        "--noapp",
         action="store_false", dest="app", default=True,
-        help="Disable the mitmproxy web app."
+        help="Disable the mitmproxy onboarding app."
     )
     group.add_argument(
         "--app-host",
         action="store", dest="app_host", default=APP_HOST, metavar="host",
-        help="Domain to serve the app from. For transparent mode, use an IP when\
-                a DNS entry for the app domain is not present. Default: %s"%APP_HOST
-
+        help="""
+            Domain to serve the onboarding app from. For transparent mode, use
+            an IP when a DNS entry for the app domain is not present. Default:
+            %s
+        """ % APP_HOST
     )
     group.add_argument(
         "--app-port",
-        action="store", dest="app_port", default=APP_PORT, type=int, metavar="80",
-        help="Port to serve the app from."
-    )
-    group.add_argument(
-        "--app-external",
-        action="store_true", dest="app_external",
-        help="Serve the app outside of the proxy."
+        action="store",
+        dest="app_port",
+        default=APP_PORT,
+        type=int,
+        metavar="80",
+        help="Port to serve the onboarding app from."
     )
 
     group = parser.add_argument_group("Client Replay")
     group.add_argument(
-        "-c",
+        "-c", "--client-replay",
         action="store", dest="client_replay", default=None, metavar="PATH",
         help="Replay client requests from a saved file."
     )
 
     group = parser.add_argument_group("Server Replay")
     group.add_argument(
-        "-S",
+        "-S", "--server-replay",
         action="store", dest="server_replay", default=None, metavar="PATH",
         help="Replay server responses from a saved file."
     )
     group.add_argument(
-        "-k",
+        "-k", "--kill",
         action="store_true", dest="kill", default=False,
         help="Kill extra requests during replay."
     )
@@ -306,19 +429,47 @@ def common_options(parser):
         "--rheader",
         action="append", dest="rheaders", type=str,
         help="Request headers to be considered during replay. "
-           "Can be passed multiple times."
+             "Can be passed multiple times."
     )
     group.add_argument(
         "--norefresh",
         action="store_true", dest="norefresh", default=False,
-        help= "Disable response refresh, "
-        "which updates times in cookies and headers for replayed responses."
+        help="""
+            Disable response refresh, which updates times in cookies and headers
+            for replayed responses.
+        """
     )
     group.add_argument(
         "--no-pop",
         action="store_true", dest="nopop", default=False,
         help="Disable response pop from response flow. "
-        "This makes it possible to replay same response multiple times."
+             "This makes it possible to replay same response multiple times."
+    )
+    payload = group.add_mutually_exclusive_group()
+    payload.add_argument(
+        "--replay-ignore-content",
+        action="store_true", dest="replay_ignore_content", default=False,
+        help="""
+            Ignore request's content while searching for a saved flow to replay
+        """
+    )
+    payload.add_argument(
+        "--replay-ignore-payload-param",
+        action="append", dest="replay_ignore_payload_params", type=str,
+        help="""
+            Request's payload parameters (application/x-www-form-urlencoded) to
+            be ignored while searching for a saved flow to replay.
+            Can be passed multiple times.
+        """
+    )
+
+    group.add_argument(
+        "--replay-ignore-param",
+        action="append", dest="replay_ignore_params", type=str,
+        help="""
+            Request's parameters to be ignored while searching for a saved flow
+            to replay. Can be passed multiple times.
+        """
     )
 
     group = parser.add_argument_group(
@@ -337,12 +488,14 @@ def common_options(parser):
     )
     group.add_argument(
         "--replace-from-file",
-        action="append", type=str, dest="replace_file", default=[],
-        metavar="PATH",
-        help="Replacement pattern, where the replacement clause is a path to a file."
+        action = "append", type=str, dest="replace_file", default=[],
+        metavar = "PATH",
+        help = """
+            Replacement pattern, where the replacement clause is a path to a
+            file.
+        """
     )
 
-
     group = parser.add_argument_group(
         "Set Headers",
         """
@@ -358,13 +511,11 @@ def common_options(parser):
         help="Header set pattern."
     )
 
-
     group = parser.add_argument_group(
         "Proxy Authentication",
         """
             Specify which users are allowed to access the proxy and the method
-            used for authenticating them. These options are ignored if the
-            proxy is in transparent or reverse proxy mode.
+            used for authenticating them.
         """
     )
     user_specification_group = group.add_mutually_exclusive_group()
@@ -378,13 +529,129 @@ def common_options(parser):
         "--singleuser",
         action="store", dest="auth_singleuser", type=str,
         metavar="USER",
-        help="Allows access to a a single user, specified in the form username:password."
+        help="""
+            Allows access to a a single user, specified in the form
+            username:password.
+        """
     )
     user_specification_group.add_argument(
         "--htpasswd",
-        action="store", dest="auth_htpasswd", type=argparse.FileType('r'),
+        action="store", dest="auth_htpasswd", type=str,
         metavar="PATH",
         help="Allow access to users specified in an Apache htpasswd file."
     )
 
-    proxy.certificate_option_group(parser)
+    config.ssl_option_group(parser)
+
+
+def mitmproxy():
+    # Don't import libmproxy.console for mitmdump, urwid is not available on all
+    # platforms.
+    from .console import palettes
+
+    parser = configargparse.ArgumentParser(
+        usage="%(prog)s [options]",
+        args_for_setting_config_path = ["--conf"],
+        default_config_files = [
+            os.path.join(config.CA_DIR, "common.conf"),
+            os.path.join(config.CA_DIR, "mitmproxy.conf")
+        ],
+        add_config_file_help = True,
+        add_env_var_help = True
+    )
+    common_options(parser)
+    parser.add_argument(
+        "--palette", type=str, default="dark",
+        action="store", dest="palette",
+        help="Select color palette: " + ", ".join(palettes.palettes.keys())
+    )
+    parser.add_argument(
+        "-e", "--eventlog",
+        action="store_true", dest="eventlog",
+        help="Show event log."
+    )
+    group = parser.add_argument_group(
+        "Filters",
+        "See help in mitmproxy for filter expression syntax."
+    )
+    group.add_argument(
+        "-i", "--intercept", action="store",
+        type=str, dest="intercept", default=None,
+        help="Intercept filter expression."
+    )
+    return parser
+
+
+def mitmdump():
+    parser = configargparse.ArgumentParser(
+        usage="%(prog)s [options] [filter]",
+        args_for_setting_config_path = ["--conf"],
+        default_config_files = [
+            os.path.join(config.CA_DIR, "common.conf"),
+            os.path.join(config.CA_DIR, "mitmdump.conf")
+        ],
+        add_config_file_help = True,
+        add_env_var_help = True
+    )
+
+    common_options(parser)
+    parser.add_argument(
+        "--keepserving",
+        action= "store_true", dest="keepserving", default=False,
+        help= """
+            Continue serving after client playback or file read. We exit by
+            default.
+        """
+    )
+    parser.add_argument(
+        "-d", "--detail",
+        action="count", dest="flow_detail", default=1,
+        help="Increase flow detail display level. Can be passed multiple times."
+    )
+    parser.add_argument('args', nargs="...")
+    return parser
+
+
+def mitmweb():
+    parser = configargparse.ArgumentParser(
+        usage="%(prog)s [options]",
+        args_for_setting_config_path = ["--conf"],
+        default_config_files = [
+            os.path.join(config.CA_DIR, "common.conf"),
+            os.path.join(config.CA_DIR, "mitmweb.conf")
+        ],
+        add_config_file_help = True,
+        add_env_var_help = True
+    )
+
+    group = parser.add_argument_group("Mitmweb")
+    group.add_argument(
+        "--wport",
+        action="store", type=int, dest="wport", default=8081,
+        metavar="PORT",
+        help="Mitmweb port."
+    )
+    group.add_argument(
+        "--wiface",
+        action="store", dest="wiface", default="127.0.0.1",
+        metavar="IFACE",
+        help="Mitmweb interface."
+    )
+    group.add_argument(
+        "--wdebug",
+        action="store_true", dest="wdebug",
+        help="Turn on mitmweb debugging"
+    )
+
+    common_options(parser)
+    group = parser.add_argument_group(
+        "Filters",
+        "See help in mitmproxy for filter expression syntax."
+    )
+    group.add_argument(
+        "-i", "--intercept", action="store",
+        type=str, dest="intercept", default=None,
+        help="Intercept filter expression."
+    )
+    return parser
+

libmproxy/console/__init__.py

@@ -1,8 +1,9 @@
+from __future__ import absolute_import
 import mailcap, mimetypes, tempfile, os, subprocess, glob, time, shlex, stat
-import os.path, sys, weakref
+import os.path, sys, weakref, traceback
 import urwid
-from .. import controller, utils, flow, script
-import flowlist, flowview, help, common, grideditor, palettes, contentview, flowdetailview
+from .. import controller, utils, flow, script, proxy
+from . import flowlist, flowview, help, common, grideditor, palettes, contentview, flowdetailview
 
 EVENTLOG_SIZE = 500
 
@@ -77,7 +78,6 @@ class PathEdit(urwid.Edit, _PathCompleter):
 class ActionBar(common.WWrap):
     def __init__(self):
         self.message("")
-        self.expire = None
 
     def selectable(self):
         return True
@@ -129,6 +129,14 @@ class StatusBar(common.WWrap):
                 r.append(":%s in file]"%self.master.server_playback.count())
             else:
                 r.append(":%s to go]"%self.master.server_playback.count())
+        if self.master.get_ignore_filter():
+            r.append("[")
+            r.append(("heading_key", "I"))
+            r.append("gnore:%d]" % len(self.master.get_ignore_filter()))
+        if self.master.get_tcp_filter():
+            r.append("[")
+            r.append(("heading_key", "T"))
+            r.append("CP:%d]" % len(self.master.get_tcp_filter()))
         if self.master.state.intercept_txt:
             r.append("[")
             r.append(("heading_key", "i"))
@@ -145,10 +153,6 @@ class StatusBar(common.WWrap):
             r.append("[")
             r.append(("heading_key", "u"))
             r.append(":%s]"%self.master.stickyauth_txt)
-        if self.master.server.config.reverse_proxy:
-            r.append("[")
-            r.append(("heading_key", "P"))
-            r.append(":%s]"%utils.unparse_url(*self.master.server.config.reverse_proxy))
         if self.master.state.default_body_view.name != "Auto":
             r.append("[")
             r.append(("heading_key", "M"))
@@ -169,14 +173,23 @@ class StatusBar(common.WWrap):
             opts.append("no-upstream-cert")
         if self.master.state.follow_focus:
             opts.append("following")
+        if self.master.stream_large_bodies:
+            opts.append("stream:%s" % utils.pretty_size(self.master.stream_large_bodies.max_size))
 
         if opts:
             r.append("[%s]"%(":".join(opts)))
 
+        if self.master.server.config.mode in ["reverse", "upstream"]:
+            dst = self.master.server.config.mode.dst
+            scheme = "https" if dst[0] else "http"
+            if dst[1] != dst[0]:
+                scheme += "2https" if dst[1] else "http"
+            r.append("[dest:%s]"%utils.unparse_url(scheme, *dst[2:]))
         if self.master.scripts:
-            r.append("[scripts:%s]"%len(self.master.scripts))
-        if self.master.debug:
-            r.append("[lt:%0.3f]"%self.master.looptime)
+            r.append("[")
+            r.append(("heading_key", "s"))
+            r.append("cripts:%s]"%len(self.master.scripts))
+        # r.append("[lt:%0.3f]"%self.master.looptime)
 
         if self.master.stream:
             r.append("[W:%s]"%self.master.stream_path)
@@ -197,7 +210,10 @@ class StatusBar(common.WWrap):
         ]
 
         if self.master.server.bound:
-            boundaddr = "[%s:%s]"%(self.master.server.address or "*", self.master.server.port)
+            host = self.master.server.address.host
+            if host == "0.0.0.0":
+                host = "*"
+            boundaddr = "[%s:%s]"%(host, self.master.server.address.port)
         else:
             boundaddr = ""
         t.extend(self.get_status())
@@ -261,16 +277,16 @@ class ConsoleState(flow.State):
         d = self.flowsettings.get(flow, {})
         return d.get(key, default)
 
-    def add_request(self, req):
-        f = flow.State.add_request(self, req)
+    def add_flow(self, f):
+        super(ConsoleState, self).add_flow(f)
         if self.focus is None:
             self.set_focus(0)
         elif self.follow_focus:
             self.set_focus(len(self.view) - 1)
         return f
 
-    def add_response(self, resp):
-        f = flow.State.add_response(self, resp)
+    def update_flow(self, f):
+        super(ConsoleState, self).update_flow(f)
         if self.focus is None:
             self.set_focus(0)
         return f
@@ -326,7 +342,6 @@ class Options(object):
         "anticache",
         "anticomp",
         "client_replay",
-        "debug",
         "eventlog",
         "keepserving",
         "kill",
@@ -342,6 +357,7 @@ class Options(object):
         "server_replay",
         "stickycookie",
         "stickyauth",
+        "stream_large_bodies",
         "verbosity",
         "wfile",
         "nopop",
@@ -390,6 +406,8 @@ class ConsoleMaster(flow.FlowMaster):
             print >> sys.stderr, "Sticky auth error:", r
             sys.exit(1)
 
+        self.set_stream_large_bodies(options.stream_large_bodies)
+
         self.refresh_server_playback = options.refresh_server_playback
         self.anticache = options.anticache
         self.anticomp = options.anticomp
@@ -407,8 +425,6 @@ class ConsoleMaster(flow.FlowMaster):
         if options.server_replay:
             self.server_playback_path(options.server_replay)
 
-        self.debug = options.debug
-
         if options.scripts:
             for i in options.scripts:
                 err = self.load_script(i)
@@ -423,7 +439,7 @@ class ConsoleMaster(flow.FlowMaster):
                 sys.exit(1)
 
         if options.app:
-            self.start_app(self.options.app_host, self.options.app_port, self.options.app_external)
+            self.start_app(self.options.app_host, self.options.app_port)
 
     def start_stream(self, path):
         path = os.path.expanduser(path)
@@ -438,20 +454,20 @@ class ConsoleMaster(flow.FlowMaster):
         status, val = s.run(method, f)
         if val:
             if status:
-                self.add_event("Method %s return: %s"%(method, val))
+                self.add_event("Method %s return: %s"%(method, val), "debug")
             else:
-                self.add_event("Method %s error: %s"%(method, val[1]))
+                self.add_event("Method %s error: %s"%(method, val[1]), "error")
 
     def run_script_once(self, command, f):
         if not command:
             return
-        self.add_event("Running script on flow: %s"%command)
+        self.add_event("Running script on flow: %s"%command, "debug")
 
         try:
             s = script.Script(command, self)
         except script.ScriptError, v:
             self.statusbar.message("Error loading script.")
-            self.add_event("Error loading script:\n%s"%v.args[0])
+            self.add_event("Error loading script:\n%s"%v.args[0], "error")
             return
 
         if f.request:
@@ -500,7 +516,8 @@ class ConsoleMaster(flow.FlowMaster):
             self.start_server_playback(
                 ret,
                 self.killextra, self.rheaders,
-                False, self.nopop
+                False, self.nopop,
+                self.options.replay_ignore_params, self.options.replay_ignore_content
             )
 
     def spawn_editor(self, data):
@@ -568,31 +585,35 @@ class ConsoleMaster(flow.FlowMaster):
         self.ui.set_terminal_properties(256)
         self.ui.register_palette(self.palette)
         self.flow_list_walker = flowlist.FlowListWalker(self, self.state)
-
         self.view = None
         self.statusbar = None
         self.header = None
         self.body = None
         self.help_context = None
-
         self.prompting = False
         self.onekey = False
 
         self.view_flowlist()
 
-        self.server.start_slave(controller.Slave, controller.Channel(self.masterq))
+        self.server.start_slave(controller.Slave, controller.Channel(self.masterq, self.should_exit))
 
         if self.options.rfile:
             ret = self.load_flows(self.options.rfile)
             if ret and self.state.flow_count():
-                self.add_event("File truncated or corrupted. Loaded as many flows as possible.")
+                self.add_event("File truncated or corrupted. Loaded as many flows as possible.","error")
             elif not self.state.flow_count():
                 self.shutdown()
                 print >> sys.stderr, "Could not load file:", ret
                 sys.exit(1)
 
-        self.ui.run_wrapper(self.loop)
-        # If True, quit just pops out to flow list view.
+        try:
+            self.ui.run_wrapper(self.loop)
+        except Exception:
+            self.ui.stop()
+            sys.stdout.flush()
+            print >> sys.stderr, traceback.format_exc()
+            print >> sys.stderr, "mitmproxy has crashed!"
+            print >> sys.stderr, "Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy"
         print >> sys.stderr, "Shutting down..."
         sys.stderr.flush()
         self.shutdown()
@@ -649,7 +670,6 @@ class ConsoleMaster(flow.FlowMaster):
         self.statusbar = StatusBar(self, flowview.footer)
         self.state.set_focus_flow(flow)
         self.state.view_mode = common.VIEW_FLOW
-
         self.make_view()
         self.help_context = flowview.help_context
 
@@ -747,7 +767,7 @@ class ConsoleMaster(flow.FlowMaster):
         self.prompt_done()
 
     def accept_all(self):
-        self.state.accept_all()
+        self.state.accept_all(self)
 
     def set_limit(self, txt):
         v = self.state.set_limit(txt)
@@ -762,15 +782,6 @@ class ConsoleMaster(flow.FlowMaster):
         self.state.default_body_view = v
         self.refresh_focus()
 
-    def set_reverse_proxy(self, txt):
-        if not txt:
-            self.server.config.reverse_proxy = None
-        else:
-            s = utils.parse_proxy_spec(txt)
-            if not s:
-                return "Invalid reverse proxy specification"
-            self.server.config.reverse_proxy = s
-
     def drawscreen(self):
         size = self.ui.get_cols_rows()
         canvas = self.view.render(size, focus=1)
@@ -783,19 +794,33 @@ class ConsoleMaster(flow.FlowMaster):
         else:
             self.view_flowlist()
 
-    def edit_scripts(self, *args, **kwargs):
-        pass
+    def edit_scripts(self, scripts):
+        commands = [x[0] for x in scripts]  # remove outer array
+        if commands == [s.command for s in self.scripts]:
+            return
+
+        self.unload_scripts()
+        for command in commands:
+            self.load_script(command)
+
+    def edit_ignore_filter(self, ignore):
+        patterns = (x[0] for x in ignore)
+        self.set_ignore_filter(patterns)
+
+    def edit_tcp_filter(self, tcp):
+        patterns = (x[0] for x in tcp)
+        self.set_tcp_filter(patterns)
 
     def loop(self):
         changed = True
         try:
-            while not controller.should_exit:
+            while not self.should_exit.is_set():
                 startloop = time.time()
                 if changed:
                     self.statusbar.redraw()
                     size = self.drawscreen()
-                changed = self.tick(self.masterq)
-                self.ui.set_input_timeouts(max_wait=0.1)
+                changed = self.tick(self.masterq, 0.01)
+                self.ui.set_input_timeouts(max_wait=0.01)
                 keys = self.ui.get_input()
                 if keys:
                     changed = True
@@ -842,6 +867,22 @@ class ConsoleMaster(flow.FlowMaster):
                                         self.setheaders.set
                                     )
                                 )
+                            elif k == "I":
+                                self.view_grideditor(
+                                    grideditor.HostPatternEditor(
+                                        self,
+                                        [[x] for x in self.get_ignore_filter()],
+                                        self.edit_ignore_filter
+                                    )
+                                )
+                            elif k == "T":
+                                self.view_grideditor(
+                                    grideditor.HostPatternEditor(
+                                        self,
+                                        [[x] for x in self.get_tcp_filter()],
+                                        self.edit_tcp_filter
+                                    )
+                                )
                             elif k == "i":
                                 self.prompt(
                                     "Intercept filter: ",
@@ -865,16 +906,6 @@ class ConsoleMaster(flow.FlowMaster):
                                     contentview.view_prompts,
                                     self.change_default_display_mode
                                 )
-                            elif k == "P":
-                                if self.server.config.reverse_proxy:
-                                    p = utils.unparse_url(*self.server.config.reverse_proxy)
-                                else:
-                                    p = ""
-                                self.prompt(
-                                    "Reverse proxy: ",
-                                    p,
-                                    self.set_reverse_proxy
-                                )
                             elif k == "R":
                                 self.view_grideditor(
                                     grideditor.ReplaceEditor(
@@ -887,7 +918,7 @@ class ConsoleMaster(flow.FlowMaster):
                                 self.view_grideditor(
                                     grideditor.ScriptEditor(
                                         self,
-                                        [[i.argv[0]] for i in self.scripts],
+                                        [[i.command] for i in self.scripts],
                                         self.edit_scripts
                                     )
                                 )
@@ -1007,11 +1038,11 @@ class ConsoleMaster(flow.FlowMaster):
         if hasattr(self.statusbar, "refresh_flow"):
             self.statusbar.refresh_flow(c)
 
-    def process_flow(self, f, r):
-        if self.state.intercept and f.match(self.state.intercept) and not f.request.is_replay():
-            f.intercept()
+    def process_flow(self, f):
+        if self.state.intercept and f.match(self.state.intercept) and not f.request.is_replay:
+            f.intercept(self)
         else:
-            r.reply()
+            f.reply()
         self.sync_list_view()
         self.refresh_flow(f)
 
@@ -1019,35 +1050,34 @@ class ConsoleMaster(flow.FlowMaster):
         self.eventlist[:] = []
 
     def add_event(self, e, level="info"):
-        if level == "info":
-            e = urwid.Text(str(e))
-        elif level == "error":
-            e = urwid.Text(("error", str(e)))
+        needed = dict(error=0, info=1, debug=2).get(level, 1)
+        if self.options.verbosity < needed:
+            return
 
+        if level == "error":
+            e = urwid.Text(("error", str(e)))
+        else:
+            e = urwid.Text(str(e))
         self.eventlist.append(e)
         if len(self.eventlist) > EVENTLOG_SIZE:
             self.eventlist.pop(0)
         self.eventlist.set_focus(len(self.eventlist)-1)
 
     # Handlers
-    def handle_log(self, l):
-        self.add_event(l.msg)
-        l.reply()
-
-    def handle_error(self, r):
-        f = flow.FlowMaster.handle_error(self, r)
+    def handle_error(self, f):
+        f = flow.FlowMaster.handle_error(self, f)
         if f:
-            self.process_flow(f, r)
+            self.process_flow(f)
         return f
 
-    def handle_request(self, r):
-        f = flow.FlowMaster.handle_request(self, r)
+    def handle_request(self, f):
+        f = flow.FlowMaster.handle_request(self, f)
         if f:
-            self.process_flow(f, r)
+            self.process_flow(f)
         return f
 
-    def handle_response(self, r):
-        f = flow.FlowMaster.handle_response(self, r)
+    def handle_response(self, f):
+        f = flow.FlowMaster.handle_response(self, f)
         if f:
-            self.process_flow(f, r)
+            self.process_flow(f)
         return f

libmproxy/console/common.py

@@ -1,6 +1,8 @@
+from __future__ import absolute_import
 import urwid
 import urwid.util
-from .. import utils, flow
+from .. import utils
+from ..protocol.http import CONTENT_MISSING
 
 
 VIEW_LIST = 0
@@ -106,7 +108,7 @@ def raw_format_flow(f, focus, extended, padding):
 
     preamble = sum(i[1] for i in req) + len(req) -1
 
-    if f["intercepting"] and not f["req_acked"]:
+    if f["intercepted"] and not f["acked"]:
         uc = "intercept"
     elif f["resp_code"] or f["err_msg"]:
         uc = "text"
@@ -136,7 +138,7 @@ def raw_format_flow(f, focus, extended, padding):
         if f["resp_is_replay"]:
             resp.append(fcol(SYMBOL_REPLAY, "replay"))
         resp.append(fcol(f["resp_code"], ccol))
-        if f["intercepting"] and f["resp_code"] and not f["resp_acked"]:
+        if f["intercepted"] and f["resp_code"] and not f["acked"]:
             rc = "intercept"
         else:
             rc = "text"
@@ -169,13 +171,13 @@ flowcache = FlowCache()
 
 def format_flow(f, focus, extended=False, hostheader=False, padding=2):
     d = dict(
-        intercepting = f.intercepting,
+        intercepted = f.intercepted,
+        acked = f.reply.acked,
 
         req_timestamp = f.request.timestamp_start,
-        req_is_replay = f.request.is_replay(),
+        req_is_replay = f.request.is_replay,
         req_method = f.request.method,
-        req_acked = f.request.reply.acked,
-        req_url = f.request.get_url(hostheader=hostheader),
+        req_url = f.request.pretty_url(hostheader=hostheader),
 
         err_msg = f.error.msg if f.error else None,
         resp_code = f.response.code if f.response else None,
@@ -183,19 +185,21 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
     if f.response:
         if f.response.content:
             contentdesc = utils.pretty_size(len(f.response.content))
-        elif f.response.content == flow.CONTENT_MISSING:
+        elif f.response.content == CONTENT_MISSING:
             contentdesc = "[content missing]"
         else:
             contentdesc = "[no content]"
 
-        delta = f.response.timestamp_end - f.response.timestamp_start
-        size = len(f.response.content) + f.response.get_header_size()
+        if f.response.timestamp_end:
+            delta = f.response.timestamp_end - f.response.timestamp_start
+        else:
+            delta = 0
+        size = f.response.size()
         rate = utils.pretty_size(size / ( delta if delta > 0 else 1 ) )
 
         d.update(dict(
             resp_code = f.response.code,
-            resp_is_replay = f.response.is_replay(),
-            resp_acked = f.response.reply.acked,
+            resp_is_replay = f.response.is_replay,
             resp_clen = contentdesc,
             resp_rate = "{0}/s".format(rate),
         ))

libmproxy/console/contentview.py

@@ -1,16 +1,14 @@
-import logging
-import re, cStringIO, traceback, json
-import urwid
-
+from __future__ import absolute_import
+import logging, subprocess, re, cStringIO, traceback, json, urwid
 from PIL import Image
 from PIL.ExifTags import TAGS
 
 import lxml.html, lxml.etree
 import netlib.utils
-import common
+from . import common
 from .. import utils, encoding, flow
 from ..contrib import jsbeautifier, html2text
-import subprocess
+from ..contrib.wbxml.ASCommandResponse import ASCommandResponse
 try:
     import pyamf
     from pyamf import remoting, flex
@@ -429,12 +427,31 @@ class ViewProtobuf:
         txt = _view_text(decoded[:limit], len(decoded), limit)
         return "Protobuf", txt
 
+class ViewWBXML:
+    name = "WBXML"
+    prompt = ("wbxml", "w")
+    content_types = [
+        "application/vnd.wap.wbxml",
+        "application/vnd.ms-sync.wbxml"
+    ]
+
+    def __call__(self, hdrs, content, limit):
+        
+        try:
+            parser = ASCommandResponse(content)
+            parsedContent = parser.xmlString
+            txt = _view_text(parsedContent, len(parsedContent), limit)
+            return "WBXML", txt
+        except:
+        	return None
+
 views = [
     ViewAuto(),
     ViewRaw(),
     ViewHex(),
     ViewJSON(),
     ViewXML(),
+    ViewWBXML(),
     ViewHTML(),
     ViewHTMLOutline(),
     ViewJavaScript(),
@@ -493,7 +510,7 @@ def get_content_view(viewmode, hdrItems, content, limit, logfunc):
     except Exception:
         s = traceback.format_exc()
         s = "Content viewer failed: \n"  + s
-        logfunc(s)
+        logfunc(s, "error")
         ret = None
     if not ret:
         ret = get("Raw")(hdrs, content, limit)

libmproxy/console/flowdetailview.py

@@ -1,5 +1,7 @@
+from __future__ import absolute_import
 import urwid
-import common
+from . import common
+from .. import utils
 
 footer = [
     ('heading_key', "q"), ":back ",
@@ -33,8 +35,17 @@ class FlowDetailsView(urwid.ListBox):
         title = urwid.AttrWrap(title, "heading")
         text.append(title)
 
-        if self.flow.response:
-            c = self.flow.response.cert
+        if self.flow.server_conn:
+            text.append(urwid.Text([("head", "Server Connection:")]))
+            sc = self.flow.server_conn
+            parts = [
+                ["Address", "%s:%s" % sc.address()],
+                ["Start time", utils.format_timestamp(sc.timestamp_start)],
+                ["End time", utils.format_timestamp(sc.timestamp_end) if sc.timestamp_end else "active"],
+            ]
+            text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
+
+            c = self.flow.server_conn.cert
             if c:
                 text.append(urwid.Text([("head", "Server Certificate:")]))
                 parts = [
@@ -43,19 +54,13 @@ class FlowDetailsView(urwid.ListBox):
                     ["Valid to", str(c.notafter)],
                     ["Valid from", str(c.notbefore)],
                     ["Serial", str(c.serial)],
-                ]
-
-                parts.append(
                     [
                         "Subject",
                         urwid.BoxAdapter(
                             urwid.ListBox(common.format_keyvals(c.subject, key="highlight", val="text")),
                             len(c.subject)
                         )
-                    ]
-                )
-
-                parts.append(
+                    ],
                     [
                         "Issuer",
                         urwid.BoxAdapter(
@@ -63,7 +68,7 @@ class FlowDetailsView(urwid.ListBox):
                             len(c.issuer)
                         )
                     ]
-                )
+                ]
 
                 if c.altnames:
                     parts.append(
@@ -74,13 +79,14 @@ class FlowDetailsView(urwid.ListBox):
                     )
                 text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
 
-        if self.flow.request.client_conn:
+        if self.flow.client_conn:
             text.append(urwid.Text([("head", "Client Connection:")]))
-            cc = self.flow.request.client_conn
+            cc = self.flow.client_conn
             parts = [
-                ["Address", "%s:%s"%tuple(cc.address)],
-                ["Requests", "%s"%cc.requestcount],
-                ["Closed", "%s"%cc.close],
+                ["Address", "%s:%s" % cc.address()],
+                ["Start time", utils.format_timestamp(cc.timestamp_start)],
+                # ["Requests", "%s"%cc.requestcount],
+                ["End time", utils.format_timestamp(cc.timestamp_end) if cc.timestamp_end else "active"],
             ]
             text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
 

libmproxy/console/flowlist.py

@@ -1,5 +1,6 @@
+from __future__ import absolute_import
 import urwid
-import common
+from . import common
 
 def _mkhelp():
     text = []
@@ -68,7 +69,7 @@ class BodyPile(urwid.Pile):
             else:
                 self.widget_list[1].header = self.inactive_header
             key = None
-        elif key == "v":
+        elif key == "e":
             self.master.toggle_eventlog()
             key = None
 
@@ -119,13 +120,15 @@ class ConnectionItem(common.WWrap):
             self.master.start_server_playback(
                 [i.copy() for i in self.master.state.view],
                 self.master.killextra, self.master.rheaders,
-                False, self.master.nopop
+                False, self.master.nopop,
+                self.master.options.replay_ignore_params, self.master.options.replay_ignore_content
             )
         elif k == "t":
             self.master.start_server_playback(
                 [self.flow.copy()],
                 self.master.killextra, self.master.rheaders,
-                False, self.master.nopop
+                False, self.master.nopop,
+                self.master.options.replay_ignore_params, self.master.options.replay_ignore_content
             )
         else:
             self.master.path_prompt(
@@ -137,7 +140,7 @@ class ConnectionItem(common.WWrap):
     def keypress(self, (maxcol,), key):
         key = common.shortcuts(key)
         if key == "a":
-            self.flow.accept_intercept()
+            self.flow.accept_intercept(self.master)
             self.master.sync_list_view()
         elif key == "d":
             self.flow.kill(self.master)
@@ -147,7 +150,6 @@ class ConnectionItem(common.WWrap):
             f = self.master.duplicate_flow(self.flow)
             self.master.view_flow(f)
         elif key == "r":
-            self.flow.backup()
             r = self.master.replay_request(self.flow)
             if r:
                 self.master.statusbar.message(r)

libmproxy/console/flowview.py

@@ -1,7 +1,9 @@
+from __future__ import absolute_import
 import os, sys, copy
 import urwid
-import common, grideditor, contentview
+from . import common, grideditor, contentview
 from .. import utils, flow, controller
+from ..protocol.http import HTTPResponse, CONTENT_MISSING, decoded
 
 
 class SearchError(Exception): pass
@@ -68,7 +70,8 @@ def _mkhelp():
         ("space", "next flow"),
         ("|", "run script on this flow"),
         ("/", "search in response body (case sensitive)"),
-        ("n", "repeat previous search"),
+        ("n", "repeat search forward"),
+        ("N", "repeat search backwards"),
     ]
     text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
     return text
@@ -149,8 +152,8 @@ class FlowView(common.WWrap):
         return (description, text_objects)
 
     def cont_view_handle_missing(self, conn, viewmode):
-            if conn.content == flow.CONTENT_MISSING:
-                msg, body = "", [urwid.Text([("error", "[content missing]")])], 0
+            if conn.content == CONTENT_MISSING:
+                msg, body = "", [urwid.Text([("error", "[content missing]")])]
             else:
                 msg, body = self.content_view(viewmode, conn)
 
@@ -173,12 +176,9 @@ class FlowView(common.WWrap):
                 key = "header",
                 val = "text"
             )
-        if conn.content is not None:
-            override = self.override_get()
-            viewmode = self.viewmode_get(override)
-            msg, body = self.cont_view_handle_missing(conn, viewmode)
-        elif conn.content == flow.CONTENT_MISSING:
-            pass
+        override = self.override_get()
+        viewmode = self.viewmode_get(override)
+        msg, body = self.cont_view_handle_missing(conn, viewmode)
         return headers, msg, body
 
     def conn_text_merge(self, headers, msg, body):
@@ -230,7 +230,7 @@ class FlowView(common.WWrap):
     def wrap_body(self, active, body):
         parts = []
 
-        if self.flow.intercepting and not self.flow.request.reply.acked:
+        if self.flow.intercepted and not self.flow.reply.acked and not self.flow.response:
             qt = "Request intercepted"
         else:
             qt = "Request"
@@ -239,7 +239,7 @@ class FlowView(common.WWrap):
         else:
             parts.append(self._tab(qt, "heading_inactive"))
 
-        if self.flow.intercepting and self.flow.response and not self.flow.response.reply.acked:
+        if self.flow.intercepted and not self.flow.reply.acked and self.flow.response:
             st = "Response intercepted"
         else:
             st = "Response"
@@ -255,7 +255,7 @@ class FlowView(common.WWrap):
                 )
         return f
 
-    def search_wrapped_around(self, last_find_line, last_search_index):
+    def search_wrapped_around(self, last_find_line, last_search_index, backwards):
         """
             returns true if search wrapped around the bottom.
         """
@@ -265,24 +265,50 @@ class FlowView(common.WWrap):
         current_search_index = self.state.get_flow_setting(self.flow,
                 "last_search_index")
 
-        if current_find_line <= last_find_line:
-            return True
-        elif current_find_line == last_find_line:
-            if current_search_index <= last_search_index:
-                return True
+        if not backwards:
+            message = "search hit BOTTOM, continuing at TOP"
+            if current_find_line <= last_find_line:
+                return True, message
+            elif current_find_line == last_find_line:
+                if current_search_index <= last_search_index:
+                    return True, message
+        else:
+            message = "search hit TOP, continuing at BOTTOM"
+            if current_find_line >= last_find_line:
+                return True, message
+            elif current_find_line == last_find_line:
+                if current_search_index >= last_search_index:
+                    return True, message
 
-        return False
+        return False, ""
 
-    def search(self, search_string):
+    def search_again(self, backwards=False):
+        """
+            runs the previous search again, forwards or backwards.
+        """
+        last_search_string = self.state.get_flow_setting(self.flow, "last_search_string")
+        if last_search_string:
+            message = self.search(last_search_string, backwards)
+            if message:
+                self.master.statusbar.message(message)
+        else:
+            message = "no previous searches have been made"
+            self.master.statusbar.message(message)
+
+        return message
+
+    def search(self, search_string, backwards=False):
         """
             similar to view_response or view_request, but instead of just
             displaying the conn, it highlights a word that the user is
             searching for and handles all the logic surrounding that.
         """
 
-        if search_string == "":
+        if not search_string:
             search_string = self.state.get_flow_setting(self.flow,
                     "last_search_string")
+            if not search_string:
+                return
 
         if self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
             text = self.flow.request
@@ -301,7 +327,7 @@ class FlowView(common.WWrap):
         # generate the body, highlight the words and get focus
         headers, msg, body = self.conn_text_raw(text)
         try:
-            body, focus_position = self.search_highlight_text(body, search_string)
+            body, focus_position = self.search_highlight_text(body, search_string, backwards=backwards)
         except SearchError:
             return "Search not supported in this view."
 
@@ -318,8 +344,10 @@ class FlowView(common.WWrap):
 
         self.last_displayed_body = list_box
 
-        if self.search_wrapped_around(last_find_line, last_search_index):
-            return "search hit BOTTOM, continuing at TOP"
+        wrapped, wrapped_message = self.search_wrapped_around(last_find_line, last_search_index, backwards)
+
+        if wrapped:
+            return wrapped_message
 
     def search_get_start(self, search_string):
         start_line = 0
@@ -344,57 +372,94 @@ class FlowView(common.WWrap):
 
         return (start_line, start_index)
 
-    def search_highlight_text(self, text_objects, search_string, looping = False):
+    def search_get_range(self, len_text_objects, start_line, backwards):
+        if not backwards:
+            loop_range = xrange(start_line, len_text_objects)
+        else:
+            loop_range = xrange(start_line, -1, -1)
+
+        return loop_range
+
+    def search_find(self, text, search_string, start_index, backwards):
+            if backwards == False:
+                find_index = text.find(search_string, start_index)
+            else:
+                if start_index != 0:
+                    start_index -= len(search_string)
+                else:
+                    start_index = None
+
+                find_index = text.rfind(search_string, 0, start_index)
+
+            return find_index
+
+    def search_highlight_text(self, text_objects, search_string, looping = False, backwards = False):
         start_line, start_index = self.search_get_start(search_string)
         i = start_line
 
         found = False
         text_objects = copy.deepcopy(text_objects)
-        for text_object in text_objects[start_line:]:
-            if i != start_line:
-                start_index = 0
+        loop_range = self.search_get_range(len(text_objects), start_line, backwards)
+        for i in loop_range:
+            text_object = text_objects[i]
 
             try:
                 text, style = text_object.get_text()
             except AttributeError:
                 raise SearchError()
-            find_index = text.find(search_string, start_index)
-            if find_index != -1:
-                before = text[:find_index]
-                after = text[find_index+len(search_string):]
-                new_text = urwid.Text(
-                    [
-                        before,
-                        (self.highlight_color, search_string),
-                        after,
-                    ]
-                )
 
+            if i != start_line:
+                start_index = 0
+
+            find_index = self.search_find(text, search_string, start_index, backwards)
+
+            if find_index != -1:
+                new_text = self.search_highlight_object(text, find_index, search_string)
+                text_objects[i] = new_text
+
+                found = True
                 self.state.add_flow_setting(self.flow, "last_search_index",
                         find_index)
                 self.state.add_flow_setting(self.flow, "last_find_line", i)
 
-                text_objects[i] = new_text
-
-                found = True
-
                 break
 
-            i += 1
-
+        # handle search WRAP
         if found:
             focus_pos = i
         else :
-            # loop from the beginning, but not forever.
-            if (start_line == 0 and start_index == 0) or looping:
+            if looping:
                 focus_pos = None
             else:
-                self.state.add_flow_setting(self.flow, "last_search_index", 0)
-                self.state.add_flow_setting(self.flow, "last_find_line", 0)
-                text_objects, focus_pos = self.search_highlight_text(text_objects, search_string, True)
+                if not backwards:
+                    self.state.add_flow_setting(self.flow, "last_search_index", 0)
+                    self.state.add_flow_setting(self.flow, "last_find_line", 0)
+                else:
+                    self.state.add_flow_setting(self.flow, "last_search_index", None)
+                    self.state.add_flow_setting(self.flow, "last_find_line", len(text_objects) - 1)
+
+                text_objects, focus_pos = self.search_highlight_text(text_objects,
+                        search_string, looping=True, backwards=backwards)
 
         return text_objects, focus_pos
 
+    def search_highlight_object(self, text_object, find_index, search_string):
+        """
+            just a little abstraction
+        """
+        before = text_object[:find_index]
+        after = text_object[find_index+len(search_string):]
+
+        new_text = urwid.Text(
+            [
+                before,
+                (self.highlight_color, search_string),
+                after,
+            ]
+        )
+
+        return new_text
+
     def view_request(self):
         self.state.view_flow_mode = common.VIEW_FLOW_REQUEST
         body = self.conn_text(self.flow.request)
@@ -460,7 +525,9 @@ class FlowView(common.WWrap):
 
     def set_url(self, url):
         request = self.flow.request
-        if not request.set_url(str(url)):
+        try:
+            request.url = str(url)
+        except ValueError:
             return "Invalid URL."
         self.master.refresh_flow(self.flow)
 
@@ -503,23 +570,27 @@ class FlowView(common.WWrap):
 
     def edit(self, part):
         if self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
-            conn = self.flow.request
+            message = self.flow.request
         else:
             if not self.flow.response:
-                self.flow.response = flow.Response(
-                    self.flow.request,
+                self.flow.response = HTTPResponse(
                     self.flow.request.httpversion,
-                    200, "OK", flow.ODictCaseless(), "", None
+                    200, "OK", flow.ODictCaseless(), ""
                 )
                 self.flow.response.reply = controller.DummyReply()
-            conn = self.flow.response
+            message = self.flow.response
 
         self.flow.backup()
         if part == "r":
-            c = self.master.spawn_editor(conn.content or "")
-            conn.content = c.rstrip("\n") # what?
+            with decoded(message):
+                # Fix an issue caused by some editors when editing a request/response body.
+                # Many editors make it hard to save a file without a terminating newline on the last
+                # line. When editing message bodies, this can cause problems. For now, I just
+                # strip the newlines off the end of the body when we return from an editor.
+                c = self.master.spawn_editor(message.content or "")
+                message.content = c.rstrip("\n")
         elif part == "f":
-            if not conn.get_form_urlencoded() and conn.content:
+            if not message.get_form_urlencoded() and message.content:
                 self.master.prompt_onekey(
                     "Existing body is not a URL-encoded form. Clear and edit?",
                     [
@@ -527,26 +598,26 @@ class FlowView(common.WWrap):
                         ("no", "n"),
                     ],
                     self.edit_form_confirm,
-                    conn
+                    message
                 )
             else:
-                self.edit_form(conn)
+                self.edit_form(message)
         elif part == "h":
-            self.master.view_grideditor(grideditor.HeaderEditor(self.master, conn.headers.lst, self.set_headers, conn))
+            self.master.view_grideditor(grideditor.HeaderEditor(self.master, message.headers.lst, self.set_headers, message))
         elif part == "p":
-            p = conn.get_path_components()
+            p = message.get_path_components()
             p = [[i] for i in p]
-            self.master.view_grideditor(grideditor.PathEditor(self.master, p, self.set_path_components, conn))
+            self.master.view_grideditor(grideditor.PathEditor(self.master, p, self.set_path_components, message))
         elif part == "q":
-            self.master.view_grideditor(grideditor.QueryEditor(self.master, conn.get_query().lst, self.set_query, conn))
+            self.master.view_grideditor(grideditor.QueryEditor(self.master, message.get_query().lst, self.set_query, message))
         elif part == "u" and self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
-            self.master.prompt_edit("URL", conn.get_url(), self.set_url)
+            self.master.prompt_edit("URL", message.url, self.set_url)
         elif part == "m" and self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
             self.master.prompt_onekey("Method", self.method_options, self.edit_method)
         elif part == "c" and self.state.view_flow_mode == common.VIEW_FLOW_RESPONSE:
-            self.master.prompt_edit("Code", str(conn.code), self.set_resp_code)
+            self.master.prompt_edit("Code", str(message.code), self.set_resp_code)
         elif part == "m" and self.state.view_flow_mode == common.VIEW_FLOW_RESPONSE:
-            self.master.prompt_edit("Message", conn.msg, self.set_resp_msg)
+            self.master.prompt_edit("Message", message.msg, self.set_resp_msg)
         self.master.refresh_flow(self.flow)
 
     def _view_nextprev_flow(self, np, flow):
@@ -579,7 +650,7 @@ class FlowView(common.WWrap):
 
     def delete_body(self, t):
         if t == "m":
-            val = flow.CONTENT_MISSING
+            val = CONTENT_MISSING
         else:
             val = None
         if self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
@@ -611,7 +682,7 @@ class FlowView(common.WWrap):
             # Why doesn't this just work??
             self.w.keypress(size, key)
         elif key == "a":
-            self.flow.accept_intercept()
+            self.flow.accept_intercept(self.master)
             self.master.view_flow(self.flow)
         elif key == "A":
             self.master.accept_all()
@@ -682,7 +753,7 @@ class FlowView(common.WWrap):
             self.master.statusbar.message("")
         elif key == "m":
             p = list(contentview.view_prompts)
-            p.insert(0, ("clear", "c"))
+            p.insert(0, ("Clear", "C"))
             self.master.prompt_onekey(
                 "Display mode",
                 p,
@@ -692,7 +763,6 @@ class FlowView(common.WWrap):
         elif key == "p":
             self.view_prev_flow(self.flow)
         elif key == "r":
-            self.flow.backup()
             r = self.master.replay_request(self.flow)
             if r:
                 self.master.statusbar.message(r)
@@ -761,13 +831,9 @@ class FlowView(common.WWrap):
                     None,
                     self.search)
         elif key == "n":
-            last_search_string = self.state.get_flow_setting(self.flow, "last_search_string")
-            if last_search_string:
-                message = self.search(last_search_string)
-                if message:
-                    self.master.statusbar.message(message)
-            else:
-                self.master.statusbar.message("no previous searches have been made")
+            self.search_again(backwards=False)
+        elif key == "N":
+            self.search_again(backwards=True)
         else:
             return key
 

libmproxy/console/grideditor.py

@@ -1,6 +1,7 @@
+from __future__ import absolute_import
 import copy, re, os
 import urwid
-import common
+from . import common
 from .. import utils, filt, script
 from netlib import http_uastrings
 
@@ -122,12 +123,13 @@ class GridWalker(urwid.ListWalker):
             except ValueError:
                 self.editor.master.statusbar.message("Invalid Python-style string encoding.", 1000)
                 return
-
         errors = self.lst[self.focus][1]
         emsg = self.editor.is_error(self.focus_col, val)
         if emsg:
             self.editor.master.statusbar.message(emsg, 1000)
             errors.add(self.focus_col)
+        else:
+            errors.discard(self.focus_col)
 
         row = list(self.lst[self.focus][0])
         row[self.focus_col] = val
@@ -319,9 +321,11 @@ class GridEditor(common.WWrap):
         elif key == "d":
             self.walker.delete_focus()
         elif key == "r":
-            self.master.path_prompt("Read file: ", "", self.read_file)
+            if self.walker.get_current_value() is not None:
+                self.master.path_prompt("Read file: ", "", self.read_file)
         elif key == "R":
-            self.master.path_prompt("Read unescaped file: ", "", self.read_file, True)
+            if self.walker.get_current_value() is not None:
+                self.master.path_prompt("Read unescaped file: ", "", self.read_file, True)
         elif key == "e":
             o = self.walker.get_current_value()
             if o is not None:
@@ -492,3 +496,15 @@ class ScriptEditor(GridEditor):
             script.Script.parse_command(val)
         except script.ScriptError, v:
             return str(v)
+
+
+class HostPatternEditor(GridEditor):
+    title = "Editing host patterns"
+    columns = 1
+    headings = ("Regex (matched on hostname:port / ip:port)",)
+
+    def is_error(self, col, val):
+        try:
+            re.compile(val, re.IGNORECASE)
+        except re.error as e:
+            return "Invalid regex: %s" % str(e)

libmproxy/console/help.py

@@ -1,5 +1,6 @@
+from __future__ import absolute_import
 import urwid
-import common
+from . import common
 from .. import filt, version
 
 footer = [
@@ -23,7 +24,7 @@ class HelpView(urwid.ListBox):
 
         text.append(urwid.Text([("head", "\n\nMovement:\n")]))
         keys = [
-            ("j, k", "up, down"),
+            ("j, k", "down, up"),
             ("h, l", "left, right (in some contexts)"),
             ("space", "page down"),
             ("pg up/down", "page up/down"),
@@ -35,6 +36,7 @@ class HelpView(urwid.ListBox):
         keys = [
             ("c", "client replay"),
             ("H", "edit global header set patterns"),
+            ("I", "set ignore pattern"),
             ("i", "set interception pattern"),
             ("M", "change global default display mode"),
                 (None,
@@ -77,6 +79,10 @@ class HelpView(urwid.ListBox):
                     common.highlight_key("xml", "x") +
                     [("text", ": XML")]
                 ),
+                (None,
+                    common.highlight_key("wbxml", "w") +
+                    [("text", ": WBXML")]
+                ),
                 (None,
                     common.highlight_key("amf", "f") +
                     [("text", ": AMF (requires PyAMF)")]
@@ -109,11 +115,11 @@ class HelpView(urwid.ListBox):
 
             ("q", "quit / return to flow list"),
             ("Q", "quit without confirm prompt"),
-            ("P", "set reverse proxy mode"),
             ("R", "edit replacement patterns"),
-            ("s", "set/unset script"),
+            ("s", "add/remove scripts"),
             ("S", "server replay"),
             ("t", "set sticky cookie expression"),
+            ("T", "set tcp proxying pattern"),
             ("u", "set sticky auth expression"),
         ]
         text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))

libmproxy/contrib/README

@@ -11,5 +11,4 @@ jsbeautifier, git checkout 25/03/12, MIT license
 
 html2text, git checkout 18/08/12, GPLv3
 
-md5crypt, PSF license, http://code.activestate.com/recipes/325204/
-
+WinDivert 1.1.4, LGPL license, http://reqrypt.org/windivert.html

libmproxy/contrib/jsbeautifier/unpackers/__init__.py

@@ -7,7 +7,7 @@
 
 import pkgutil
 import re
-from jsbeautifier.unpackers import evalbased
+from . import evalbased
 
 # NOTE: AT THE MOMENT, IT IS DEACTIVATED FOR YOUR SECURITY: it runs js!
 BLACKLIST = ['jsbeautifier.unpackers.evalbased']

libmproxy/contrib/jsbeautifier/unpackers/myobfuscate.py

@@ -40,7 +40,7 @@ try:
 except ImportError:
     from urllib.parse import unquote
 
-from jsbeautifier.unpackers import UnpackingError
+from . import UnpackingError
 
 PRIORITY = 1
 

libmproxy/contrib/jsbeautifier/unpackers/packer.py

@@ -14,7 +14,7 @@
 
 import re
 import string
-from jsbeautifier.unpackers import UnpackingError
+from . import UnpackingError
 
 PRIORITY = 1
 

libmproxy/contrib/wbxml/ASCommandResponse.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+'''
+@author: David Shaw, david.shaw.aw@gmail.com
+
+Inspired by EAS Inspector for Fiddler
+https://easinspectorforfiddler.codeplex.com
+
+----- The MIT License (MIT) ----- 
+Filename: ASCommandResponse.py
+Copyright (c) 2014, David P. Shaw
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+'''
+from ASWBXML import ASWBXML
+import logging
+
+class ASCommandResponse:
+
+	def __init__(self, response):
+		self.wbxmlBody = response
+		try:
+			if ( len(response) > 0):
+				self.xmlString = self.decodeWBXML(self.wbxmlBody)
+			else:
+				logging.error("Empty WBXML body passed")
+		except Exception as e:
+			logging.error("Error: {0}".format(e.message))
+			self.xmlString = None
+
+	def getWBXMLBytes(self):
+		return self.wbxmlBytes
+	
+	def getXMLString(self):
+		return self.xmlString
+	
+	def decodeWBXML(self, body):
+		self.instance = ASWBXML()
+		self.instance.loadBytes(body)
+		return self.instance.getXml()
+
+if __name__ == "__main__":
+	import os	
+	logging.basicConfig(level=logging.INFO)
+
+	projectDir = os.path.dirname(os.path.realpath("."))
+	samplesDir = os.path.join(projectDir, "Samples/")
+	listOfSamples = os.listdir(samplesDir)
+
+	for filename in listOfSamples:
+		byteWBXML = open(samplesDir + os.sep + filename, "rb").read()
+		
+		logging.info("-"*100)
+		logging.info(filename)
+		logging.info("-"*100)
+		instance = ASCommandResponse(byteWBXML)
+		logging.info(instance.xmlString)
+		

libmproxy/contrib/wbxml/ASWBXML.py

@@ -0,0 +1,903 @@
+#!/usr/bin/env python
+'''
+@author: David Shaw, david.shaw.aw@gmail.com
+
+Inspired by EAS Inspector for Fiddler
+https://easinspectorforfiddler.codeplex.com
+
+----- The MIT License (MIT) ----- 
+Filename: ASWBXML.py
+Copyright (c) 2014, David P. Shaw
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+'''
+import xml.dom.minidom
+import logging
+
+from ASWBXMLCodePage import ASWBXMLCodePage
+from ASWBXMLByteQueue import ASWBXMLByteQueue
+from GlobalTokens import GlobalTokens
+from InvalidDataException import InvalidDataException
+
+class ASWBXML:
+	versionByte = 0x03
+	publicIdentifierByte = 0x01
+	characterSetByte = 0x6A
+	stringTableLengthByte = 0x00
+	
+	def __init__(self):
+		
+		# empty on init
+		self.xmlDoc = xml.dom.minidom.Document()
+		self.currentCodePage = 0
+		self.defaultCodePage = -1
+		
+		# Load up code pages
+		# Currently there are 25 code pages as per MS-ASWBXML
+		self.codePages = []
+
+		# region Code Page Initialization
+		# Code Page 0: AirSync
+		# region AirSync Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "AirSync:"
+		page.xmlns = "airsync"
+
+		page.addToken(0x05, "Sync")
+		page.addToken(0x06, "Responses")
+		page.addToken(0x07, "Add")
+		page.addToken(0x08, "Change")
+		page.addToken(0x09, "Delete")
+		page.addToken(0x0A, "Fetch")
+		page.addToken(0x0B, "SyncKey")
+		page.addToken(0x0C, "ClientId")
+		page.addToken(0x0D, "ServerId")
+		page.addToken(0x0E, "Status")
+		page.addToken(0x0F, "Collection")
+		page.addToken(0x10, "Class")
+		page.addToken(0x12, "CollectionId")
+		page.addToken(0x13, "GetChanges")
+		page.addToken(0x14, "MoreAvailable")
+		page.addToken(0x15, "WindowSize")
+		page.addToken(0x16, "Commands")
+		page.addToken(0x17, "Options")
+		page.addToken(0x18, "FilterType")
+		page.addToken(0x1B, "Conflict")
+		page.addToken(0x1C, "Collections")
+		page.addToken(0x1D, "ApplicationData")
+		page.addToken(0x1E, "DeletesAsMoves")
+		page.addToken(0x20, "Supported")
+		page.addToken(0x21, "SoftDelete")
+		page.addToken(0x22, "MIMESupport")
+		page.addToken(0x23, "MIMETruncation")
+		page.addToken(0x24, "Wait")
+		page.addToken(0x25, "Limit")
+		page.addToken(0x26, "Partial")
+		page.addToken(0x27, "ConversationMode")
+		page.addToken(0x28, "MaxItems")
+		page.addToken(0x29, "HeartbeatInterval")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 1: Contacts
+		# region Contacts Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Contacts:"
+		page.xmlns = "contacts"
+
+		page.addToken(0x05, "Anniversary")
+		page.addToken(0x06, "AssistantName")
+		page.addToken(0x07, "AssistantTelephoneNumber")
+		page.addToken(0x08, "Birthday")
+		page.addToken(0x0C, "Business2PhoneNumber")
+		page.addToken(0x0D, "BusinessCity")
+		page.addToken(0x0E, "BusinessCountry")
+		page.addToken(0x0F, "BusinessPostalCode")
+		page.addToken(0x10, "BusinessState")
+		page.addToken(0x11, "BusinessStreet")
+		page.addToken(0x12, "BusinessFaxNumber")
+		page.addToken(0x13, "BusinessPhoneNumber")
+		page.addToken(0x14, "CarPhoneNumber")
+		page.addToken(0x15, "Categories")
+		page.addToken(0x16, "Category")
+		page.addToken(0x17, "Children")
+		page.addToken(0x18, "Child")
+		page.addToken(0x19, "CompanyName")
+		page.addToken(0x1A, "Department")
+		page.addToken(0x1B, "Email1Address")
+		page.addToken(0x1C, "Email2Address")
+		page.addToken(0x1D, "Email3Address")
+		page.addToken(0x1E, "FileAs")
+		page.addToken(0x1F, "FirstName")
+		page.addToken(0x20, "Home2PhoneNumber")
+		page.addToken(0x21, "HomeCity")
+		page.addToken(0x22, "HomeCountry")
+		page.addToken(0x23, "HomePostalCode")
+		page.addToken(0x24, "HomeState")
+		page.addToken(0x25, "HomeStreet")
+		page.addToken(0x26, "HomeFaxNumber")
+		page.addToken(0x27, "HomePhoneNumber")
+		page.addToken(0x28, "JobTitle")
+		page.addToken(0x29, "LastName")
+		page.addToken(0x2A, "MiddleName")
+		page.addToken(0x2B, "MobilePhoneNumber")
+		page.addToken(0x2C, "OfficeLocation")
+		page.addToken(0x2D, "OtherCity")
+		page.addToken(0x2E, "OtherCountry")
+		page.addToken(0x2F, "OtherPostalCode")
+		page.addToken(0x30, "OtherState")
+		page.addToken(0x31, "OtherStreet")
+		page.addToken(0x32, "PagerNumber")
+		page.addToken(0x33, "RadioPhoneNumber")
+		page.addToken(0x34, "Spouse")
+		page.addToken(0x35, "Suffix")
+		page.addToken(0x36, "Title")
+		page.addToken(0x37, "Webpage")
+		page.addToken(0x38, "YomiCompanyName")
+		page.addToken(0x39, "YomiFirstName")
+		page.addToken(0x3A, "YomiLastName")
+		page.addToken(0x3C, "Picture")
+		page.addToken(0x3D, "Alias")
+		page.addToken(0x3E, "WeightedRank")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 2: Email
+		# region Email Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Email:"
+		page.xmlns = "email"
+
+		page.addToken(0x0F, "DateReceived")
+		page.addToken(0x11, "DisplayTo")
+		page.addToken(0x12, "Importance")
+		page.addToken(0x13, "MessageClass")
+		page.addToken(0x14, "Subject")
+		page.addToken(0x15, "Read")
+		page.addToken(0x16, "To")
+		page.addToken(0x17, "CC")
+		page.addToken(0x18, "From")
+		page.addToken(0x19, "ReplyTo")
+		page.addToken(0x1A, "AllDayEvent")
+		page.addToken(0x1B, "Categories")
+		page.addToken(0x1C, "Category")
+		page.addToken(0x1D, "DTStamp")
+		page.addToken(0x1E, "EndTime")
+		page.addToken(0x1F, "InstanceType")
+		page.addToken(0x20, "BusyStatus")
+		page.addToken(0x21, "Location")
+		page.addToken(0x22, "MeetingRequest")
+		page.addToken(0x23, "Organizer")
+		page.addToken(0x24, "RecurrenceId")
+		page.addToken(0x25, "Reminder")
+		page.addToken(0x26, "ResponseRequested")
+		page.addToken(0x27, "Recurrences")
+		page.addToken(0x28, "Recurrence")
+		page.addToken(0x29, "Recurrence_Type")
+		page.addToken(0x2A, "Recurrence_Until")
+		page.addToken(0x2B, "Recurrence_Occurrences")
+		page.addToken(0x2C, "Recurrence_Interval")
+		page.addToken(0x2D, "Recurrence_DayOfWeek")
+		page.addToken(0x2E, "Recurrence_DayOfMonth")
+		page.addToken(0x2F, "Recurrence_WeekOfMonth")
+		page.addToken(0x30, "Recurrence_MonthOfYear")
+		page.addToken(0x31, "StartTime")
+		page.addToken(0x32, "Sensitivity")
+		page.addToken(0x33, "TimeZone")
+		page.addToken(0x34, "GlobalObjId")
+		page.addToken(0x35, "ThreadTopic")
+		page.addToken(0x39, "InternetCPID")
+		page.addToken(0x3A, "Flag")
+		page.addToken(0x3B, "FlagStatus")
+		page.addToken(0x3C, "ContentClass")
+		page.addToken(0x3D, "FlagType")
+		page.addToken(0x3E, "CompleteTime")
+		page.addToken(0x3F, "DisallowNewTimeProposal")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 3: AirNotify - retired
+		# region AirNotify Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = ""
+		page.xmlns = ""
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 4: Calendar
+		# region Calendar Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Calendar:"
+		page.xmlns = "calendar"
+
+		page.addToken(0x05, "TimeZone")
+		page.addToken(0x06, "AllDayEvent")
+		page.addToken(0x07, "Attendees")
+		page.addToken(0x08, "Attendee")
+		page.addToken(0x09, "Attendee_Email")
+		page.addToken(0x0A, "Attendee_Name")
+		page.addToken(0x0D, "BusyStatus")
+		page.addToken(0x0E, "Categories")
+		page.addToken(0x0F, "Category")
+		page.addToken(0x11, "DTStamp")
+		page.addToken(0x12, "EndTime")
+		page.addToken(0x13, "Exception")
+		page.addToken(0x14, "Exceptions")
+		page.addToken(0x15, "Exception_Deleted")
+		page.addToken(0x16, "Exception_StartTime")
+		page.addToken(0x17, "Location")
+		page.addToken(0x18, "MeetingStatus")
+		page.addToken(0x19, "Organizer_Email")
+		page.addToken(0x1A, "Organizer_Name")
+		page.addToken(0x1B, "Recurrence")
+		page.addToken(0x1C, "Recurrence_Type")
+		page.addToken(0x1D, "Recurrence_Until")
+		page.addToken(0x1E, "Recurrence_Occurrences")
+		page.addToken(0x1F, "Recurrence_Interval")
+		page.addToken(0x20, "Recurrence_DayOfWeek")
+		page.addToken(0x21, "Recurrence_DayOfMonth")
+		page.addToken(0x22, "Recurrence_WeekOfMonth")
+		page.addToken(0x23, "Recurrence_MonthOfYear")
+		page.addToken(0x24, "Reminder")
+		page.addToken(0x25, "Sensitivity")
+		page.addToken(0x26, "Subject")
+		page.addToken(0x27, "StartTime")
+		page.addToken(0x28, "UID")
+		page.addToken(0x29, "Attendee_Status")
+		page.addToken(0x2A, "Attendee_Type")
+		page.addToken(0x33, "DisallowNewTimeProposal")
+		page.addToken(0x34, "ResponseRequested")
+		page.addToken(0x35, "AppointmentReplyTime")
+		page.addToken(0x36, "ResponseType")
+		page.addToken(0x37, "CalendarType")
+		page.addToken(0x38, "IsLeapMonth")
+		page.addToken(0x39, "FirstDayOfWeek")
+		page.addToken(0x3A, "OnlineMeetingConfLink")
+		page.addToken(0x3B, "OnlineMeetingExternalLink")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 5: Move
+		# region Move Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Move:"
+		page.xmlns = "move"
+
+		page.addToken(0x05, "MoveItems")
+		page.addToken(0x06, "Move")
+		page.addToken(0x07, "SrcMsgId")
+		page.addToken(0x08, "SrcFldId")
+		page.addToken(0x09, "DstFldId")
+		page.addToken(0x0A, "Response")
+		page.addToken(0x0B, "Status")
+		page.addToken(0x0C, "DstMsgId")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 6: ItemEstimate
+		# region ItemEstimate Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "GetItemEstimate:"
+		page.xmlns = "getitemestimate"
+
+		page.addToken(0x05, "GetItemEstimate")
+		page.addToken(0x06, "Version")
+		page.addToken(0x07, "Collections")
+		page.addToken(0x08, "Collection")
+		page.addToken(0x09, "Class")
+		page.addToken(0x0A, "CollectionId")
+		page.addToken(0x0B, "DateTime")
+		page.addToken(0x0C, "Estimate")
+		page.addToken(0x0D, "Response")
+		page.addToken(0x0E, "Status")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 7: FolderHierarchy
+		# region FolderHierarchy Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "FolderHierarchy:"
+		page.xmlns = "folderhierarchy"
+
+		page.addToken(0x07, "DisplayName")
+		page.addToken(0x08, "ServerId")
+		page.addToken(0x09, "ParentId")
+		page.addToken(0x0A, "Type")
+		page.addToken(0x0C, "Status")
+		page.addToken(0x0E, "Changes")
+		page.addToken(0x0F, "Add")
+		page.addToken(0x10, "Delete")
+		page.addToken(0x11, "Update")
+		page.addToken(0x12, "SyncKey")
+		page.addToken(0x13, "FolderCreate")
+		page.addToken(0x14, "FolderDelete")
+		page.addToken(0x15, "FolderUpdate")
+		page.addToken(0x16, "FolderSync")
+		page.addToken(0x17, "Count")
+
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 8: MeetingResponse
+		# region MeetingResponse Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "MeetingResponse:"
+		page.xmlns = "meetingresponse"
+
+		page.addToken(0x05, "CalendarId")
+		page.addToken(0x06, "CollectionId")
+		page.addToken(0x07, "MeetingResponse")
+		page.addToken(0x08, "RequestId")
+		page.addToken(0x09, "Request")
+		page.addToken(0x0A, "Result")
+		page.addToken(0x0B, "Status")
+		page.addToken(0x0C, "UserResponse")
+		page.addToken(0x0E, "InstanceId")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 9: Tasks
+		# region Tasks Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Tasks:"
+		page.xmlns = "tasks"
+
+		page.addToken(0x08, "Categories")
+		page.addToken(0x09, "Category")
+		page.addToken(0x0A, "Complete")
+		page.addToken(0x0B, "DateCompleted")
+		page.addToken(0x0C, "DueDate")
+		page.addToken(0x0D, "UTCDueDate")
+		page.addToken(0x0E, "Importance")
+		page.addToken(0x0F, "Recurrence")
+		page.addToken(0x10, "Recurrence_Type")
+		page.addToken(0x11, "Recurrence_Start")
+		page.addToken(0x12, "Recurrence_Until")
+		page.addToken(0x13, "Recurrence_Occurrences")
+		page.addToken(0x14, "Recurrence_Interval")
+		page.addToken(0x15, "Recurrence_DayOfMonth")
+		page.addToken(0x16, "Recurrence_DayOfWeek")
+		page.addToken(0x17, "Recurrence_WeekOfMonth")
+		page.addToken(0x18, "Recurrence_MonthOfYear")
+		page.addToken(0x19, "Recurrence_Regenerate")
+		page.addToken(0x1A, "Recurrence_DeadOccur")
+		page.addToken(0x1B, "ReminderSet")
+		page.addToken(0x1C, "ReminderTime")
+		page.addToken(0x1D, "Sensitivity")
+		page.addToken(0x1E, "StartDate")
+		page.addToken(0x1F, "UTCStartDate")
+		page.addToken(0x20, "Subject")
+		page.addToken(0x22, "OrdinalDate")
+		page.addToken(0x23, "SubOrdinalDate")
+		page.addToken(0x24, "CalendarType")
+		page.addToken(0x25, "IsLeapMonth")
+		page.addToken(0x26, "FirstDayOfWeek")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 10: ResolveRecipients
+		# region ResolveRecipients Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "ResolveRecipients:"
+		page.xmlns = "resolverecipients"
+
+		page.addToken(0x05, "ResolveRecipients")
+		page.addToken(0x06, "Response")
+		page.addToken(0x07, "Status")
+		page.addToken(0x08, "Type")
+		page.addToken(0x09, "Recipient")
+		page.addToken(0x0A, "DisplayName")
+		page.addToken(0x0B, "EmailAddress")
+		page.addToken(0x0C, "Certificates")
+		page.addToken(0x0D, "Certificate")
+		page.addToken(0x0E, "MiniCertificate")
+		page.addToken(0x0F, "Options")
+		page.addToken(0x10, "To")
+		page.addToken(0x11, "CertificateRetrieval")
+		page.addToken(0x12, "RecipientCount")
+		page.addToken(0x13, "MaxCertificates")
+		page.addToken(0x14, "MaxAmbiguousRecipients")
+		page.addToken(0x15, "CertificateCount")
+		page.addToken(0x16, "Availability")
+		page.addToken(0x17, "StartTime")
+		page.addToken(0x18, "EndTime")
+		page.addToken(0x19, "MergedFreeBusy")
+		page.addToken(0x1A, "Picture")
+		page.addToken(0x1B, "MaxSize")
+		page.addToken(0x1C, "Data")
+		page.addToken(0x1D, "MaxPictures")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 11: ValidateCert
+		# region ValidateCert Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "ValidateCert:"
+		page.xmlns = "validatecert"
+
+		page.addToken(0x05, "ValidateCert")
+		page.addToken(0x06, "Certificates")
+		page.addToken(0x07, "Certificate")
+		page.addToken(0x08, "CertificateChain")
+		page.addToken(0x09, "CheckCRL")
+		page.addToken(0x0A, "Status")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 12: Contacts2
+		# region Contacts2 Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Contacts2:"
+		page.xmlns = "contacts2"
+
+		page.addToken(0x05, "CustomerId")
+		page.addToken(0x06, "GovernmentId")
+		page.addToken(0x07, "IMAddress")
+		page.addToken(0x08, "IMAddress2")
+		page.addToken(0x09, "IMAddress3")
+		page.addToken(0x0A, "ManagerName")
+		page.addToken(0x0B, "CompanyMainPhone")
+		page.addToken(0x0C, "AccountName")
+		page.addToken(0x0D, "NickName")
+		page.addToken(0x0E, "MMS")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 13: Ping
+		# region Ping Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Ping:"
+		page.xmlns = "ping"
+
+		page.addToken(0x05, "Ping")
+		page.addToken(0x06, "AutdState")  # Per MS-ASWBXML, this tag is not used by protocol
+		page.addToken(0x07, "Status")
+		page.addToken(0x08, "HeartbeatInterval")
+		page.addToken(0x09, "Folders")
+		page.addToken(0x0A, "Folder")
+		page.addToken(0x0B, "Id")
+		page.addToken(0x0C, "Class")
+		page.addToken(0x0D, "MaxFolders")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 14: Provision
+		# region Provision Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Provision:"
+		page.xmlns = "provision"
+
+		page.addToken(0x05, "Provision")
+		page.addToken(0x06, "Policies")
+		page.addToken(0x07, "Policy")
+		page.addToken(0x08, "PolicyType")
+		page.addToken(0x09, "PolicyKey")
+		page.addToken(0x0A, "Data")
+		page.addToken(0x0B, "Status")
+		page.addToken(0x0C, "RemoteWipe")
+		page.addToken(0x0D, "EASProvisionDoc")
+		page.addToken(0x0E, "DevicePasswordEnabled")
+		page.addToken(0x0F, "AlphanumericDevicePasswordRequired")
+		page.addToken(0x10, "RequireStorageCardEncryption")
+		page.addToken(0x11, "PasswordRecoveryEnabled")
+		page.addToken(0x13, "AttachmentsEnabled")
+		page.addToken(0x14, "MinDevicePasswordLength")
+		page.addToken(0x15, "MaxInactivityTimeDeviceLock")
+		page.addToken(0x16, "MaxDevicePasswordFailedAttempts")
+		page.addToken(0x17, "MaxAttachmentSize")
+		page.addToken(0x18, "AllowSimpleDevicePassword")
+		page.addToken(0x19, "DevicePasswordExpiration")
+		page.addToken(0x1A, "DevicePasswordHistory")
+		page.addToken(0x1B, "AllowStorageCard")
+		page.addToken(0x1C, "AllowCamera")
+		page.addToken(0x1D, "RequireDeviceEncryption")
+		page.addToken(0x1E, "AllowUnsignedApplications")
+		page.addToken(0x1F, "AllowUnsignedInstallationPackages")
+		page.addToken(0x20, "MinDevicePasswordComplexCharacters")
+		page.addToken(0x21, "AllowWiFi")
+		page.addToken(0x22, "AllowTextMessaging")
+		page.addToken(0x23, "AllowPOPIMAPEmail")
+		page.addToken(0x24, "AllowBluetooth")
+		page.addToken(0x25, "AllowIrDA")
+		page.addToken(0x26, "RequireManualSyncWhenRoaming")
+		page.addToken(0x27, "AllowDesktopSync")
+		page.addToken(0x28, "MaxCalendarAgeFilter")
+		page.addToken(0x29, "AllowHTMLEmail")
+		page.addToken(0x2A, "MaxEmailAgeFilter")
+		page.addToken(0x2B, "MaxEmailBodyTruncationSize")
+		page.addToken(0x2C, "MaxEmailHTMLBodyTruncationSize")
+		page.addToken(0x2D, "RequireSignedSMIMEMessages")
+		page.addToken(0x2E, "RequireEncryptedSMIMEMessages")
+		page.addToken(0x2F, "RequireSignedSMIMEAlgorithm")
+		page.addToken(0x30, "RequireEncryptionSMIMEAlgorithm")
+		page.addToken(0x31, "AllowSMIMEEncryptionAlgorithmNegotiation")
+		page.addToken(0x32, "AllowSMIMESoftCerts")
+		page.addToken(0x33, "AllowBrowser")
+		page.addToken(0x34, "AllowConsumerEmail")
+		page.addToken(0x35, "AllowRemoteDesktop")
+		page.addToken(0x36, "AllowInternetSharing")
+		page.addToken(0x37, "UnapprovedInROMApplicationList")
+		page.addToken(0x38, "ApplicationName")
+		page.addToken(0x39, "ApprovedApplicationList")
+		page.addToken(0x3A, "Hash")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 15: Search
+		# region Search Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Search:"
+		page.xmlns = "search"
+
+		page.addToken(0x05, "Search")
+		page.addToken(0x07, "Store")
+		page.addToken(0x08, "Name")
+		page.addToken(0x09, "Query")
+		page.addToken(0x0A, "Options")
+		page.addToken(0x0B, "Range")
+		page.addToken(0x0C, "Status")
+		page.addToken(0x0D, "Response")
+		page.addToken(0x0E, "Result")
+		page.addToken(0x0F, "Properties")
+		page.addToken(0x10, "Total")
+		page.addToken(0x11, "EqualTo")
+		page.addToken(0x12, "Value")
+		page.addToken(0x13, "And")
+		page.addToken(0x14, "Or")
+		page.addToken(0x15, "FreeText")
+		page.addToken(0x17, "DeepTraversal")
+		page.addToken(0x18, "LongId")
+		page.addToken(0x19, "RebuildResults")
+		page.addToken(0x1A, "LessThan")
+		page.addToken(0x1B, "GreaterThan")
+		page.addToken(0x1E, "UserName")
+		page.addToken(0x1F, "Password")
+		page.addToken(0x20, "ConversationId")
+		page.addToken(0x21, "Picture")
+		page.addToken(0x22, "MaxSize")
+		page.addToken(0x23, "MaxPictures")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 16: GAL
+		# region GAL Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "GAL:"
+		page.xmlns = "gal"
+
+		page.addToken(0x05, "DisplayName")
+		page.addToken(0x06, "Phone")
+		page.addToken(0x07, "Office")
+		page.addToken(0x08, "Title")
+		page.addToken(0x09, "Company")
+		page.addToken(0x0A, "Alias")
+		page.addToken(0x0B, "FirstName")
+		page.addToken(0x0C, "LastName")
+		page.addToken(0x0D, "HomePhone")
+		page.addToken(0x0E, "MobilePhone")
+		page.addToken(0x0F, "EmailAddress")
+		page.addToken(0x10, "Picture")
+		page.addToken(0x11, "Status")
+		page.addToken(0x12, "Data")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 17: AirSyncBase
+		# region AirSyncBase Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "AirSyncBase:"
+		page.xmlns = "airsyncbase"
+
+		page.addToken(0x05, "BodyPreference")
+		page.addToken(0x06, "Type")
+		page.addToken(0x07, "TruncationSize")
+		page.addToken(0x08, "AllOrNone")
+		page.addToken(0x0A, "Body")
+		page.addToken(0x0B, "Data")
+		page.addToken(0x0C, "EstimatedDataSize")
+		page.addToken(0x0D, "Truncated")
+		page.addToken(0x0E, "Attachments")
+		page.addToken(0x0F, "Attachment")
+		page.addToken(0x10, "DisplayName")
+		page.addToken(0x11, "FileReference")
+		page.addToken(0x12, "Method")
+		page.addToken(0x13, "ContentId")
+		page.addToken(0x14, "ContentLocation")
+		page.addToken(0x15, "IsInline")
+		page.addToken(0x16, "NativeBodyType")
+		page.addToken(0x17, "ContentType")
+		page.addToken(0x18, "Preview")
+		page.addToken(0x19, "BodyPartPreference")
+		page.addToken(0x1A, "BodyPart")
+		page.addToken(0x1B, "Status")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 18: Settings
+		# region Settings Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Settings:"
+		page.xmlns = "settings"
+
+		page.addToken(0x05, "Settings")
+		page.addToken(0x06, "Status")
+		page.addToken(0x07, "Get")
+		page.addToken(0x08, "Set")
+		page.addToken(0x09, "Oof")
+		page.addToken(0x0A, "OofState")
+		page.addToken(0x0B, "StartTime")
+		page.addToken(0x0C, "EndTime")
+		page.addToken(0x0D, "OofMessage")
+		page.addToken(0x0E, "AppliesToInternal")
+		page.addToken(0x0F, "AppliesToExternalKnown")
+		page.addToken(0x10, "AppliesToExternalUnknown")
+		page.addToken(0x11, "Enabled")
+		page.addToken(0x12, "ReplyMessage")
+		page.addToken(0x13, "BodyType")
+		page.addToken(0x14, "DevicePassword")
+		page.addToken(0x15, "Password")
+		page.addToken(0x16, "DeviceInformation")
+		page.addToken(0x17, "Model")
+		page.addToken(0x18, "IMEI")
+		page.addToken(0x19, "FriendlyName")
+		page.addToken(0x1A, "OS")
+		page.addToken(0x1B, "OSLanguage")
+		page.addToken(0x1C, "PhoneNumber")
+		page.addToken(0x1D, "UserInformation")
+		page.addToken(0x1E, "EmailAddresses")
+		page.addToken(0x1F, "SmtpAddress")
+		page.addToken(0x20, "UserAgent")
+		page.addToken(0x21, "EnableOutboundSMS")
+		page.addToken(0x22, "MobileOperator")
+		page.addToken(0x23, "PrimarySmtpAddress")
+		page.addToken(0x24, "Accounts")
+		page.addToken(0x25, "Account")
+		page.addToken(0x26, "AccountId")
+		page.addToken(0x27, "AccountName")
+		page.addToken(0x28, "UserDisplayName")
+		page.addToken(0x29, "SendDisabled")
+		page.addToken(0x2B, "RightsManagementInformation")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 19: DocumentLibrary
+		# region DocumentLibrary Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "DocumentLibrary:"
+		page.xmlns = "documentlibrary"
+
+		page.addToken(0x05, "LinkId")
+		page.addToken(0x06, "DisplayName")
+		page.addToken(0x07, "IsFolder")
+		page.addToken(0x08, "CreationDate")
+		page.addToken(0x09, "LastModifiedDate")
+		page.addToken(0x0A, "IsHidden")
+		page.addToken(0x0B, "ContentLength")
+		page.addToken(0x0C, "ContentType")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 20: ItemOperations
+		# region ItemOperations Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "ItemOperations:"
+		page.xmlns = "itemoperations"
+
+		page.addToken(0x05, "ItemOperations")
+		page.addToken(0x06, "Fetch")
+		page.addToken(0x07, "Store")
+		page.addToken(0x08, "Options")
+		page.addToken(0x09, "Range")
+		page.addToken(0x0A, "Total")
+		page.addToken(0x0B, "Properties")
+		page.addToken(0x0C, "Data")
+		page.addToken(0x0D, "Status")
+		page.addToken(0x0E, "Response")
+		page.addToken(0x0F, "Version")
+		page.addToken(0x10, "Schema")
+		page.addToken(0x11, "Part")
+		page.addToken(0x12, "EmptyFolderContents")
+		page.addToken(0x13, "DeleteSubFolders")
+		page.addToken(0x14, "UserName")
+		page.addToken(0x15, "Password")
+		page.addToken(0x16, "Move")
+		page.addToken(0x17, "DstFldId")
+		page.addToken(0x18, "ConversationId")
+		page.addToken(0x19, "MoveAlways")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 21: ComposeMail
+		# region ComposeMail Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "ComposeMail:"
+		page.xmlns = "composemail"
+
+		page.addToken(0x05, "SendMail")
+		page.addToken(0x06, "SmartForward")
+		page.addToken(0x07, "SmartReply")
+		page.addToken(0x08, "SaveInSentItems")
+		page.addToken(0x09, "ReplaceMime")
+		page.addToken(0x0B, "Source")
+		page.addToken(0x0C, "FolderId")
+		page.addToken(0x0D, "ItemId")
+		page.addToken(0x0E, "LongId")
+		page.addToken(0x0F, "InstanceId")
+		page.addToken(0x10, "MIME")
+		page.addToken(0x11, "ClientId")
+		page.addToken(0x12, "Status")
+		page.addToken(0x13, "AccountId")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 22: Email2
+		# region Email2 Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Email2:"
+		page.xmlns = "email2"
+
+		page.addToken(0x05, "UmCallerID")
+		page.addToken(0x06, "UmUserNotes")
+		page.addToken(0x07, "UmAttDuration")
+		page.addToken(0x08, "UmAttOrder")
+		page.addToken(0x09, "ConversationId")
+		page.addToken(0x0A, "ConversationIndex")
+		page.addToken(0x0B, "LastVerbExecuted")
+		page.addToken(0x0C, "LastVerbExecutionTime")
+		page.addToken(0x0D, "ReceivedAsBcc")
+		page.addToken(0x0E, "Sender")
+		page.addToken(0x0F, "CalendarType")
+		page.addToken(0x10, "IsLeapMonth")
+		page.addToken(0x11, "AccountId")
+		page.addToken(0x12, "FirstDayOfWeek")
+		page.addToken(0x13, "MeetingMessageType")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 23: Notes
+		# region Notes Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Notes:"
+		page.xmlns = "notes"
+
+		page.addToken(0x05, "Subject")
+		page.addToken(0x06, "MessageClass")
+		page.addToken(0x07, "LastModifiedDate")
+		page.addToken(0x08, "Categories")
+		page.addToken(0x09, "Category")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 24: RightsManagement
+		# region RightsManagement Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "RightsManagement:"
+		page.xmlns = "rightsmanagement"
+
+		page.addToken(0x05, "RightsManagementSupport")
+		page.addToken(0x06, "RightsManagementTemplates")
+		page.addToken(0x07, "RightsManagementTemplate")
+		page.addToken(0x08, "RightsManagementLicense")
+		page.addToken(0x09, "EditAllowed")
+		page.addToken(0x0A, "ReplyAllowed")
+		page.addToken(0x0B, "ReplyAllAllowed")
+		page.addToken(0x0C, "ForwardAllowed")
+		page.addToken(0x0D, "ModifyRecipientsAllowed")
+		page.addToken(0x0E, "ExtractAllowed")
+		page.addToken(0x0F, "PrintAllowed")
+		page.addToken(0x10, "ExportAllowed")
+		page.addToken(0x11, "ProgrammaticAccessAllowed")
+		page.addToken(0x12, "RMOwner")
+		page.addToken(0x13, "ContentExpiryDate")
+		page.addToken(0x14, "TemplateID")
+		page.addToken(0x15, "TemplateName")
+		page.addToken(0x16, "TemplateDescription")
+		page.addToken(0x17, "ContentOwner")
+		page.addToken(0x18, "RemoveRightsManagementDistribution")
+		self.codePages.append(page)
+		# endregion
+		# endregion
+	
+	def loadXml(self, strXML):
+		# note xmlDoc has .childNodes and .parentNode
+		self.xmlDoc = xml.dom.minidom.parseString(strXML)
+
+	def getXml(self):
+		if (self.xmlDoc != None):
+			try:
+				return self.xmlDoc.toprettyxml(indent="    ", newl="\n")
+			except:
+				return self.xmlDoc.toxml()
+	
+	def loadBytes(self, byteWBXML):
+		
+		currentNode = self.xmlDoc
+		
+		wbXMLBytes = ASWBXMLByteQueue(byteWBXML)
+		# Version is ignored
+		version = wbXMLBytes.dequeueAndLog()
+
+		# Public Identifier is ignored
+		publicId = wbXMLBytes.dequeueMultibyteInt()
+		
+		logging.debug("Version: %d, Public Identifier: %d" % (version, publicId))
+		
+		# Character set
+		# Currently only UTF-8 is supported, throw if something else
+		charset = wbXMLBytes.dequeueMultibyteInt()
+		if (charset != 0x6A):
+			raise InvalidDataException("ASWBXML only supports UTF-8 encoded XML.")
+
+		# String table length
+		# This should be 0, MS-ASWBXML does not use string tables
+		stringTableLength = wbXMLBytes.dequeueMultibyteInt()
+		if (stringTableLength != 0):
+			raise InvalidDataException("WBXML data contains a string table.")
+
+		# Now we should be at the body of the data.
+		# Add the declaration
+		unusedArray = [GlobalTokens.ENTITY, GlobalTokens.EXT_0, GlobalTokens.EXT_1, GlobalTokens.EXT_2, GlobalTokens.EXT_I_0, GlobalTokens.EXT_I_1, GlobalTokens.EXT_I_2, GlobalTokens.EXT_T_0, GlobalTokens.EXT_T_1, GlobalTokens.EXT_T_2, GlobalTokens.LITERAL, GlobalTokens.LITERAL_A, GlobalTokens.LITERAL_AC, GlobalTokens.LITERAL_C, GlobalTokens.PI, GlobalTokens.STR_T]
+		
+		while ( wbXMLBytes.qsize() > 0):
+			currentByte = wbXMLBytes.dequeueAndLog()
+			if ( currentByte == GlobalTokens.SWITCH_PAGE ):
+				newCodePage = wbXMLBytes.dequeueAndLog()
+				if (newCodePage >= 0 and newCodePage < 25):
+					self.currentCodePage = newCodePage
+				else:
+					raise InvalidDataException("Unknown code page ID 0x{0:X} encountered in WBXML".format(currentByte))
+			elif  ( currentByte == GlobalTokens.END ):
+				if (currentNode != None and currentNode.parentNode != None):
+					currentNode = currentNode.parentNode
+				else:
+					raise InvalidDataException("END global token encountered out of sequence")
+					break
+			elif  ( currentByte == GlobalTokens.OPAQUE ):
+				CDATALength = wbXMLBytes.dequeueMultibyteInt()
+				newOpaqueNode = self.xmlDoc.createCDATASection(wbXMLBytes.dequeueString(CDATALength))
+				currentNode.appendChild(newOpaqueNode)
+
+			elif  ( currentByte == GlobalTokens.STR_I ):
+				newTextNode = self.xmlDoc.createTextNode(wbXMLBytes.dequeueString())
+				currentNode.appendChild(newTextNode)
+
+			elif ( currentByte in unusedArray):
+				raise InvalidDataException("Encountered unknown global token 0x{0:X}.".format(currentByte))
+			else:
+				hasAttributes = (currentByte & 0x80) > 0
+				hasContent = (currentByte & 0x40) > 0
+
+				token = currentByte & 0x3F
+				if (hasAttributes):
+					raise InvalidDataException("Token 0x{0:X} has attributes.".format(token))
+
+				strTag = self.codePages[self.currentCodePage].getTag(token)
+				if (strTag == None):
+					strTag = "UNKNOWN_TAG_{0,2:X}".format(token)
+
+				newNode = self.xmlDoc.createElement(strTag)
+				# not sure if this should be set on every node or not
+				#newNode.setAttribute("xmlns", self.codePages[self.currentCodePage].xmlns)
+				
+				currentNode.appendChild(newNode)
+
+				if (hasContent):
+					currentNode = newNode
+
+		logging.debug("Total bytes dequeued: %d" % wbXMLBytes.bytesDequeued)

libmproxy/contrib/wbxml/ASWBXMLByteQueue.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+'''
+@author: David Shaw, david.shaw.aw@gmail.com
+
+Inspired by EAS Inspector for Fiddler
+https://easinspectorforfiddler.codeplex.com
+
+----- The MIT License (MIT) ----- 
+Filename: ASWBXMLByteQueue.py
+Copyright (c) 2014, David P. Shaw
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+'''
+from Queue import Queue
+import logging
+
+class ASWBXMLByteQueue(Queue):
+
+    def __init__(self, wbxmlBytes):
+        
+        self.bytesDequeued = 0
+        self.bytesEnqueued = 0
+        
+        Queue.__init__(self)
+
+        for byte in wbxmlBytes:
+            self.put(ord(byte))
+            self.bytesEnqueued += 1
+        
+        
+        logging.debug("Array byte count: %d, enqueued: %d" % (self.qsize(), self.bytesEnqueued))
+    
+    """
+    Created to debug the dequeueing of bytes
+    """
+    def dequeueAndLog(self):
+        singleByte = self.get()
+        self.bytesDequeued += 1
+        logging.debug("Dequeued byte 0x{0:X} ({1} total)".format(singleByte, self.bytesDequeued))
+        return singleByte
+    
+    """
+    Return true if the continuation bit is set in the byte
+    """
+    def checkContinuationBit(self, byteval):
+        continuationBitmask = 0x80
+        return (continuationBitmask & byteval) != 0
+    
+    def dequeueMultibyteInt(self):
+        iReturn = 0
+        singleByte = 0xFF
+         
+        while True:
+            iReturn <<= 7
+            if (self.qsize() == 0):
+                break
+            else:
+                singleByte = self.dequeueAndLog()
+            iReturn += int(singleByte & 0x7F)
+            if not self.checkContinuationBit(singleByte):
+                return iReturn
+
+    def dequeueString(self, length=None):
+        if ( length != None):
+            currentByte = 0x00
+            strReturn = ""
+            for i in range(0, length):
+                # TODO: Improve this handling. We are technically UTF-8, meaning
+                # that characters could be more than one byte long. This will fail if we have
+                # characters outside of the US-ASCII range
+                if ( self.qsize() == 0 ):
+                    break
+                currentByte = self.dequeueAndLog()
+                strReturn += chr(currentByte)
+
+        else:
+            currentByte = 0x00
+            strReturn = ""
+            while True:
+                currentByte = self.dequeueAndLog()
+                if (currentByte != 0x00):
+                    strReturn += chr(currentByte)
+                else:
+                    break
+
+        return strReturn
+

libmproxy/contrib/wbxml/ASWBXMLCodePage.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+'''
+@author: David Shaw, david.shaw.aw@gmail.com
+
+Inspired by EAS Inspector for Fiddler
+https://easinspectorforfiddler.codeplex.com
+
+----- The MIT License (MIT) ----- 
+Filename: ASWBXMLCodePage.py
+Copyright (c) 2014, David P. Shaw
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+'''
+class ASWBXMLCodePage:
+	def __init__(self):
+		self.namespace = ""
+		self.xmlns = ""
+		self.tokenLookup = {}
+		self.tagLookup = {}
+	
+	def addToken(self, token, tag):
+		self.tokenLookup[token] = tag
+		self.tagLookup[tag] = token
+	
+	def getToken(self, tag):
+		if self.tagLookup.has_key(tag):
+			return self.tagLookup[tag]
+		return 0xFF
+	
+	def getTag(self, token):
+		if self.tokenLookup.has_key(token):
+			return self.tokenLookup[token]
+		return None
+	
+	def __repr__(self):
+		return str(self.tokenLookup)

libmproxy/contrib/wbxml/GlobalTokens.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+'''
+@author: David Shaw, david.shaw.aw@gmail.com
+
+Inspired by EAS Inspector for Fiddler
+https://easinspectorforfiddler.codeplex.com
+
+----- The MIT License (MIT) ----- 
+Filename: GlobalTokens.py
+Copyright (c) 2014, David P. Shaw
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+'''
+class GlobalTokens:
+    SWITCH_PAGE = 0x00
+    END = 0x01
+    ENTITY = 0x02
+    STR_I = 0x03
+    LITERAL = 0x04
+    EXT_I_0 = 0x40
+    EXT_I_1 = 0x41
+    EXT_I_2 = 0x42
+    PI = 0x43
+    LITERAL_C = 0x44
+    EXT_T_0 = 0x80
+    EXT_T_1 = 0x81
+    EXT_T_2 = 0x82
+    STR_T = 0x83
+    LITERAL_A = 0x84
+    EXT_0 = 0xC0
+    EXT_1 = 0xC1
+    EXT_2 = 0xC2
+    OPAQUE = 0xC3
+    LITERAL_AC = 0xC4

libmproxy/contrib/wbxml/InvalidDataException.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+'''
+@author: David Shaw, david.shaw.aw@gmail.com
+
+Inspired by EAS Inspector for Fiddler
+https://easinspectorforfiddler.codeplex.com
+
+----- The MIT License (MIT) ----- 
+Filename: InvalidDataException.py
+Copyright (c) 2014, David P. Shaw
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+'''
+class InvalidDataException(Exception):
+    pass

libmproxy/controller.py

@@ -1,8 +1,6 @@
+from __future__ import absolute_import
 import Queue, threading
 
-should_exit = False
-
-
 class DummyReply:
     """
         A reply object that does nothing. Useful when we need an object to seem
@@ -36,8 +34,9 @@ class Reply:
 
 
 class Channel:
-    def __init__(self, q):
+    def __init__(self, q, should_exit):
         self.q = q
+        self.should_exit = should_exit
 
     def ask(self, mtype, m):
         """
@@ -46,11 +45,11 @@ class Channel:
         """
         m.reply = Reply(m)
         self.q.put((mtype, m))
-        while not should_exit:
+        while not self.should_exit.is_set():
             try:
                 # The timeout is here so we can handle a should_exit event.
                 g = m.reply.q.get(timeout=0.5)
-            except Queue.Empty: # pragma: nocover
+            except Queue.Empty:  # pragma: nocover
                 continue
             return g
 
@@ -72,12 +71,13 @@ class Slave(threading.Thread):
         self.channel, self.server = channel, server
         self.server.set_channel(channel)
         threading.Thread.__init__(self)
+        self.name = "SlaveThread (%s:%s)" % (self.server.address.host, self.server.address.port)
 
     def run(self):
         self.server.serve_forever()
 
 
-class Master:
+class Master(object):
     """
         Masters get and respond to messages from slaves.
     """
@@ -87,8 +87,9 @@ class Master:
         """
         self.server = server
         self.masterq = Queue.Queue()
+        self.should_exit = threading.Event()
 
-    def tick(self, q):
+    def tick(self, q, timeout):
         changed = False
         try:
             # This endless loop runs until the 'Queue.Empty'
@@ -96,8 +97,7 @@ class Master:
             # the queue, this speeds up every request by 0.1 seconds,
             # because get_input(..) function is not blocking.
             while True:
-                # Small timeout to prevent pegging the CPU
-                msg = q.get(timeout=0.01)
+                msg = q.get(timeout=timeout)
                 self.handle(*msg)
                 changed = True
         except Queue.Empty:
@@ -105,11 +105,10 @@ class Master:
         return changed
 
     def run(self):
-        global should_exit
-        should_exit = False
-        self.server.start_slave(Slave, Channel(self.masterq))
-        while not should_exit:
-            self.tick(self.masterq)
+        self.should_exit.clear()
+        self.server.start_slave(Slave, Channel(self.masterq, self.should_exit))
+        while not self.should_exit.is_set():
+            self.tick(self.masterq, 0.01)
         self.shutdown()
 
     def handle(self, mtype, obj):
@@ -121,8 +120,7 @@ class Master:
             obj.reply()
 
     def shutdown(self):
-        global should_exit
-        if not should_exit:
-            should_exit = True
+        if not self.should_exit.is_set():
+            self.should_exit.set()
             if self.server:
                 self.server.shutdown()

libmproxy/dump.py

@@ -1,20 +1,25 @@
-import sys, os
+from __future__ import absolute_import
+import sys
+import os
 import netlib.utils
-import flow, filt, utils
+from . import flow, filt, utils
+from .protocol import http
 
-class DumpError(Exception): pass
+
+class DumpError(Exception):
+    pass
 
 
 class Options(object):
     attributes = [
         "app",
-        "app_external",
         "app_host",
         "app_port",
         "anticache",
         "anticomp",
         "client_replay",
-        "eventlog",
+        "filtstr",
+        "flow_detail",
         "keepserving",
         "kill",
         "no_server",
@@ -29,9 +34,14 @@ class Options(object):
         "showhost",
         "stickycookie",
         "stickyauth",
+        "stream_large_bodies",
         "verbosity",
-        "wfile",
+        "outfile",
+        "replay_ignore_content",
+        "replay_ignore_params",
+        "replay_ignore_payload_params",
     ]
+
     def __init__(self, **kwargs):
         for k, v in kwargs.items():
             setattr(self, k, v)
@@ -42,35 +52,39 @@ class Options(object):
 
 def str_response(resp):
     r = "%s %s"%(resp.code, resp.msg)
-    if resp.is_replay():
+    if resp.is_replay:
         r = "[replay] " + r
     return r
 
 
-def str_request(req, showhost):
-    if req.client_conn:
-        c = req.client_conn.address[0]
+def str_request(f, showhost):
+    if f.client_conn:
+        c = f.client_conn.address.host
     else:
         c = "[replay]"
-    r = "%s %s %s"%(c, req.method, req.get_url(showhost))
-    if req.stickycookie:
+    r = "%s %s %s"%(c, f.request.method, f.request.pretty_url(showhost))
+    if f.request.stickycookie:
         r = "[stickycookie] " + r
     return r
 
 
 class DumpMaster(flow.FlowMaster):
-    def __init__(self, server, options, filtstr, outfile=sys.stdout):
+    def __init__(self, server, options, outfile=sys.stdout):
         flow.FlowMaster.__init__(self, server, flow.State())
         self.outfile = outfile
         self.o = options
         self.anticache = options.anticache
         self.anticomp = options.anticomp
-        self.eventlog = options.eventlog
         self.showhost = options.showhost
+        self.replay_ignore_params = options.replay_ignore_params
+        self.replay_ignore_content = options.replay_ignore_content
         self.refresh_server_playback = options.refresh_server_playback
+        self.replay_ignore_payload_params = options.replay_ignore_payload_params
 
-        if filtstr:
-            self.filt = filt.parse(filtstr)
+        self.set_stream_large_bodies(options.stream_large_bodies)
+
+        if options.filtstr:
+            self.filt = filt.parse(options.filtstr)
         else:
             self.filt = None
 
@@ -80,10 +94,10 @@ class DumpMaster(flow.FlowMaster):
         if options.stickyauth:
             self.set_stickyauth(options.stickyauth)
 
-        if options.wfile:
-            path = os.path.expanduser(options.wfile)
+        if options.outfile:
+            path = os.path.expanduser(options.outfile[0])
             try:
-                f = file(path, "wb")
+                f = file(path, options.outfile[1])
                 self.start_stream(f, self.filt)
             except IOError, v:
                 raise DumpError(v.strerror)
@@ -101,7 +115,10 @@ class DumpMaster(flow.FlowMaster):
                 self._readflow(options.server_replay),
                 options.kill, options.rheaders,
                 not options.keepserving,
-                options.nopop
+                options.nopop,
+                options.replay_ignore_params,
+                options.replay_ignore_content,
+                options.replay_ignore_payload_params,
             )
 
         if options.client_replay:
@@ -126,10 +143,10 @@ class DumpMaster(flow.FlowMaster):
             try:
                 self.load_flows(freader)
             except flow.FlowReadError, v:
-                self.add_event("Flow file corrupted. Stopped loading.")
+                self.add_event("Flow file corrupted. Stopped loading.", "error")
 
         if self.o.app:
-            self.start_app(self.o.app_host, self.o.app_port, self.o.app_external)
+            self.start_app(self.o.app_host, self.o.app_port)
 
     def _readflow(self, path):
         path = os.path.expanduser(path)
@@ -141,7 +158,8 @@ class DumpMaster(flow.FlowMaster):
         return flows
 
     def add_event(self, e, level="info"):
-        if self.eventlog:
+        needed = dict(error=0, info=1, debug=2).get(level, 1)
+        if self.o.verbosity >= needed:
             print >> self.outfile, e
             self.outfile.flush()
 
@@ -155,13 +173,18 @@ class DumpMaster(flow.FlowMaster):
             return
 
         if f.response:
-            sz = utils.pretty_size(len(f.response.content))
-            if self.o.verbosity > 0:
+            if self.o.flow_detail > 0:
+                if f.response.content == http.CONTENT_MISSING:
+                    sz = "(content missing)"
+                else:
+                    sz = utils.pretty_size(len(f.response.content))
                 result = " << %s %s"%(str_response(f.response), sz)
-            if self.o.verbosity > 1:
+            if self.o.flow_detail > 1:
                 result = result + "\n\n" + self.indent(4, f.response.headers)
-            if self.o.verbosity > 2:
-                if utils.isBin(f.response.content):
+            if self.o.flow_detail > 2:
+                if f.response.content == http.CONTENT_MISSING:
+                    cont = self.indent(4, "(content missing)")
+                elif utils.isBin(f.response.content):
                     d = netlib.utils.hexdump(f.response.content)
                     d = "\n".join("%s\t%s %s"%i for i in d)
                     cont = self.indent(4, d)
@@ -173,47 +196,45 @@ class DumpMaster(flow.FlowMaster):
         elif f.error:
             result = " << %s"%f.error.msg
 
-        if self.o.verbosity == 1:
-            print >> self.outfile, str_request(f.request, self.showhost)
+        if self.o.flow_detail == 1:
+            print >> self.outfile, str_request(f, self.showhost)
             print >> self.outfile, result
-        elif self.o.verbosity == 2:
-            print >> self.outfile, str_request(f.request, self.showhost)
+        elif self.o.flow_detail == 2:
+            print >> self.outfile, str_request(f, self.showhost)
             print >> self.outfile, self.indent(4, f.request.headers)
             print >> self.outfile
             print >> self.outfile, result
             print >> self.outfile, "\n"
-        elif self.o.verbosity >= 3:
-            print >> self.outfile, str_request(f.request, self.showhost)
+        elif self.o.flow_detail >= 3:
+            print >> self.outfile, str_request(f, self.showhost)
             print >> self.outfile, self.indent(4, f.request.headers)
-            if utils.isBin(f.request.content):
-                print >> self.outfile, self.indent(4, netlib.utils.hexdump(f.request.content))
+            if f.request.content != http.CONTENT_MISSING and utils.isBin(f.request.content):
+                d = netlib.utils.hexdump(f.request.content)
+                d = "\n".join("%s\t%s %s"%i for i in d)
+                print >> self.outfile, self.indent(4, d)
             elif f.request.content:
                 print >> self.outfile, self.indent(4, f.request.content)
             print >> self.outfile
             print >> self.outfile, result
             print >> self.outfile, "\n"
-        if self.o.verbosity:
+        if self.o.flow_detail:
             self.outfile.flush()
 
-    def handle_log(self, l):
-        self.add_event(l.msg)
-        l.reply()
-
-    def handle_request(self, r):
-        f = flow.FlowMaster.handle_request(self, r)
+    def handle_request(self, f):
+        flow.FlowMaster.handle_request(self, f)
         if f:
-            r.reply()
+            f.reply()
         return f
 
-    def handle_response(self, msg):
-        f = flow.FlowMaster.handle_response(self, msg)
+    def handle_response(self, f):
+        flow.FlowMaster.handle_response(self, f)
         if f:
-            msg.reply()
+            f.reply()
             self._process_flow(f)
         return f
 
-    def handle_error(self, msg):
-        f = flow.FlowMaster.handle_error(self, msg)
+    def handle_error(self, f):
+        flow.FlowMaster.handle_error(self, f)
         if f:
             self._process_flow(f)
         return f
@@ -223,8 +244,7 @@ class DumpMaster(flow.FlowMaster):
 
     def run(self):  # pragma: no cover
         if self.o.rfile and not self.o.keepserving:
-            for script in self.scripts:
-                self.unload_script(script)
+            self.shutdown()
             return
         try:
             return flow.FlowMaster.run(self)

libmproxy/encoding.py

@@ -1,6 +1,7 @@
 """
     Utility functions for decoding response bodies.
 """
+from __future__ import absolute_import
 import cStringIO
 import gzip, zlib
 

libmproxy/filt.py

@@ -31,8 +31,10 @@
         ~c CODE     Response code.
         rex         Equivalent to ~u rex
 """
+from __future__ import absolute_import
 import re, sys
-import contrib.pyparsing as pp
+from .contrib import pyparsing as pp
+from .protocol.http import decoded
 
 
 class _Token:
@@ -164,10 +166,14 @@ class FBod(_Rex):
     code = "b"
     help = "Body"
     def __call__(self, f):
-        if f.request.content and re.search(self.expr, f.request.content):
-            return True
-        elif f.response and f.response.content and re.search(self.expr, f.response.content):
-            return True
+        if f.request and f.request.content:
+            with decoded(f.request):
+                if re.search(self.expr, f.request.content):
+                    return True
+        if f.response and f.response.content:
+            with decoded(f.response):
+                if re.search(self.expr, f.response.content):
+                    return True
         return False
 
 
@@ -175,16 +181,20 @@ class FBodRequest(_Rex):
     code = "bq"
     help = "Request body"
     def __call__(self, f):
-        if f.request.content and re.search(self.expr, f.request.content):
-            return True
+        if f.request and f.request.content:
+            with decoded(f.request):
+                if re.search(self.expr, f.request.content):
+                    return True
 
 
 class FBodResponse(_Rex):
     code = "bs"
     help = "Response body"
     def __call__(self, f):
-        if f.response and f.response.content and re.search(self.expr, f.response.content):
-            return True
+        if f.response and f.response.content:
+            with decoded(f.response):
+                if re.search(self.expr, f.response.content):
+                    return True
 
 
 class FMethod(_Rex):
@@ -212,7 +222,7 @@ class FUrl(_Rex):
         return klass(*toks)
 
     def __call__(self, f):
-        return re.search(self.expr, f.request.get_url())
+        return re.search(self.expr, f.request.url)
 
 
 class _Int(_Action):
@@ -333,9 +343,34 @@ bnf = _make()
 
 def parse(s):
     try:
-        return bnf.parseString(s, parseAll=True)[0]
+        filt = bnf.parseString(s, parseAll=True)[0]
+        filt.pattern = s
+        return filt
     except pp.ParseException:
         return None
     except ValueError:
         return None
 
+
+help = []
+for i in filt_unary:
+    help.append(
+        ("~%s"%i.code, i.help)
+    )
+for i in filt_rex:
+    help.append(
+        ("~%s regex"%i.code, i.help)
+    )
+for i in filt_int:
+    help.append(
+        ("~%s int"%i.code, i.help)
+    )
+help.sort()
+help.extend(
+    [
+        ("!", "unary not"),
+        ("&", "and"),
+        ("|", "or"),
+        ("(...)", "grouping"),
+    ]
+)

libmproxy/main.py

@@ -0,0 +1,161 @@
+from __future__ import print_function, absolute_import
+import os
+import signal
+import sys
+import netlib.version
+from . import version, cmdline
+from .proxy import process_proxy_options, ProxyServerError
+from .proxy.server import DummyServer, ProxyServer
+
+
+# This file is not included in coverage analysis or tests - anything that can be
+# tested should live elsewhere.
+
+def check_versions():
+    """
+    Having installed a wrong version of pyOpenSSL or netlib is unfortunately a
+    very common source of error. Check before every start that both versions are
+    somewhat okay.
+    """
+    # We don't introduce backward-incompatible changes in patch versions. Only
+    # consider major and minor version.
+    if netlib.version.IVERSION[:2] != version.IVERSION[:2]:
+        print(
+            "Warning: You are using mitmdump %s with netlib %s. "
+            "Most likely, that won't work - please upgrade!" % (
+                version.VERSION, netlib.version.VERSION
+            ),
+            file=sys.stderr
+        )
+    import OpenSSL
+    import inspect
+    v = tuple([int(x) for x in OpenSSL.__version__.split(".")][:2])
+    if v < (0, 14):
+        print(
+            "You are using an outdated version of pyOpenSSL:"
+            " mitmproxy requires pyOpenSSL 0.14 or greater.",
+            file=sys.stderr
+        )
+        # Some users apparently have multiple versions of pyOpenSSL installed.
+        # Report which one we got.
+        pyopenssl_path = os.path.dirname(inspect.getfile(OpenSSL))
+        print(
+            "Your pyOpenSSL %s installation is located at %s" % (
+                OpenSSL.__version__, pyopenssl_path
+            ),
+            file=sys.stderr
+        )
+        sys.exit(1)
+
+
+def assert_utf8_env():
+    spec = ""
+    for i in ["LANG", "LC_CTYPE", "LC_ALL"]:
+        spec += os.environ.get(i, "").lower()
+    if "utf" not in spec:
+        print(
+            "Error: mitmproxy requires a UTF console environment.",
+            file=sys.stderr
+        )
+        print(
+            "Set your LANG enviroment variable to something like en_US.UTF-8",
+            file=sys.stderr
+        )
+        sys.exit(1)
+
+
+def get_server(dummy_server, options):
+    if dummy_server:
+        return DummyServer(options)
+    else:
+        try:
+            return ProxyServer(options)
+        except ProxyServerError, v:
+            print(str(v), file=sys.stderr)
+            sys.exit(1)
+
+
+def mitmproxy():  # pragma: nocover
+    from . import console
+
+    check_versions()
+    assert_utf8_env()
+
+    parser = cmdline.mitmproxy()
+    options = parser.parse_args()
+    if options.quiet:
+        options.verbose = 0
+
+    proxy_config = process_proxy_options(parser, options)
+    console_options = console.Options(**cmdline.get_common_options(options))
+    console_options.palette = options.palette
+    console_options.eventlog = options.eventlog
+    console_options.intercept = options.intercept
+
+    server = get_server(console_options.no_server, proxy_config)
+
+    m = console.ConsoleMaster(server, console_options)
+    try:
+        m.run()
+    except KeyboardInterrupt:
+        pass
+
+
+def mitmdump():  # pragma: nocover
+    from . import dump
+
+    check_versions()
+
+    parser = cmdline.mitmdump()
+    options = parser.parse_args()
+    if options.quiet:
+        options.verbose = 0
+        options.flow_detail = 0
+
+    proxy_config = process_proxy_options(parser, options)
+    dump_options = dump.Options(**cmdline.get_common_options(options))
+    dump_options.flow_detail = options.flow_detail
+    dump_options.keepserving = options.keepserving
+    dump_options.filtstr = " ".join(options.args) if options.args else None
+
+    server = get_server(dump_options.no_server, proxy_config)
+
+    try:
+        master = dump.DumpMaster(server, dump_options)
+
+        def cleankill(*args, **kwargs):
+            master.shutdown()
+
+        signal.signal(signal.SIGTERM, cleankill)
+        master.run()
+    except dump.DumpError as e:
+        print("mitmdump: %s" % e, file=sys.stderr)
+        sys.exit(1)
+    except KeyboardInterrupt:
+        pass
+
+
+def mitmweb():  # pragma: nocover
+    from . import web
+
+    check_versions()
+    parser = cmdline.mitmweb()
+
+    options = parser.parse_args()
+    if options.quiet:
+        options.verbose = 0
+
+    proxy_config = process_proxy_options(parser, options)
+    web_options = web.Options(**cmdline.get_common_options(options))
+    web_options.intercept = options.intercept
+    web_options.wdebug = options.wdebug
+    web_options.wiface = options.wiface
+    web_options.wport = options.wport
+
+    server = get_server(web_options.no_server, proxy_config)
+
+    m = web.WebMaster(server, web_options)
+    try:
+        m.run()
+    except KeyboardInterrupt:
+        pass