always show error messages

Using uppercase C to 'clear' display mode, because lowercase 'c' is used for css

.gitattributes

@@ -0,0 +1 @@
+libmproxy/web/static/**/* -diff

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 MANIFEST
 /build
 /dist
@@ -9,9 +10,18 @@ MANIFEST
 *.py[cdo]
 *.swp
 *.swo
+mitmproxy.egg-info/
 mitmproxyc
 mitmdumpc
 .coverage
 .idea
 netlib
+pathod
 libpathod
+
+# UI
+
+node_modules
+bower_components
+*.compiled.js
+*.map

.travis.yml

@@ -2,11 +2,8 @@ language: python
 python:
   - "2.7"
 # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
-install: 
-  - "pip install --upgrade git+https://github.com/mitmproxy/netlib.git"
-  - "pip install --upgrade git+https://github.com/mitmproxy/pathod.git"
-  - "pip install -r requirements.txt --use-mirrors"
-  - "pip install -r test/requirements.txt --use-mirrors"
+install:
+  - "pip install --src . -r requirements.txt"
 # command to run tests, e.g. python setup.py test
 script: 
   - "nosetests --with-cov --cov-report term-missing"

CHANGELOG

@@ -1,6 +1,51 @@
-26 Februrary 2014: mitmproxy 0.10.1:
 
-    * Fix compatibility with pyOpenSSL 0.14
+15 November 2014: mitmproxy 0.11.1:
+
+    * Bug fixes: connection leaks some crashes 
+
+
+7 November 2014: mitmproxy 0.11:
+    
+    * Performance improvements for mitmproxy console
+
+    * SOCKS5 proxy mode allows mitmproxy to act as a SOCKS5 proxy server
+
+    * Data streaming for response bodies exceeding a threshold
+      (bradpeabody@gmail.com)
+
+    * Ignore hosts or IP addresses, forwarding both HTTP and HTTPS traffic
+      untouched
+
+    * Finer-grained control of traffic replay, including options to ignore
+      contents or parameters when matching flows (marcelo.glezer@gmail.com)
+
+    * Pass arguments to inline scripts 
+
+    * Configurable size limit on HTTP request and response bodies
+
+    * Per-domain specification of interception certificates and keys (see
+      --cert option)
+
+    * Certificate forwarding, relaying upstream SSL certificates verbatim (see
+      --cert-forward)
+
+    * Search and highlighting for HTTP request and response bodies in
+      mitmproxy console (pedro@worcel.com)
+
+    * Transparent proxy support on Windows
+
+    * Improved error messages and logging
+
+    * Support for FreeBSD in transparent mode, using pf (zbrdge@gmail.com)
+
+    * Content view mode for WBXML (davidshaw835@air-watch.com)
+
+    * Better documentation, with a new section on proxy modes
+
+    * Generic TCP proxy mode
+
+    * Countless bugfixes and other small improvements
+
 
 
 28 January 2014: mitmproxy 0.10:

CONTRIBUTORS

@@ -1,51 +1,65 @@
-   854	Aldo Cortesi
-    64	Maximilian Hils
+   902	Aldo Cortesi
+   323	Maximilian Hils
     18	Henrik Nordstrom
     13	Thomas Roth
+    12	Pedro Worcel
     11	Stephen Altamirano
     10	András Veres-Szentkirályi
-     8	Jason A. Novak
      8	Rouli
+     8	Jason A. Novak
      7	Alexis Hildebrandt
-     6	Pedro Worcel
      5	Tomaz Muraus
+     5	Brad Peabody
      5	Matthias Urlichs
      4	root
-     4	Bryan Bishop
      4	Marc Liyanage
      4	Valtteri Virtanen
-     3	Kyle Manna
+     4	Bryan Bishop
      3	Chris Neasbitt
-     2	alts
-     2	Heikki Hannikainen
-     2	Jim Lloyd
+     3	Zack B
+     3	Eli Shvartsman
+     3	Kyle Manna
      2	Michael Frister
+     2	Bennett Blodinger
+     2	Jim Lloyd
      2	Rob Wills
-     2	Jaime Soriano Pastor
      2	israel
+     2	Jaime Soriano Pastor
+     2	Heikki Hannikainen
      2	Mark E. Haase
+     2	alts
+     1	davidpshaw
+     1	deployable
+     1	joebowbeer
+     1	meeee
+     1	phil plante
+     1	Michael Bisbjerg
+     1	Andy Smith
+     1	Dan Wilbraham
+     1	David Shaw
+     1	Eric Entzel
+     1	Felix Wolfsteller
+     1	Henrik Nordström
+     1	Ivaylo Popov
+     1	JC
+     1	Jakub Nawalaniec
+     1	James Billingham
+     1	Jean Regisser
+     1	Kit Randel
+     1	Marcelo Glezer
+     1	Mathieu Mitchell
+     1	Mikhail Korobov
+     1	Nicolas Esteves
+     1	Oleksandr Sheremet
      1	Paul
      1	Rich Somerfield
      1	Rory McCann
-     1	Felix Wolfsteller
      1	Rune Halvorsen
      1	Sahn Lam
-     1	Eric Entzel
-     1	Dan Wilbraham
+     1	Seppo Yli-Olli
+     1	Sergey Chipiga
+     1	Steven Van Acker
      1	Ulrich Petri
-     1	Andy Smith
+     1	Vyacheslav Bakhmutov
      1	Yuangxuan Wang
      1	capt8bit
-     1	joebowbeer
-     1	meeee
-     1	James Billingham
-     1	Jakub Nawalaniec
-     1	JC
-     1	Kit Randel
-     1	phil plante
-     1	Mathieu Mitchell
-     1	Ivaylo Popov
-     1	Henrik Nordström
-     1	Michael Bisbjerg
-     1	Nicolas Esteves
-     1	Oleksandr Sheremet

MANIFEST.in

@@ -1,11 +1,8 @@
-include LICENSE
-include CHANGELOG
-include CONTRIBUTORS
-include README.txt
-include setup.py
+include mitmproxy mitmdump
+include LICENSE CHANGELOG CONTRIBUTORS README.txt
 exclude README.mkd
 recursive-include examples *
 recursive-include doc *
 recursive-include test *
-recursive-include libmproxy/resources *
-recursive-exclude test *.swo *.swp *.pyc
+recursive-include libmproxy *
+recursive-exclude * *.pyc *.pyo *.swo *.swp

README.mkd

@@ -13,6 +13,9 @@ mitmproxy.org website:
 [mitmproxy.org](http://mitmproxy.org).
 
 
+You can find complete directions for installing mitmproxy [here](http://mitmproxy.org/doc/install.html). 
+
+
 Features
 --------
 
@@ -26,9 +29,17 @@ Features
 - SSL certificates for interception are generated on the fly.
 - And much, much more.
 
+__mitmproxy__ is tested and developed on OSX, Linux and OpenBSD. On Windows,
+only mitmdump is supported, which does not have a graphical user interface.
+
+
+
+Hacking
+-------
+
+
+### Requirements
 
-Requirements
-------------
 
 * [Python](http://www.python.org) 2.7.x.
 * [netlib](http://pypi.python.org/pypi/netlib), version matching mitmproxy.
@@ -40,22 +51,42 @@ Optional packages for extended content decoding:
 * [protobuf](https://code.google.com/p/protobuf/) version 2.5.0 or newer.
 * [cssutils](http://cthedot.de/cssutils/) version 1.0 or newer.
 
-__mitmproxy__ is tested and developed on OSX, Linux and OpenBSD. Windows is not
-officially supported at the moment.
+For convenience, all optional dependencies can be installed with 
+
+`pip install "mitmproxy[contentviews]"`
+
+### Setting up a dev environment
+
+The following procedure is recommended to set up your dev environment:
+
+```
+$ git clone https://github.com/mitmproxy/mitmproxy.git
+$ cd mitmproxy
+$ pip install --src . -r requirements.txt
+```
+
+This installs the latest GitHub versions of mitmproxy, netlib and pathod into `mitmproxy/`. All other development dependencies save countershape are installed into their usual locations. 
 
 
-Hacking
--------
+### Testing
 
-The following components are needed if you plan to hack on mitmproxy:
+The test suite requires the `dev` extra requirements listed in [setup.py](https://github.com/mitmproxy/mitmproxy/blob/master/setup.py) and [pathod](http://pathod.net), version matching mitmproxy. Install these with:
 
-* The test suite uses the [nose](http://readthedocs.org/docs/nose/en/latest/) unit testing
-  framework and requires [pathod](http://pathod.org) and [flask](http://flask.pocoo.org/).
-* Rendering the documentation requires [countershape](http://github.com/cortesi/countershape).
+`
+pip install "mitmproxy[dev]""`
 
-For convenience, all dependencies save countershape can be installed from pypi
-to a virtualenv with 'pip install -r requirements.txt'.
 
 Please ensure that all patches are accompanied by matching changes in the test
 suite. The project maintains 100% test coverage.
 
+
+### Docs
+
+Rendering the documentation requires [countershape](http://github.com/cortesi/countershape). After installation, you can render the documentation to the doc like this:
+
+`cshape doc-src doc`
+
+
+
+
+

doc-src/_nav.html

@@ -2,6 +2,7 @@
     $!nav(idxpath, this, state)!$
     $!nav("install.html", this, state)!$
     $!nav("howmitmproxy.html", this, state)!$
+    $!nav("modes.html", this, state)!$
 
     <li class="nav-header">Tools</li>
         $!nav("mitmproxy.html", this, state)!$
@@ -9,17 +10,23 @@
 
     <li class="nav-header">Features</li>
         $!nav("anticache.html", this, state)!$
-        $!nav("clientreplay.html", this, state)!$
+
         $!nav("filters.html", this, state)!$
-        $!nav("forwardproxy.html", this, state)!$
-        $!nav("proxyauth.html", this, state)!$
         $!nav("replacements.html", this, state)!$
+        $!nav("clientreplay.html", this, state)!$
         $!nav("serverreplay.html", this, state)!$
         $!nav("setheaders.html", this, state)!$
-        $!nav("sticky.html", this, state)!$
+        $!nav("passthrough.html", this, state)!$
+        $!nav("proxyauth.html", this, state)!$
         $!nav("reverseproxy.html", this, state)!$
+        $!nav("responsestreaming.html", this, state)!$
+        $!nav("socksproxy.html", this, state)!$
+        $!nav("sticky.html", this, state)!$
+        $!nav("tcpproxy.html", this, state)!$
+        $!nav("upstreamproxy.html", this, state)!$
         $!nav("upstreamcerts.html", this, state)!$
 
+
     <li class="nav-header">Installing Certificates</li>
         $!nav("ssl.html", this, state)!$
         $!nav("certinstall/webapp.html", this, state)!$
@@ -46,5 +53,6 @@
         $!nav("tutorials/transparent-dhcp.html", this, state)!$
 
     <li class="nav-header">Hacking</li>
+        $!nav("dev/architecture.html", this, state)!$
         $!nav("dev/testing.html", this, state)!$
 </ul>

doc-src/certinstall/firefox.html

@@ -7,7 +7,7 @@ app](@!urlTo("webapp.html")!@). If this fails, do the following:
 <ol class="tlist">
     <li> If needed, copy the ~/.mitmproxy/mitmproxy-ca-cert.pem file to the target. </li>
 
-    <li>Open preferences, click on "Advanced", then select"Encryption":
+    <li>Open preferences, click on "Advanced", then select"Certificates":
         <img src="@!urlTo('firefox3.jpg')!@"/>
     </li>
 
@@ -21,7 +21,7 @@ app](@!urlTo("webapp.html")!@). If this fails, do the following:
 ## Installing the certificate
 
 <ol class="tlist">
-    <li>Tick "Trust this CS to identify web sites", and click "Ok":
+    <li>Tick "Trust this CA to identify web sites", and click "Ok":
         <img src="@!urlTo('firefox3-trust.jpg')!@"/>
     </li>
 

doc-src/certinstall/index.py

@@ -9,4 +9,5 @@ pages = [
     Page("ios-simulator.html", "IOS Simulator"),
     Page("android.html", "Android"),
     Page("java.html", "Java"),
+    Page("mitm.it-error.html", "Error: No proxy configured"),
 ]

doc-src/certinstall/mitm.it-error.html

@@ -0,0 +1,5 @@
+**Looks like you wanted to install the mitmproxy CA using the web app?**
+
+Unfortunately, there's been no mitmproxy instance on the wire that could have intercepted your request.
+Please configure your client to use mitmproxy and try again.<br>
+The request to <a href="http://mitm.it/">http://mitm.it/</a> must go through your mitmproxy instance.

doc-src/certinstall/webapp.html

@@ -8,3 +8,6 @@ domain **mitm.it**. You should see something like this:
 
 Just click on the relevant icon, and then follow the setup instructions
 for the platform you're on.
+
+Make sure you aren't using a bandwith optimizer (like Google's Data Compression
+Proxy on Chrome for Android) or the page will not load.

doc-src/custom-routing.txt

@@ -0,0 +1,24 @@
+# Adapted from http://tldp.org/HOWTO/TransparentProxy-6.html (6.2 Second method)
+# Note that the choice of firewall mark (3) and routing table (2) was fairly arbitrary.
+# If you are already using policy routing or firewall marking for some other purpose,
+# make sure you choose unique numbers here. Otherwise, don't worry about it.
+
+
+
+# On the router, run
+
+PROXY_IP=192.168.1.100
+TARGET_IP=192.168.1.110
+
+iptables -t mangle -A PREROUTING -j ACCEPT -p tcp -m multiport --dports 80,443 -s ! $TARGET_IP
+# Alternative to MITM the whole network:
+# iptables -t mangle -A PREROUTING -j ACCEPT -p tcp -m multiport --dports 80,443 -s  $PROXY_IP
+iptables -t mangle -A PREROUTING -j MARK --set-mark 3 -p tcp -m multiport --dports 80,443
+ip rule add fwmark 3 table 2
+ip route add default via $PROXY_IP dev br0 table 2
+
+
+
+# On the proxy machine, run
+
+iptables -A PREROUTING -t nat -i eth0 -p tcp -m multiport --dports 80,443 -j REDIRECT --to-port 8080

doc-src/dev/architecture.html

@@ -0,0 +1,8 @@
+To give you a better understanding of how mitmproxy works, mitmproxy's high-level architecture is detailed
+in the following graphic:
+
+<img src="@!urlTo('schematics/architecture.png')!@">
+
+<a href="@!urlTo('schematics/architecture.pdf')!@">(architecture.pdf)</a>
+<p>Please don't refrain from asking any further
+questions on the mailing list, the IRC channel or the GitHub issue tracker.</p>

doc-src/dev/index.py

@@ -2,5 +2,6 @@ from countershape import Page
 
 pages = [
      Page("testing.html", "Testing"),
+     Page("architecture.html", "Architecture"),
 #    Page("addingviews.html", "Writing Content Views"),
 ]

doc-src/features/forwardproxy.html

@@ -1,16 +0,0 @@
-
-In this mode, mitmproxy accepts proxy requests and unconditionally forwards all
-requests to a specified upstream server. This is in contrast to <a
-href="@!urlTo("reverseproxy.html")!@">reverse proxy mode</a>, in which
-mitmproxy forwards ordinary HTTP requests to an upstream server.
-
-<table class="table">
-    <tbody>
-        <tr>
-            <th width="20%">command-line</th> <td>-F http[s]://hostname[:port]</td>
-        </tr>
-        <tr>
-            <th>mitmproxy shortcut</th> <td><b>F</b></td>
-        </tr>
-    </tbody>
-</table>

doc-src/features/index.py

@@ -4,12 +4,16 @@ pages = [
     Page("anticache.html", "Anticache"),
     Page("clientreplay.html", "Client-side replay"),
     Page("filters.html", "Filter expressions"),
-    Page("forwardproxy.html", "Forward proxy mode"),
+    Page("passthrough.html", "Ignore Domains"),
+    Page("proxyauth.html", "Proxy Authentication"),
+    Page("replacements.html", "Replacements"),
+    Page("responsestreaming.html", "Response Streaming"),
+    Page("reverseproxy.html", "Reverse proxy mode"),
+    Page("socksproxy.html", "SOCKS Mode"),
     Page("setheaders.html", "Set Headers"),
     Page("serverreplay.html", "Server-side replay"),
     Page("sticky.html", "Sticky cookies and auth"),
-    Page("proxyauth.html", "Proxy Authentication"),
-    Page("replacements.html", "Replacements"),
-    Page("reverseproxy.html", "Reverse proxy mode"),
+    Page("tcpproxy.html", "TCP Proxy"),
     Page("upstreamcerts.html", "Upstream Certs"),
-]
+    Page("upstreamproxy.html", "Upstream proxy mode"),
+]

doc-src/features/passthrough.html

@@ -0,0 +1,81 @@
+There are two main reasons why you may want to exempt some traffic from mitmproxy's interception mechanism:
+
+- **Certificate pinning:** Some traffic is is protected using
+  [certificate pinning](https://security.stackexchange.com/questions/29988/what-is-certificate-pinning) and mitmproxy's
+  interception leads to errors. For example, Windows Update or the Apple App Store fail to work if mitmproxy is active.
+- **Convenience:** You really don't care about some parts of the traffic and just want them to go away.
+
+If you want to peek into (SSL-protected) non-HTTP connections, check out the [tcp proxy](@!urlTo("tcpproxy.html")!@) feature.
+If you want to ignore traffic from mitmproxy's processing because of large response bodies, take a look at the
+[response streaming](@!urlTo("responsestreaming.html")!@) feature.
+
+## How it works
+
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>--ignore regex</td>
+        </tr>
+        <tr>
+            <th>mitmproxy shortcut</th> <td><b>I</b></td>
+        </tr>
+    </tbody>
+</table>
+
+
+mitmproxy allows you to specify a regex which is matched against a <code>host:port</code> string (e.g. "example.com:443")
+to determine hosts that should be excluded. 
+
+There are two important quirks to consider:
+
+- **In transparent mode, the ignore pattern is matched against the IP.** While we usually infer the hostname from the 
+  Host header if the --host argument is passed to mitmproxy, we do not have access to this information before the SSL
+  handshake.
+- In regular mode, explicit HTTP requests are never ignored.[^explicithttp] The ignore pattern is applied on CONNECT
+  requests, which initiate HTTPS or clear-text WebSocket connections.
+
+
+### Tutorial
+
+If you just want to ignore one specific domain, there's usually a bulletproof method to do so:
+
+1. Run mitmproxy or mitmdump in verbose mode (-v) and observe the host:port information in the serverconnect 
+messages. mitmproxy will filter on these.
+2. Take the host:port string, surround it with ^ and $, escape all dots (. becomes \\.) 
+and use this as your ignore pattern:
+
+<pre class="terminal">
+$ mitmdump -v
+127.0.0.1:50588: clientconnect
+127.0.0.1:50588: request
+  -> CONNECT example.com:443 HTTP/1.1
+127.0.0.1:50588: Set new server address: example.com:443
+<span style="color: white">127.0.0.1:50588: serverconnect
+  -> example.com:443</span>
+^C
+$ <span style="color: white">mitmproxy --ignore ^example\.com:443$</span>    
+</pre>
+
+Here are some other examples for ignore patterns:
+<pre>
+# Exempt traffic from the iOS App Store (usually just works):
+--ignore apple.com:443  
+# "Correct" version without false-positives:
+--ignore ^(.+\.)?apple\.com:443$ 
+    
+# Ignore example.com on all ports, but no subdomains:
+--ignore ^example.com:
+    
+# Transparent mode:
+--ignore 17\.178\.96\.59:443
+# IP address range:
+--ignore 17\.178\.\d+\.\d+:443
+</pre>
+
+### See Also
+
+- [TCP Proxy](@!urlTo("tcpproxy.html")!@)
+- [Response Streaming](@!urlTo("responsestreaming.html")!@)
+
+[^explicithttp]: This stems from an limitation of explicit HTTP proxying: A single connection can be re-used for multiple target domains - a <code>GET http://example.com/</code> request may be followed by a <code>GET http://evil.com/</code> request on the same connection. If we start to ignore the connection after the first request, we would miss the relevant second one.

doc-src/features/responsestreaming.html

@@ -0,0 +1,54 @@
+By using mitmproxy's streaming feature, response contents can be passed to the client incrementally before they have been fully received by the proxy.
+This is especially useful for large binary files such as videos, where buffering the whole file slows down the client's browser.
+
+By default, mitmproxy will read the entire response, perform any indicated
+manipulations on it and then send the (possibly modified) response to
+the client. In some cases this is undesirable and you may wish to "stream"
+the reponse back to the client. When streaming is enabled, the response is
+not buffered on the proxy but directly sent back to the client instead.
+
+<h2>On the command-line</h2>
+
+Streaming can be enabled on the command line for all response bodies exceeding a certain size.  The SIZE argument understands
+k/m/g suffixes, e.g. 3m for 3 megabytes.
+
+<table class="table">
+    <tbody>
+    <tr>
+        <th width="20%">command-line</th>
+        <td>
+            --stream SIZE
+        </td>
+    </tr>
+    </tbody>
+</table>
+
+
+<h2>Caveats</h2>
+
+When response streaming is enabled, <strong>streamed response contents will not be
+    recorded or preserved in any way.</strong>
+
+When response streaming is enabled, the response body cannot be modified.
+
+<h2>Customizing Response Streaming</h2>
+
+You can also use an <a href="@!urlTo("scripting/inlinescripts.html")!@">inline script</a> to customize exactly
+which responses are streamed.
+
+Responses that should be tagged for streaming by setting their respective .stream attribute to True:
+
+$!example("examples/stream.py")!$
+
+
+<h2>Implementation Details</h2>
+
+When response streaming is enabled, portions of the code which would have otherwise performed changes
+on the response body will see an empty response body instead (<code>libmproxy.protocol.http.CONTENT_MISSING</code>). Any modifications will be ignored.
+
+Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a <code>Transfer-Encoding:
+    chunked</code> header, the response will be streamed one chunk at a time.
+
+### See Also
+
+- [Ignore Domains](@!urlTo("passthrough.html")!@)

doc-src/features/reverseproxy.html

@@ -1,19 +1,28 @@
 
 In reverse proxy mode, mitmproxy accepts standard HTTP requests and forwards
-them to the specified upstream server. This is in contrast to <a
-href="@!urlTo("forwardproxy.html")!@">forward proxy mode</a>, in which
-mitmproxy forwards HTTP proxy requests to an upstream server.
-
-Note that the displayed URL for flows in this mode will use the value of the
-__Host__ header field from the request, not the reverse proxy server. 
+them to the specified upstream server. This is in contrast to
+<a href="@!urlTo("upstreamproxy.html")!@">upstream proxy mode</a>, in which
+mitmproxy forwards HTTP proxy requests to an upstream proxy server.
 
 <table class="table">
     <tbody>
         <tr>
-            <th width="20%">command-line</th> <td>-P http[s]://hostname[:port]</td>
-        </tr>
-        <tr>
-            <th>mitmproxy shortcut</th> <td><b>P</b></td>
+            <th width="20%">command-line</th> <td>-R <i>schema</i>://hostname[:port]</td>
         </tr>
     </tbody>
 </table>
+
+Here, **schema** is one of http, https, http2https or https2http. The latter
+two extended schema specifications control the use of HTTP and HTTPS on
+mitmproxy and the upstream server. You can indicate that mitmproxy should use
+HTTP, and the upstream server uses HTTPS like this:
+
+	http2https://hostname:port
+
+And you can indicate that mitmproxy should use HTTPS while the upstream
+service uses HTTP like this:
+
+	https2http://hostname:port
+
+
+

doc-src/features/socksproxy.html

@@ -0,0 +1,10 @@
+
+In this mode, mitmproxy acts as a SOCKS5 proxy server.
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>--socks</td>
+        </tr>
+    </tbody>
+</table>

doc-src/features/tcpproxy.html

@@ -0,0 +1,30 @@
+WebSockets or other non-HTTP protocols are not supported by mitmproxy yet. However, you can exempt hostnames from
+processing, so that mitmproxy acts as a generic TCP forwarder. This feature is closely related to the
+[ignore domains](@!urlTo("passthrough.html")!@) functionality, but differs in two important aspects:
+
+- The raw TCP messages are printed to the event log.
+- SSL connections will be intercepted.
+
+Please note that message interception or modification are not possible yet.
+If you are not interested in the raw TCP messages, you should use the ignore domains feature.
+
+## How it works
+
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>--tcp HOST</td>
+        </tr>
+        <tr>
+            <th>mitmproxy shortcut</th> <td><b>T</b></td>
+        </tr>
+    </tbody>
+</table>
+
+For a detailed description on the structure of the hostname pattern, please refer to the [Ignore Domains](@!urlTo("passthrough.html")!@) feature.
+
+### See Also
+
+- [Ignore Domains](@!urlTo("passthrough.html")!@)
+- [Response Streaming](@!urlTo("responsestreaming.html")!@)

doc-src/features/upstreamproxy.html

@@ -0,0 +1,27 @@
+
+In this mode, mitmproxy accepts proxy requests and unconditionally forwards all
+requests to a specified upstream proxy server. This is in contrast to <a
+href="@!urlTo("reverseproxy.html")!@">reverse proxy mode</a>, in which
+mitmproxy forwards ordinary HTTP requests to an upstream server.
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>-U http://hostname[:port]</td>
+        </tr>
+    </tbody>
+</table>
+
+Here, **schema** is one of http, https, http2https or https2http. The latter
+two extended schema specifications control the use of HTTP and HTTPS on
+mitmproxy and the upstream server. You can indicate that mitmproxy should use
+HTTP, and the upstream server uses HTTPS like this:
+
+	http2https://hostname:port
+
+And you can indicate that mitmproxy should use HTTPS while the upstream
+service uses HTTP like this:
+
+	https2http://hostname:port
+
+

doc-src/index.html

@@ -1,4 +1,27 @@
 
-@!index_contents!@
+__mitmproxy__ is an interactive, SSL-capable man-in-the-middle proxy for HTTP
+with a console interface.
+
+__mitmdump__ is the command-line version of mitmproxy. Think tcpdump for HTTP.
+
+__libmproxy__ is the library that mitmproxy and mitmdump are built on.
+
+Documentation, tutorials and distribution packages can be found on the
+mitmproxy.org website:
+
+[mitmproxy.org](http://mitmproxy.org).
 
 
+Features
+--------
+
+- Intercept HTTP requests and responses and modify them on the fly.
+- Save complete HTTP conversations for later replay and analysis.
+- Replay the client-side of an HTTP conversations.
+- Replay HTTP responses of a previously recorded server.
+- Reverse proxy mode to forward traffic to a specified server.
+- Transparent proxy mode on OSX and Linux.
+- Make scripted changes to HTTP traffic using Python.
+- SSL certificates for interception are generated on the fly.
+- And much, much more.
+

doc-src/index.py

@@ -1,6 +1,8 @@
-import os, sys, datetime
+import os
+import sys
+import datetime
 import countershape
-from countershape import Page, Directory, PythonModule, markup, model
+from countershape import Page, Directory, markup, model
 import countershape.template
 sys.path.insert(0, "..")
 from libmproxy import filt, version
@@ -23,18 +25,18 @@ ns.docMaintainer = "Aldo Cortesi"
 ns.docMaintainerEmail = "aldo@corte.si"
 ns.copyright = u"\u00a9 mitmproxy project, %s" % datetime.date.today().year
 
+
 def mpath(p):
     p = os.path.join(MITMPROXY_SRC, p)
     return os.path.expanduser(p)
 
-with open(mpath("README.mkd")) as f:
-        readme = f.read()
-        ns.index_contents = readme.split("\n", 1)[1] #remove first line (contains build status)
 
 def example(s):
     d = file(mpath(s)).read().rstrip()
     extemp = """<div class="example">%s<div class="example_legend">(%s)</div></div>"""
     return extemp%(countershape.template.Syntax("py")(d), s)
+
+
 ns.example = example
 
 
@@ -73,12 +75,14 @@ def nav(page, current, state):
 ns.nav = nav
 ns.navbar = countershape.template.File(None, "_nav.html")
 
+
 pages = [
     Page("index.html", "Introduction"),
     Page("install.html", "Installation"),
     Page("mitmproxy.html", "mitmproxy"),
     Page("mitmdump.html", "mitmdump"),
     Page("howmitmproxy.html", "How mitmproxy works"),
+    Page("modes.html", "Modes of Operation"),
 
     Page("ssl.html", "Overview"),
     Directory("certinstall"),

doc-src/install.html

@@ -1,40 +1,33 @@
 
+## Installing from source
+
 The preferred way to install mitmproxy - whether you're installing the latest
 release or from source - is to use [pip](http://www.pip-installer.org/). If you
 don't already have pip on your system, you can find installation instructions
 [here](http://www.pip-installer.org/en/latest/installing.html).
 
-
-## Installing the latest release
-
-A single command will download and install the latest release of mitmproxy,
-along with all its dependencies:
-
 <pre class="terminal">
 pip install mitmproxy
 </pre>
 
+If you also want to install the optional packages AMF, protobuf and CSS
+content views, do this:
 
-## Installing from source
-
-When installing from source, the easiest method is still to use pip. In this
-case run:
-    
 <pre class="terminal">
-pip install /path/to/source
+pip install "mitmproxy[contentviews]"
 </pre>
 
-Note that if you're installing current git master, you will also have to
-install the current git master of [netlib](http://github.com/mitmproxy/netlib) by
-hand.
 
 ## OSX
 
+The easiest way to get up and running on OSX is to download the pre-built
+binary packages from [mitmproxy.org](http://mitmproxy.org). If you still want
+to install using pip, there are a few things to keep in mind:
+
 - If you're running a Python interpreter installed with homebrew (or similar),
 you may have to install some dependencies by hand. 
 - Make sure that XCode is installed from the App Store, and that the
 command-line tools have been downloaded (XCode/Preferences/Downloads).
-- Now use __pip__ to do the installation, as above.
 
 There are a few bits of customization you might want to do to make mitmproxy
 comfortable to use on OSX. The default color scheme is optimized for a dark
@@ -50,3 +43,17 @@ image/*; /usr/bin/open -Wn %s
 video/*; /usr/bin/open -Wn %s
 </pre>
 
+
+## Ubuntu
+
+On Ubuntu, you will need the following native packages to install mitmproxy
+from source:
+
+- build-essential
+- python-dev
+- libffi-dev
+- libssl-dev
+- libxml2-dev
+- libxslt1-dev
+
+

doc-src/modes.html

@@ -0,0 +1,222 @@
+
+Mitmproxy has four modes of operation that allow you to use mitmproxy in a
+variety of scenarios:
+
+- **Regular** (the default)
+- **Transparent**
+- **Reverse Proxy**
+- **Upstream Proxy**
+
+Now, which one should you pick? Use this flow chart:
+
+<img src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"/>
+
+<div class="page-header">
+    <h1>Regular Proxy</h1>
+</div>
+
+Mitmproxy's regular mode is the simplest and the easiest to set up.
+
+1. Start mitmproxy.
+2. Configure your client to use mitmproxy. For instance on IOS, the settings might look like <a href="@!urlTo('screenshots/ios-manual.png')!@">this</a>.
+3. Quick Check: You should already be able to visit an unencrypted HTTP site
+through the proxy.
+4. Open the magic domain <strong>mitm.it</strong> and install the certificate for your device.
+
+<div class="well">
+<strong>Heads Up:</strong> Unfortunately, some applications bypass the
+system HTTP proxy settings - Android applications are a common example. In
+these cases, you need to use mitmproxy's transparent mode.
+</div>
+
+If you are proxying an external device, your network will probably look like this:
+
+<img src="@!urlTo('schematics/proxy-modes-regular.png')!@">
+
+The square brackets signify the source and destination IP addresses. Your
+client explicitly connects to mitmproxy and mitmproxy explicitly connects
+to the target server.
+
+<div class="page-header">
+    <h1>Transparent Proxy</h1>
+</div>
+
+In transparent mode, traffic is directed into a proxy at the network layer,
+without any client configuration required. This makes transparent proxying
+ideal for situations where you can't change client behaviour. In the graphic
+below, a machine running mitmproxy has been inserted between the router and
+the internet:
+
+<a href="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
+</a>
+
+The square brackets signify the source and destination IP addresses. Round
+brackets mark the next hop on the *Ethernet/data link* layer. This distinction
+is important: when the packet arrives at the mitmproxy machine, it must still
+be addressed to the target server. This means that Network Address Translation
+should not be applied before the traffic reaches mitmproxy, since this would
+remove the target information, leaving mitmproxy unable to determine the real
+destination.
+
+<a href="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@"></a>
+
+<h2>Common Configurations</h2>
+
+There are many ways to configure your network for transparent proxying. We'll
+look at three common scenarios:
+
+1. Configuring the client to use a custom gateway/router/"next hop"
+2. Implementing custom routing on the router
+
+In most cases, the first option is recommended due to its ease of use.
+
+<h3>(a) Custom Gateway</h3>
+
+One simple way to get traffic to the mitmproxy machine with the destination IP
+intact, is to simply configure the client with the mitmproxy box as the
+default gateway.
+
+<a href="@!urlTo('schematics/proxy-modes-transparent-2.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-transparent-2.png')!@"></a>
+
+In this scenario, we would:
+
+- Configure the proxy machine for transparent mode. You can find instructions
+in the <em>Transparent Proxying</em> section of the mitmproxy docs.
+
+- Configure the client to use the proxy machine's IP as the default gateway.
+<a href="@!urlTo('screenshots/ios-gateway.png')!@">Here</a> is what this would
+look like on IOS.
+
+- Quick Check: At this point, you should already be able to visit an
+unencrypted HTTP site over the proxy.
+
+- Open the magic domain <strong>mitm.it</strong> and install the certificate
+for your device.
+
+Setting the custom gateway on clients can be automated by serving the settings
+out to clients over DHCP. This lets set up an interception network where all
+clients are proxied automatically, which can save time and effort. 
+
+
+<div class="well">
+    <strong style="text-align: center; display: block">Troubleshooting Transparent Mode</strong>
+
+    <p>Incorrect transparent mode configurations are a frequent source of
+    error. If it doesn't work for you, try the following things:</p>
+
+    <ul>
+        <li>
+            Open mitmproxy's event log (press `e`) - do you see clientconnect
+            messages? If not, the packets are not arriving at the proxy. One common
+            cause is the occurrence of ICMP redirects, which means that your
+            machine is telling the client that there's a faster way to the
+            internet by contacting your router directly (see the
+            <em>Transparent Proxying</em> section on how to disable them). If in
+            doubt, <a href="https://wireshark.org/">Wireshark</a> may help you
+            to see whether something arrives at your machine or not.
+        </li>
+        <li>
+            Make sure you have not explicitly configured an HTTP proxy on the
+            client. This is not needed in transparent mode.
+        </li>
+        <li>
+            Re-check the instructions in the <em>Transparent Proxying</em> section. Anything you missed?
+        </li>
+    </ul>
+
+    If you encounter any other pitfalls that should be listed here, please let us know!
+
+</div>
+
+<h3>(b) Custom Routing</h3>
+
+In some cases, you may need more fine-grained control of which traffic reaches
+the mitmproxy instance, and which doesn't. You may, for instance, choose only
+to divert traffic to some hosts into the transparent proxy. There are a huge
+number of ways to accomplish this, and much will depend on the router or
+packet filter you're using. In most cases, the configuration will look like
+this:
+
+<a href="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
+</a>
+
+
+<div class="page-header">
+    <h1>Reverse Proxy</h1>
+</div>
+
+Mitmproxy is usually used with a client that uses the proxy to access the
+Internet. Using reverse proxy mode, you can use mitmproxy to act like a normal
+HTTP server:
+
+<a href="@!urlTo('schematics/proxy-modes-reverse.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-reverse.png')!@">
+</a>
+
+There are various use-cases:
+
+- Say you have an internal API running at http://example.local/. You could now
+set up mitmproxy in reverse proxy mode at http://debug.example.local/ and
+dynamically point clients to this new API endpoint, which provides clients
+with the same data and you with debug information. Similarly, you could move
+your real server to a different IP/port and set up mitmproxy at the original
+place to debug all sessions.
+
+- Say you're a web developer working on example.com (with a development
+version running on localhost:8000). You can modify your hosts file so that
+example.com points to 127.0.0.1 and then run mitmproxy in reverse proxy mode
+on port 80. You can test your app on the example.com domain and get all
+requests recorded in mitmproxy.
+
+- Say you have some toy project that should get SSL support. Simply set up
+mitmproxy with SSL termination and you're done (<code>mitmdump -p 443 -R
+https2http://localhost:80/</code>). There are better tools for this specific
+task, but mitmproxy is very quick and simple way to set up an SSL-speaking
+server.
+
+- Want to add a non-SSL-capable compression proxy in front of your server? You
+could even spawn a mitmproxy instance that terminates SSL (https2http://...),
+point it to the compression proxy and let the compression proxy point to a
+SSL-initiating mitmproxy (http2https://...), which then points to the real
+server. As you see, it's a fairly flexible thing.
+
+Note that mitmproxy supports either an HTTP or an HTTPS upstream server, not
+both at the same time. You can work around this by spawning a second mitmproxy
+instance.
+
+<div class="well">
+    <strong style="text-align: center; display: block">Caveat: Interactive Use</strong>
+
+
+One caveat is that reverse proxy mode is often not sufficient for interactive
+browsing. Consider trying to clone Google by using:
+
+<code>mitmproxy -R http://google.com/</code>
+
+This works for the initial request, but the HTML served to the client remains
+unchanged. As soon as the user clicks on an non-relative URL (or downloads a
+non-relative image resource), traffic no longer passes through mitmproxy, and
+the client connects to Google directly again.
+
+</div>
+
+
+
+<div class="page-header">
+    <h1>Upstream Proxy</h1>
+</div>
+
+If you want to chain proxies by adding mitmproxy in front of a different proxy
+appliance, you can use mitmproxy's upstream mode. In upstream mode, all
+requests are unconditionally transferred to an upstream proxy of your choice.
+
+<a href="@!urlTo('schematics/proxy-modes-upstream.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-upstream.png')!@"></a>
+
+mitmproxy supports both explicit HTTP and explicit HTTPS in upstream proxy
+mode. You could in theory chain multiple mitmproxy instances in a row, but
+that doesn't make any sense in practice (i.e. outside of our tests). 

doc-src/scripting/inlinescripts.html

@@ -1,5 +1,5 @@
 __mitmproxy__ has a powerful scripting API that allows you to modify flows
-on-the-fly or rewrite previously saved flows locally. 
+on-the-fly or rewrite previously saved flows locally.
 
 The mitmproxy scripting API is event driven - a script is simply a Python
 module that exposes a set of event methods. Here's a complete mitmproxy script
@@ -21,6 +21,11 @@ We can now run this script using mitmdump or mitmproxy as follows:
 
 The new header will be added to all responses passing through the proxy.
 
+## Example Scripts
+
+mitmproxy comes with a variety of example inline scripts, which demonstrate
+many basic tasks. We encourage you to either browse them locally or in our
+[GitHub repo](https://github.com/mitmproxy/mitmproxy/tree/master/examples).
 
 
 ## Events
@@ -30,38 +35,45 @@ The new header will be added to all responses passing through the proxy.
 Called once on startup, before any other events.
 
 
-### clientconnect(ScriptContext, ClientConnect)
+### clientconnect(ScriptContext, ConnectionHandler)
 
 Called when a client initiates a connection to the proxy. Note that
 a connection can correspond to multiple HTTP requests.
 
-
-### serverconnect(ScriptContext, ServerConnection)
+### serverconnect(ScriptContext, ConnectionHandler)
 
 Called when the proxy initiates a connection to the target server. Note that
 a connection can correspond to multiple HTTP requests.
 
-### request(ScriptContext, Flow)
+### request(ScriptContext, HTTPFlow)
 
-Called when a client request has been received. The __Flow__ object is
+Called when a client request has been received. The __HTTPFlow__ object is
 guaranteed to have a non-None __request__ attribute.
 
+### responseheaders(ScriptContext, HTTPFlow)
 
-### response(ScriptContext, Flow)
+Called when the headers of a server response have been received.
+This will always be called before the response hook.
+The __HTTPFlow__ object is guaranteed to have non-None __request__ and
+__response__ attributes.  __response.content__ will be None,
+as the response body has not been read yet.
 
-Called when a server response has been received. The __Flow__ object is
+### response(ScriptContext, HTTPFlow)
+
+Called when a server response has been received. The __HTTPFlow__ object is
 guaranteed to have non-None __request__ and __response__ attributes.
+Note that if response streaming is enabled for this response,
+__response.content__ will not contain the response body.
 
-
-### error(ScriptContext, Flow)
+### error(ScriptContext, HTTPFlow)
 
 Called when a flow error has occurred, e.g. invalid server responses, or
 interrupted connections. This is distinct from a valid server HTTP error
-response, which is simply a response with an HTTP error code. The __Flow__
+response, which is simply a response with an HTTP error code. The __HTTPFlow__
 object is guaranteed to have non-None __request__ and __error__ attributes.
 
 
-### clientdisconnect(ScriptContext, ClientDisconnect)
+### clientdisconnect(ScriptContext, ConnectionHandler)
 
 Called when a client disconnects from the proxy.
 
@@ -76,43 +88,46 @@ The main classes you will deal with in writing mitmproxy scripts are:
 
 <table class="table">
     <tr>
-        <th>libmproxy.flow.ClientConnection</th>
+        <th>libmproxy.proxy.server.ConnectionHandler</th>
+        <td>Describes a proxy client connection session. Always has a client_conn attribute, might have a server_conn
+            attribute.
+        </td>
+    </tr>
+    <tr>
+        <th>libmproxy.proxy.connection.ClientConnection</th>
         <td>Describes a client connection.</td>
     </tr>
     <tr>
-        <th>libmproxy.flow.ClientDisconnection</th>
-        <td>Describes a client disconnection.</td>
+        <th>libmproxy.proxy.connection.ServerConnection</th>
+        <td>Describes a server connection.</td>
     </tr>
     <tr>
-        <th>libmproxy.flow.Error</th>
-        <td>A communications error.</td>
-    </tr>
-    <tr>
-        <th>libmproxy.flow.Flow</th>
+        <th>libmproxy.protocol.http.HTTPFlow</th>
         <td>A collection of objects representing a single HTTP transaction.</td>
     </tr>
     <tr>
-        <th>libmproxy.flow.Headers</th>
-        <td>HTTP headers for a request or response.</td>
+        <th>libmproxy.protocol.http.HTTPResponse</th>
+        <td>An HTTP response.</td>
+    </tr>
+    <tr>
+        <th>libmproxy.protocol.http.HTTPRequest</th>
+        <td>An HTTP request.</td>
+    </tr>
+    <tr>
+        <th>libmproxy.protocol.primitives.Error</th>
+        <td>A communications error.</td>
+    </tr>
+    <tr>
+        <th>libmproxy.script.ScriptContext</th>
+        <td> A handle for interacting with mitmproxy's from within scripts.</td>
     </tr>
     <tr>
         <th>libmproxy.flow.ODict</th>
 
         <td>A dictionary-like object for managing sets of key/value data. There
-        is also a variant called CaselessODict that ignores key case for some
-        calls (used mainly for headers).</td>
-    </tr>
-    <tr>
-        <th>libmproxy.flow.Response</th>
-        <td>An HTTP response.</td>
-    </tr>
-    <tr>
-        <th>libmproxy.flow.Request</th>
-        <td>An HTTP request.</td>
-    </tr>
-    <tr>
-        <th>libmproxy.flow.ScriptContext</th>
-        <td> A handle for interacting with mitmproxy's from within scripts.  </td>
+            is also a variant called CaselessODict that ignores key case for some
+            calls (used mainly for headers).
+        </td>
     </tr>
     <tr>
         <th>libmproxy.certutils.SSLCert</th>
@@ -120,11 +135,12 @@ The main classes you will deal with in writing mitmproxy scripts are:
     </tr>
 </table>
 
-The canonical API documentation is the code. You can view the API documentation
-using pydoc (which is installed with Python by default), like this:
+The canonical API documentation is the code, which you can browse locally or in our
+[GitHub repo](https://github.com/mitmproxy/mitmproxy).
+You can view the API documentation using pydoc (which is installed with Python by default), like this:
 
 <pre class="terminal">
-> pydoc libmproxy.flow.Request
+> pydoc libmproxy.protocol.http.HTTPRequest
 </pre>
 
 
@@ -135,6 +151,13 @@ While that's a very desirable behaviour under some circumstances, scripts can be
 
 $!example("examples/nonblocking.py")!$
 
+## Make scripts configurable with arguments
+
+Sometimes, you want to pass runtime arguments to the inline script. This can be simply done by surrounding the script call with quotes, e.g.
+<code>mitmdump -s "script.py --foo 42"</code>. The arguments are then exposed in the start event:
+
+$!example("examples/modify_response_body.py")!$
+
 ## Running scripts on saved flows
 
 Sometimes, we want to run a script on __Flow__ objects that are already
@@ -144,6 +167,11 @@ flows from a file (see the "scripted data transformation" example on the
 one-shot script on a single flow through the _|_ (pipe) shortcut in mitmproxy.
 
 In this case, there are no client connections, and the events are run in the
-following order: __start__, __request__, __response__, __error__, __done__.  If
+following order: __start__, __request__, __responseheaders__, __response__, __error__, __done__.  If
 the flow doesn't have a __response__ or __error__ associated with it, the
-matching event will be skipped. 
+matching events will be skipped.
+
+## Spaces in the script path
+By default, spaces are interpreted as separator between the inline script and its arguments (e.g. <code>-s "foo.py
+    42"</code>). Consequently, the script path needs to be wrapped in a separate pair of quotes if it contains spaces:
+<code>-s "'./foo bar/baz.py' 42"</code>.

doc-src/scripting/libmproxy.html

@@ -1,3 +1,13 @@
+<div class="well">
+    <strong>Heads up!</strong> We strongly encourage you to use <a href="@!urlTo("scripting/inlinescripts.html")!@">inline scripts</a> rather than libmproxy
+    directly.<br><br>
+    <ul>
+        <li>Inline Scripts are equally powerful and provide an easier syntax.</li>
+        <li>Most examples are written as inline scripts.</li>
+        <li>Multiple inline scripts can be combined and used together.</li>
+        <li>Inline Scripts can either be executed headless with mitmdump or within the mitmproxy UI.</li>
+    </ul>
+</div>
 
 All of mitmproxy's basic functionality is exposed through the __libmproxy__
 library. The example below shows a simple implementation of the "sticky cookie"

doc-src/ssl.html

@@ -41,10 +41,26 @@ The files created by mitmproxy in the .mitmproxy directory are as follows:
 Using a custom certificate
 --------------------------
 
-You can use your own certificate by passing the __--cert__ option to mitmproxy.
+You can use your own certificate by passing the <kbd>--cert</kbd> option to mitmproxy. mitmproxy then uses the provided
+certificate for interception of the specified domains instead of generating a cert signed by its own CA.
 
-The certificate file is expected to be in the PEM format. You can generate
-a certificate in this format using these instructions:
+The certificate file is expected to be in the PEM format.
+You can include intermediary certificates right below your leaf certificate, so that you PEM file roughly looks like
+this:
+
+<pre>
+-----BEGIN PRIVATE KEY-----
+&lt;private key&gt;
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+&lt;cert&gt;
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+&lt;intermediary cert (optional)&gt;
+-----END CERTIFICATE-----
+</pre>
+
+For example, you can generate a certificate in this format using these instructions:
 
 <pre class="terminal">
 > openssl genrsa -out cert.key 8192
@@ -55,6 +71,15 @@ a certificate in this format using these instructions:
 </pre>
 
 
+Using a custom certificate authority
+------------------------------------
+
+By default, mitmproxy will (generate and) use <samp>~/.mitmproxy/mitmproxy-ca.pem</samp> as the default certificate
+authority to generate certificates for all domains for which no custom certificate is provided (see above).
+You can use your own certificate authority by passing the <kbd>--confdir</kbd> option to mitmproxy.
+mitmproxy will then look for <samp>mitmproxy-ca.pem</samp> in the specified directory. If no such file exists,
+it will be generated automatically.
+
 Installing the mitmproxy CA
 ---------------------------
 

doc-src/transparent/linux.html

@@ -15,6 +15,16 @@ achieve transparent mode.
 
     </li>
 
+    <li> If your target machine is on the same physical network and you configured it to use a custom gateway,
+    disable ICMP redirects:
+
+    <pre class="terminal">echo 0 | sudo tee /proc/sys/net/ipv4/conf/*/send_redirects</pre>
+
+    You may also want to consider enabling this permanently in
+    <b>/etc/sysctl.conf</b> as demonstrated <a href="http://unix.stackexchange.com/a/58081">here</a>.
+
+    </li>
+
     <li> Create an iptables ruleset that redirects the desired traffic to the
     mitmproxy port. Details will differ according to your setup, but the
     ruleset should look something like this:

examples/README

@@ -1,10 +1,23 @@
-add_header.py           Simple script that just adds a header to every request.
-dup_and_replay.py       Duplicates each request, changes it, and then replays the modified request.
-flowbasic               Basic use of mitmproxy as a library.
-modify_form.py          Modify all form submissions to add a parameter.
-modify_querystring.py   Modify all query strings to add a parameters.
-proxapp                 How to embed a WSGI app in a mitmproxy server
-stub.py                 Script stub with a method definition for every event.
-stickycookies           An example of writing a custom proxy with libmproxy.
-upsidedownternet.py     Rewrites traffic to turn PNGs upside down.
-mitmproxywrapper.py     Bracket mitmproxy run with proxy enable/disable on OS X
+# inline script examples
+add_header.py             Simple script that just adds a header to every request.
+change_upstream_proxy.py  Dynamically change the upstream proxy
+dup_and_replay.py         Duplicates each request, changes it, and then replays the modified request.
+iframe_injector.py        Inject configurable iframe into pages.
+modify_form.py            Modify all form submissions to add a parameter.
+modify_querystring.py     Modify all query strings to add a parameters.
+modify_response_body.py   Replace arbitrary strings in all responses
+nonblocking.py            Demonstrate parallel processing with a blocking script.
+proxapp.py                How to embed a WSGI app in a mitmproxy server
+redirect_requests.py      Redirect requests or directly reply to them.
+stub.py                   Script stub with a method definition for every event.
+upsidedownternet.py       Rewrites traffic to turn images upside down.
+
+
+# libmproxy examples
+flowbasic                 Basic use of mitmproxy as a library.
+stickycookies             An example of writing a custom proxy with libmproxy.
+
+
+# misc
+read_dumpfile             Read a dumpfile generated by mitmproxy.
+mitmproxywrapper.py       Bracket mitmproxy run with proxy enable/disable on OS X

examples/add_header.py

@@ -1,2 +1,2 @@
 def response(context, flow):
-    flow.response.headers["newheader"] = ["foo"]
+    flow.response.headers["newheader"] = ["foo"]

examples/change_upstream_proxy.py

@@ -0,0 +1,21 @@
+# This scripts demonstrates how mitmproxy can switch to a second/different upstream proxy
+# in upstream proxy mode.
+#
+# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s "change_upstream_proxy.py host"
+from libmproxy.protocol.http import send_connect_request
+
+alternative_upstream_proxy = ("localhost", 8082)
+def should_redirect(flow):
+    return flow.request.host == "example.com"
+
+
+def request(context, flow):
+    if flow.live and should_redirect(flow):
+
+        # If you want to change the target server, you should modify flow.request.host and flow.request.port
+        # flow.live.change_server should only be used by inline scripts to change the upstream proxy,
+        # unless you are sure that you know what you are doing.
+        server_changed = flow.live.change_server(alternative_upstream_proxy, persistent_change=True)
+        if flow.request.scheme == "https" and server_changed:
+            send_connect_request(flow.live.c.server_conn, flow.request.host, flow.request.port)
+            flow.live.c.establish_ssl(server=True)

examples/dup_and_replay.py

@@ -1,4 +1,4 @@
-def request(ctx, flow):
-   f = ctx.duplicate_flow(flow)
-   f.request.path = "/changed"
-   ctx.replay_request(f)
+def request(context, flow):
+    f = context.duplicate_flow(flow)
+    f.request.path = "/changed"
+    context.replay_request(f)

examples/flowbasic

@@ -3,11 +3,15 @@
     This example shows how to build a proxy based on mitmproxy's Flow
     primitives.
 
+    Heads Up: In the majority of cases, you want to use inline scripts.
+
     Note that request and response messages are not automatically replied to,
     so we need to implement handlers to do this.
 """
 import os
-from libmproxy import proxy, flow
+from libmproxy import flow, proxy
+from libmproxy.proxy.server import ProxyServer
+
 
 class MyMaster(flow.FlowMaster):
     def run(self):
@@ -16,24 +20,25 @@ class MyMaster(flow.FlowMaster):
         except KeyboardInterrupt:
             self.shutdown()
 
-    def handle_request(self, r):
-        f = flow.FlowMaster.handle_request(self, r)
+    def handle_request(self, f):
+        f = flow.FlowMaster.handle_request(self, f)
         if f:
-            r.reply()
+            f.reply()
         return f
 
-    def handle_response(self, r):
-        f = flow.FlowMaster.handle_response(self, r)
+    def handle_response(self, f):
+        f = flow.FlowMaster.handle_response(self, f)
         if f:
-            r.reply()
+            f.reply()
         print f
         return f
 
 
 config = proxy.ProxyConfig(
-    cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
+    port=8080,
+    confdir="~/.mitmproxy/"  # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file.
 )
 state = flow.State()
-server = proxy.ProxyServer(config, 8080)
+server = ProxyServer(config)
 m = MyMaster(server, state)
 m.run()

examples/iframe_injector

@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-"""
-    Zap encoding in requests and inject iframe after body tag in html responses.
-    Usage:
-        iframe_injector http://someurl/somefile.html
-"""
-from libmproxy import controller, proxy
-import os
-import sys
-
-
-class InjectingMaster(controller.Master):
-    def __init__(self, server, iframe_url):
-        controller.Master.__init__(self, server)
-        self._iframe_url = iframe_url
-
-    def run(self):
-        try:
-            return controller.Master.run(self)
-        except KeyboardInterrupt:
-            self.shutdown()
-
-    def handle_request(self, msg):
-        if 'Accept-Encoding' in msg.headers:
-            msg.headers["Accept-Encoding"] = 'none'
-        msg.reply()
-
-    def handle_response(self, msg):
-        if msg.content:
-            c = msg.replace('<body>', '<body><iframe src="%s" frameborder="0" height="0" width="0"></iframe>' % self._iframe_url)
-            if c > 0:
-                print 'Iframe injected!'
-        msg.reply()
-
-
-def main(argv):
-    if len(argv) != 2:
-        print "Usage: %s IFRAME_URL" % argv[0]
-        sys.exit(1)
-    iframe_url = argv[1]
-    config = proxy.ProxyConfig(
-        cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
-    )
-    server = proxy.ProxyServer(config, 8080)
-    print 'Starting proxy...'
-    m = InjectingMaster(server, iframe_url)
-    m.run()
-
-if __name__ == '__main__':
-    main(sys.argv)

examples/iframe_injector.py

@@ -0,0 +1,22 @@
+# Usage: mitmdump -s "iframe_injector.py url"
+# (this script works best with --anticache)
+from bs4 import BeautifulSoup
+from libmproxy.protocol.http import decoded
+
+
+def start(context, argv):
+    if len(argv) != 2:
+        raise ValueError('Usage: -s "iframe_injector.py url"')
+    context.iframe_url = argv[1]
+
+
+def response(context, flow):
+    if flow.request.host in context.iframe_url:
+        return
+    with decoded(flow.response):  # Remove content encoding (gzip, ...)
+        html = BeautifulSoup(flow.response.content)
+        if html.body:
+            iframe = html.new_tag("iframe", src=context.iframe_url, frameborder=0, height=0, width=0)
+            html.body.insert(0, iframe)
+            flow.response.content = str(html)
+            context.log("Iframe inserted.")

examples/mitmproxywrapper.py

@@ -86,7 +86,6 @@ class Wrapper(object):
             cmd = ['mitmproxy', '-p', str(self.port)]
             if self.extra_arguments:
                 cmd.extend(self.extra_arguments)
-            cmd.extend(['--palette', 'light'])
             subprocess.check_call(cmd)
 
     def wrap_honeyproxy(self):

examples/modify_form.py

@@ -1,8 +1,6 @@
 
 def request(context, flow):
     if "application/x-www-form-urlencoded" in flow.request.headers["content-type"]:
-        frm = flow.request.get_form_urlencoded()
-        frm["mitmproxy"] = ["rocks"]
-        flow.request.set_form_urlencoded(frm)
-
-
+        form = flow.request.get_form_urlencoded()
+        form["mitmproxy"] = ["rocks"]
+        flow.request.set_form_urlencoded(form)

examples/modify_querystring.py

@@ -3,5 +3,4 @@ def request(context, flow):
     q = flow.request.get_query()
     if q:
         q["mitmproxy"] = ["rocks"]
-        flow.request.set_query(q)
-
+        flow.request.set_query(q)

examples/modify_response_body.py

@@ -0,0 +1,15 @@
+# Usage: mitmdump -s "modify_response_body.py mitmproxy bananas"
+# (this script works best with --anticache)
+from libmproxy.protocol.http import decoded
+
+
+def start(context, argv):
+    if len(argv) != 3:
+        raise ValueError('Usage: -s "modify-response-body.py old new"')
+    # You may want to use Python's argparse for more sophisticated argument parsing.
+    context.old, context.new = argv[1], argv[2]
+
+
+def response(context, flow):
+    with decoded(flow.response):  # automatically decode gzipped responses.
+        flow.response.content = flow.response.content.replace(context.old, context.new)

examples/nonblocking.py

@@ -1,8 +1,9 @@
 import time
 from libmproxy.script import concurrent
 
-@concurrent
+
+@concurrent  # Remove this and see what happens
 def request(context, flow):
     print "handle request: %s%s" % (flow.request.host, flow.request.path)
     time.sleep(5)
-    print "start  request: %s%s" % (flow.request.host, flow.request.path)
+    print "start  request: %s%s" % (flow.request.host, flow.request.path)

examples/proxapp

@@ -1,47 +0,0 @@
-#!/usr/bin/env python
-"""
-This example shows how to graft a WSGI app onto mitmproxy. In this
-instance, we're using the Bottle framework (http://bottlepy.org/) to expose
-a single simplest-possible page.
-"""
-import bottle
-import os
-from libmproxy import proxy, flow
-
-@bottle.route('/')
-def index():
-    return 'Hi!'
-
-
-class MyMaster(flow.FlowMaster):
-    def run(self):
-        try:
-            flow.FlowMaster.run(self)
-        except KeyboardInterrupt:
-            self.shutdown()
-
-    def handle_request(self, r):
-        f = flow.FlowMaster.handle_request(self, r)
-        if f:
-            r.reply()
-        return f
-
-    def handle_response(self, r):
-        f = flow.FlowMaster.handle_response(self, r)
-        if f:
-            r.reply()
-        print f
-        return f
-
-
-config = proxy.ProxyConfig(
-    cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
-)
-state = flow.State()
-server = proxy.ProxyServer(config, 8080)
-# Register the app using the magic domain "proxapp" on port 80. Requests to
-# this domain and port combination will now be routed to the WSGI app instance.
-server.apps.add(bottle.app(), "proxapp", 80)
-m = MyMaster(server, state)
-m.run()
-

examples/proxapp.py

@@ -0,0 +1,24 @@
+"""
+This example shows how to graft a WSGI app onto mitmproxy. In this
+instance, we're using the Flask framework (http://flask.pocoo.org/) to expose
+a single simplest-possible page.
+"""
+from flask import Flask
+
+app = Flask("proxapp")
+
+
+@app.route('/')
+def hello_world():
+    return 'Hello World!'
+
+
+# Register the app using the magic domain "proxapp" on port 80. Requests to
+# this domain and port combination will now be routed to the WSGI app instance.
+def start(context, argv):
+    context.app_registry.add(app, "proxapp", 80)
+
+    # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design.
+    # mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set)
+    # but won't send any data.
+    context.app_registry.add(app, "example.com", 443)

examples/read_dumpfile

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+#
+# Simple script showing how to read a mitmproxy dump file
+#
+
+from libmproxy import flow
+import json, sys
+
+with open("logfile", "rb") as logfile:
+    freader = flow.FlowReader(logfile)
+    try:
+        for f in freader.stream():
+            print(f)
+            print(f.request.host)
+            json.dump(f.get_state(), sys.stdout, indent=4)
+            print ""
+    except flow.FlowReadError, v:
+        print "Flow file corrupted. Stopped loading."

examples/redirect_requests.py

@@ -1,19 +1,24 @@
-from libmproxy.flow import Response
+from libmproxy.protocol.http import HTTPResponse
 from netlib.odict import ODictCaseless
 
 """
 This example shows two ways to redirect flows to other destinations.
 """
 
+
 def request(context, flow):
-    if flow.request.host.endswith("example.com"):
-        resp = Response(flow.request,
-                        [1,1],
-                        200, "OK",
-                        ODictCaseless([["Content-Type","text/html"]]),
-                        "helloworld",
-                        None)
-        flow.request.reply(resp)
-    if flow.request.host.endswith("example.org"):
+    # pretty_host(hostheader=True) takes the Host: header of the request into account,
+    # which is useful in transparent mode where we usually only have the IP otherwise.
+
+    # Method 1: Answer with a locally generated response
+    if flow.request.pretty_host(hostheader=True).endswith("example.com"):
+        resp = HTTPResponse(
+            [1, 1], 200, "OK",
+            ODictCaseless([["Content-Type", "text/html"]]),
+            "helloworld")
+        flow.reply(resp)
+
+    # Method 2: Redirect the request to a different server
+    if flow.request.pretty_host(hostheader=True).endswith("example.org"):
         flow.request.host = "mitmproxy.org"
-        flow.request.headers["Host"] = ["mitmproxy.org"]
+        flow.request.update_host_header()

examples/stickycookies

@@ -1,12 +1,14 @@
 #!/usr/bin/env python
 """
 This example builds on mitmproxy's base proxying infrastructure to
-implement functionality similar to the "sticky cookies" option. This is at
-a lower level than the Flow mechanism, so we're dealing directly with
-request and response objects.
+implement functionality similar to the "sticky cookies" option.
+
+Heads Up: In the majority of cases, you want to use inline scripts.
 """
-from libmproxy import controller, proxy
 import os
+from libmproxy import controller, proxy
+from libmproxy.proxy.server import ProxyServer
+
 
 class StickyMaster(controller.Master):
     def __init__(self, server):
@@ -19,24 +21,22 @@ class StickyMaster(controller.Master):
         except KeyboardInterrupt:
             self.shutdown()
 
-    def handle_request(self, msg):
-        hid = (msg.host, msg.port)
-        if msg.headers["cookie"]:
-            self.stickyhosts[hid] = msg.headers["cookie"]
+    def handle_request(self, flow):
+        hid = (flow.request.host, flow.request.port)
+        if flow.request.headers["cookie"]:
+            self.stickyhosts[hid] = flow.request.headers["cookie"]
         elif hid in self.stickyhosts:
-            msg.headers["cookie"] = self.stickyhosts[hid]
-        msg.reply()
+            flow.request.headers["cookie"] = self.stickyhosts[hid]
+        flow.reply()
 
-    def handle_response(self, msg):
-        hid = (msg.request.host, msg.request.port)
-        if msg.headers["set-cookie"]:
-            self.stickyhosts[hid] = msg.headers["set-cookie"]
-        msg.reply()
+    def handle_response(self, flow):
+        hid = (flow.request.host, flow.request.port)
+        if flow.response.headers["set-cookie"]:
+            self.stickyhosts[hid] = flow.response.headers["set-cookie"]
+        flow.reply()
 
 
-config = proxy.ProxyConfig(
-    cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
-)
-server = proxy.ProxyServer(config, 8080)
+config = proxy.ProxyConfig(port=8080)
+server = ProxyServer(config)
 m = StickyMaster(server)
 m.run()

examples/stream.py

@@ -0,0 +1,5 @@
+def responseheaders(context, flow):
+    """
+    Enables streaming for all responses.
+    """
+    flow.response.stream = True

examples/stub.py

@@ -1,54 +1,63 @@
 """
     This is a script stub, with definitions for all events.
 """
-def start(ctx, argv):
+def start(context, argv):
     """
         Called once on script startup, before any other events.
     """
-    ctx.log("start")
+    context.log("start")
 
-def clientconnect(ctx, client_connect):
+def clientconnect(context, conn_handler):
     """
         Called when a client initiates a connection to the proxy. Note that a
         connection can correspond to multiple HTTP requests
     """
-    ctx.log("clientconnect")
+    context.log("clientconnect")
 
-def serverconnect(ctx, server_connection):
+def serverconnect(context, conn_handler):
     """
         Called when the proxy initiates a connection to the target server. Note that a
         connection can correspond to multiple HTTP requests
     """
-    ctx.log("serverconnect")
+    context.log("serverconnect")
 
-def request(ctx, flow):
+def request(context, flow):
     """
         Called when a client request has been received.
     """
-    ctx.log("request")
+    context.log("request")
 
-def response(ctx, flow):
+
+def responseheaders(context, flow):
+    """
+        Called when the response headers for a server response have been received,
+        but the response body has not been processed yet. Can be used to tell mitmproxy
+        to stream the response.
+    """
+    context.log("responseheaders")
+
+def response(context, flow):
     """
        Called when a server response has been received.
     """
-    ctx.log("response")
+    context.log("response")
 
-def error(ctx, flow):
+def error(context, flow):
     """
         Called when a flow error has occured, e.g. invalid server responses, or
         interrupted connections. This is distinct from a valid server HTTP error
         response, which is simply a response with an HTTP error code.
     """
-    ctx.log("error")
+    context.log("error")
 
-def clientdisconnect(ctx, client_disconnect):
+def clientdisconnect(context, conn_handler):
     """
         Called when a client disconnects from the proxy.
     """
-    ctx.log("clientdisconnect")
+    context.log("clientdisconnect")
 
-def done(ctx):
+def done(context):
     """
         Called once on script shutdown, after any other events.
     """
-    ctx.log("done")
+    context.log("done")

examples/upsidedownternet.py

@@ -1,10 +1,16 @@
 import cStringIO
 from PIL import Image
+from libmproxy.protocol.http import decoded
 
 def response(context, flow):
-    if flow.response.headers["content-type"] == ["image/png"]:
-        s = cStringIO.StringIO(flow.response.content)
-        img = Image.open(s).rotate(180)
-        s2 = cStringIO.StringIO()
-        img.save(s2, "png")
-        flow.response.content = s2.getvalue()
+    if flow.response.headers.get_first("content-type", "").startswith("image"):
+        with decoded(flow.response):  # automatically decode gzipped responses.
+            try:
+                s = cStringIO.StringIO(flow.response.content)
+                img = Image.open(s).rotate(180)
+                s2 = cStringIO.StringIO()
+                img.save(s2, "png")
+                flow.response.content = s2.getvalue()
+                flow.response.headers["content-type"] = ["image/png"]
+            except:  # Unknown image types etc.
+                pass

libmproxy/app.py

@@ -1,27 +0,0 @@
-import flask
-import os.path
-
-mapp = flask.Flask(__name__)
-mapp.debug = True
-
-def master():
-    return flask.request.environ["mitmproxy.master"]
-
-@mapp.route("/")
-def index():
-    return flask.render_template("index.html", section="home")
-
-
-@mapp.route("/cert/pem")
-def certs_pem():
-    capath = master().server.config.cacert
-    p = os.path.splitext(capath)[0] + "-cert.pem"
-    return flask.Response(open(p).read(), mimetype='application/x-x509-ca-cert')
-
-
-@mapp.route("/cert/p12")
-def certs_p12():
-    capath = master().server.config.cacert
-    p = os.path.splitext(capath)[0] + "-cert.p12"
-    return flask.Response(open(p).read(), mimetype='application/x-pkcs12')
-

libmproxy/cmdline.py

@@ -1,12 +1,17 @@
-import proxy
-import re, filt
-import argparse
+from __future__ import absolute_import
+import re
+from argparse import ArgumentTypeError
+from netlib import http
+from . import filt, utils
+from .proxy import config
 
 APP_HOST = "mitm.it"
 APP_PORT = 80
 
-class ParseException(Exception): pass
-class OptionException(Exception): pass
+
+class ParseException(Exception):
+    pass
+
 
 def _parse_hook(s):
     sep, rem = s[0], s[1:]
@@ -17,13 +22,13 @@ def _parse_hook(s):
     elif len(parts) == 3:
         patt, a, b = parts
     else:
-        raise ParseException("Malformed hook specifier - too few clauses: %s"%s)
+        raise ParseException("Malformed hook specifier - too few clauses: %s" % s)
 
     if not a:
-        raise ParseException("Empty clause: %s"%str(patt))
+        raise ParseException("Empty clause: %s" % str(patt))
 
     if not filt.parse(patt):
-        raise ParseException("Malformed filter pattern: %s"%patt)
+        raise ParseException("Malformed filter pattern: %s" % patt)
 
     return patt, a, b
 
@@ -58,7 +63,7 @@ def parse_replace_hook(s):
     try:
         re.compile(regex)
     except re.error, e:
-        raise ParseException("Malformed replacement regex: %s"%str(e.message))
+        raise ParseException("Malformed replacement regex: %s" % str(e.message))
     return patt, regex, replacement
 
 
@@ -91,6 +96,25 @@ def parse_setheader(s):
     return _parse_hook(s)
 
 
+def parse_server_spec(url):
+    normalized_url = re.sub("^https?2", "", url)
+
+    p = http.parse_url(normalized_url)
+    if not p or not p[1]:
+        raise ArgumentTypeError("Invalid server specification: %s" % url)
+
+    if url.lower().startswith("https2http"):
+        ssl = [True, False]
+    elif url.lower().startswith("http2https"):
+        ssl = [False, True]
+    elif url.lower().startswith("https"):
+        ssl = [True, True]
+    else:
+        ssl = [False, False]
+
+    return ssl + list(p[1:3])
+
+
 def get_common_options(options):
     stickycookie, stickyauth = None, None
     if options.stickycookie_filt:
@@ -99,67 +123,64 @@ def get_common_options(options):
     if options.stickyauth_filt:
         stickyauth = options.stickyauth_filt
 
+    stream_large_bodies = utils.parse_size(options.stream_large_bodies)
+
     reps = []
     for i in options.replace:
         try:
             p = parse_replace_hook(i)
         except ParseException, e:
-            raise OptionException(e.message)
+            raise ArgumentTypeError(e.message)
         reps.append(p)
     for i in options.replace_file:
         try:
             patt, rex, path = parse_replace_hook(i)
         except ParseException, e:
-            raise OptionException(e.message)
+            raise ArgumentTypeError(e.message)
         try:
             v = open(path, "rb").read()
         except IOError, e:
-            raise OptionException("Could not read replace file: %s"%path)
+            raise ArgumentTypeError("Could not read replace file: %s" % path)
         reps.append((patt, rex, v))
 
-
     setheaders = []
     for i in options.setheader:
         try:
             p = parse_setheader(i)
         except ParseException, e:
-            raise OptionException(e.message)
+            raise ArgumentTypeError(e.message)
         setheaders.append(p)
 
     return dict(
-        app = options.app,
-        app_host = options.app_host,
-        app_port = options.app_port,
-        app_external = options.app_external,
+        app=options.app,
+        app_host=options.app_host,
+        app_port=options.app_port,
 
-        anticache = options.anticache,
-        anticomp = options.anticomp,
-        client_replay = options.client_replay,
-        eventlog = options.eventlog,
-        kill = options.kill,
-        no_server = options.no_server,
-        refresh_server_playback = not options.norefresh,
-        rheaders = options.rheaders,
-        rfile = options.rfile,
-        replacements = reps,
-        setheaders = setheaders,
-        server_replay = options.server_replay,
-        scripts = options.scripts,
-        stickycookie = stickycookie,
-        stickyauth = stickyauth,
-        showhost = options.showhost,
-        wfile = options.wfile,
-        verbosity = options.verbose,
-        nopop = options.nopop,
+        anticache=options.anticache,
+        anticomp=options.anticomp,
+        client_replay=options.client_replay,
+        kill=options.kill,
+        no_server=options.no_server,
+        refresh_server_playback=not options.norefresh,
+        rheaders=options.rheaders,
+        rfile=options.rfile,
+        replacements=reps,
+        setheaders=setheaders,
+        server_replay=options.server_replay,
+        scripts=options.scripts,
+        stickycookie=stickycookie,
+        stickyauth=stickyauth,
+        stream_large_bodies=stream_large_bodies,
+        showhost=options.showhost,
+        wfile=options.wfile,
+        verbosity=options.verbose,
+        nopop=options.nopop,
+        replay_ignore_content = options.replay_ignore_content,
+        replay_ignore_params = options.replay_ignore_params
     )
 
 
 def common_options(parser):
-    parser.add_argument(
-        "-b",
-        action="store", type = str, dest="addr", default='',
-        help = "Address to bind proxy to (defaults to all interfaces)"
-    )
     parser.add_argument(
         "--anticache",
         action="store_true", dest="anticache", default=False,
@@ -167,33 +188,13 @@ def common_options(parser):
     )
     parser.add_argument(
         "--confdir",
-        action="store", type = str, dest="confdir", default='~/.mitmproxy',
-        help = "Configuration directory. (~/.mitmproxy)"
+        action="store", type=str, dest="confdir", default='~/.mitmproxy',
+        help="Configuration directory, contains default CA file. (~/.mitmproxy)"
     )
     parser.add_argument(
-        "-e",
-        action="store_true", dest="eventlog",
-        help="Show event log."
-    )
-    parser.add_argument(
-        "-n",
-        action="store_true", dest="no_server",
-        help="Don't start a proxy server."
-    )
-    parser.add_argument(
-        "-p",
-        action="store", type = int, dest="port", default=8080,
-        help = "Proxy service port."
-    )
-    parser.add_argument(
-        "-P",
-        action="store", dest="reverse_proxy", default=None,
-        help="Reverse proxy to upstream server: http[s]://host[:port]"
-    )
-    parser.add_argument(
-        "-F",
-        action="store", dest="forward_proxy", default=None,
-        help="Proxy to unconditionally forward to: http[s]://host[:port]"
+        "--host",
+        action="store_true", dest="showhost", default=False,
+        help="Use the Host header to construct URLs for display."
     )
     parser.add_argument(
         "-q",
@@ -216,11 +217,6 @@ def common_options(parser):
         action="store", dest="stickycookie_filt", default=None, metavar="FILTER",
         help="Set sticky cookie filter. Matched against requests."
     )
-    parser.add_argument(
-        "-T",
-        action="store_true", dest="transparent_proxy", default=False,
-        help="Set transparent proxy mode."
-    )
     parser.add_argument(
         "-u",
         action="store", dest="stickyauth_filt", default=None, metavar="FILTER",
@@ -228,8 +224,8 @@ def common_options(parser):
     )
     parser.add_argument(
         "-v",
-        action="count", dest="verbose", default=1,
-        help="Increase verbosity. Can be passed multiple times."
+        action="store_const", dest="verbose", default=1, const=2,
+        help="Increase event log verbosity."
     )
     parser.add_argument(
         "-w",
@@ -245,43 +241,116 @@ def common_options(parser):
         "-Z",
         action="store", dest="body_size_limit", default=None,
         metavar="SIZE",
-        help="Byte size limit of HTTP request and response bodies."\
+        help="Byte size limit of HTTP request and response bodies." \
              " Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
     )
     parser.add_argument(
-        "--host",
-        action="store_true", dest="showhost", default=False,
-        help="Use the Host header to construct URLs for display."
+        "--stream",
+        action="store", dest="stream_large_bodies", default=None,
+        metavar="SIZE",
+        help="""
+        Stream data to the client if response body exceeds the given threshold.
+        If streamed, the body will not be stored in any way. Understands k/m/g
+        suffixes, i.e. 3m for 3 megabytes.
+         """
     )
 
-    parser.add_argument(
-        "--no-upstream-cert", default=False,
-        action="store_true", dest="no_upstream_cert",
-        help="Don't connect to upstream server to look up certificate details."
+    group = parser.add_argument_group("Proxy Options")
+    # We could make a mutually exclusive group out of -R, -U, -T, but we don't
+    # do that because  - --upstream-server should be in that group as well, but
+    # it's already in a different group.  - our own error messages are more
+    # helpful
+    group.add_argument(
+        "-b",
+        action="store", type=str, dest="addr", default='',
+        help="Address to bind proxy to (defaults to all interfaces)"
+    )
+    group.add_argument(
+        "-I", "--ignore",
+        action="append", type=str, dest="ignore_hosts", default=[],
+        metavar="HOST",
+        help="""
+            Ignore host and forward all traffic without processing it. In
+            transparent mode, it is recommended to use an IP address (range),
+            not the hostname. In regular mode, only SSL traffic is ignored and
+            the hostname should be used. The supplied value is interpreted as a
+            regular expression and matched on the ip or the hostname. Can be
+            passed multiple times.
+        """
+    )
+    group.add_argument(
+        "--tcp",
+        action="append", type=str, dest="tcp_hosts", default=[],
+        metavar="HOST",
+        help="Generic TCP SSL proxy mode for all hosts that match the pattern. Similar to --ignore,"
+             "but SSL connections are intercepted. The communication contents are printed to the event log in verbose mode."
+    )
+    group.add_argument(
+        "-n",
+        action="store_true", dest="no_server",
+        help="Don't start a proxy server."
+    )
+    group.add_argument(
+        "-p",
+        action="store", type=int, dest="port", default=8080,
+        help="Proxy service port."
+    )
+    group.add_argument(
+        "-R",
+        action="store", type=parse_server_spec, dest="reverse_proxy", default=None,
+        help="Forward all requests to upstream HTTP server: http[s][2http[s]]://host[:port]"
+    )
+    group.add_argument(
+        "--socks",
+        action="store_true", dest="socks_proxy", default=False,
+        help="Set SOCKS5 proxy mode."
+    )
+    group.add_argument(
+        "-T",
+        action="store_true", dest="transparent_proxy", default=False,
+        help="Set transparent proxy mode."
+    )
+    group.add_argument(
+        "-U",
+        action="store", type=parse_server_spec, dest="upstream_proxy", default=None,
+        help="Forward all requests to upstream proxy server: http://host[:port]"
     )
 
-    group = parser.add_argument_group("Web App")
+    group = parser.add_argument_group(
+        "Advanced Proxy Options",
+        """
+            The following options allow a custom adjustment of the proxy behavior.
+            Normally, you don't want to use these options directly and use the provided wrappers instead (-R, -U, -T).
+        """.strip()
+    )
+    group.add_argument(
+        "--http-form-in", dest="http_form_in", default=None,
+        action="store", choices=("relative", "absolute"),
+        help="Override the HTTP request form accepted by the proxy"
+    )
+    group.add_argument(
+        "--http-form-out", dest="http_form_out", default=None,
+        action="store", choices=("relative", "absolute"),
+        help="Override the HTTP request form sent upstream by the proxy"
+    )
+
+    group = parser.add_argument_group("Onboarding App")
     group.add_argument(
         "-a",
         action="store_false", dest="app", default=True,
-        help="Disable the mitmproxy web app."
+        help="Disable the mitmproxy onboarding app."
     )
     group.add_argument(
         "--app-host",
         action="store", dest="app_host", default=APP_HOST, metavar="host",
-        help="Domain to serve the app from. For transparent mode, use an IP when\
-                a DNS entry for the app domain is not present. Default: %s"%APP_HOST
+        help="Domain to serve the onboarding app from. For transparent mode, use an IP when\
+                a DNS entry for the app domain is not present. Default: %s" % APP_HOST
 
     )
     group.add_argument(
         "--app-port",
         action="store", dest="app_port", default=APP_PORT, type=int, metavar="80",
-        help="Port to serve the app from."
-    )
-    group.add_argument(
-        "--app-external",
-        action="store_true", dest="app_external",
-        help="Serve the app outside of the proxy."
+        help="Port to serve the onboarding app from."
     )
 
     group = parser.add_argument_group("Client Replay")
@@ -306,19 +375,30 @@ def common_options(parser):
         "--rheader",
         action="append", dest="rheaders", type=str,
         help="Request headers to be considered during replay. "
-           "Can be passed multiple times."
+             "Can be passed multiple times."
     )
     group.add_argument(
         "--norefresh",
         action="store_true", dest="norefresh", default=False,
-        help= "Disable response refresh, "
-        "which updates times in cookies and headers for replayed responses."
+        help="Disable response refresh, "
+             "which updates times in cookies and headers for replayed responses."
     )
     group.add_argument(
         "--no-pop",
         action="store_true", dest="nopop", default=False,
         help="Disable response pop from response flow. "
-        "This makes it possible to replay same response multiple times."
+             "This makes it possible to replay same response multiple times."
+    )
+    group.add_argument(
+        "--replay-ignore-content",
+        action="store_true", dest="replay_ignore_content", default=False,
+        help="Ignore request's content while searching for a saved flow to replay"
+    )
+    group.add_argument(
+        "--replay-ignore-param",
+        action="append", dest="replay_ignore_params", type=str,
+        help="Request's parameters to be ignored while searching for a saved flow to replay"
+           "Can be passed multiple times."
     )
 
     group = parser.add_argument_group(
@@ -342,7 +422,6 @@ def common_options(parser):
         help="Replacement pattern, where the replacement clause is a path to a file."
     )
 
-
     group = parser.add_argument_group(
         "Set Headers",
         """
@@ -358,13 +437,11 @@ def common_options(parser):
         help="Header set pattern."
     )
 
-
     group = parser.add_argument_group(
         "Proxy Authentication",
         """
             Specify which users are allowed to access the proxy and the method
-            used for authenticating them. These options are ignored if the
-            proxy is in transparent or reverse proxy mode.
+            used for authenticating them.
         """
     )
     user_specification_group = group.add_mutually_exclusive_group()
@@ -382,9 +459,9 @@ def common_options(parser):
     )
     user_specification_group.add_argument(
         "--htpasswd",
-        action="store", dest="auth_htpasswd", type=argparse.FileType('r'),
+        action="store", dest="auth_htpasswd", type=str,
         metavar="PATH",
         help="Allow access to users specified in an Apache htpasswd file."
     )
 
-    proxy.certificate_option_group(parser)
+    config.ssl_option_group(parser)

libmproxy/console/__init__.py

@@ -1,8 +1,9 @@
+from __future__ import absolute_import
 import mailcap, mimetypes, tempfile, os, subprocess, glob, time, shlex, stat
-import os.path, sys, weakref
+import os.path, sys, weakref, traceback
 import urwid
-from .. import controller, utils, flow, script
-import flowlist, flowview, help, common, grideditor, palettes, contentview, flowdetailview
+from .. import controller, utils, flow, script, proxy
+from . import flowlist, flowview, help, common, grideditor, palettes, contentview, flowdetailview
 
 EVENTLOG_SIZE = 500
 
@@ -77,7 +78,6 @@ class PathEdit(urwid.Edit, _PathCompleter):
 class ActionBar(common.WWrap):
     def __init__(self):
         self.message("")
-        self.expire = None
 
     def selectable(self):
         return True
@@ -129,6 +129,14 @@ class StatusBar(common.WWrap):
                 r.append(":%s in file]"%self.master.server_playback.count())
             else:
                 r.append(":%s to go]"%self.master.server_playback.count())
+        if self.master.get_ignore_filter():
+            r.append("[")
+            r.append(("heading_key", "I"))
+            r.append("gnore:%d]" % len(self.master.get_ignore_filter()))
+        if self.master.get_tcp_filter():
+            r.append("[")
+            r.append(("heading_key", "T"))
+            r.append("CP:%d]" % len(self.master.get_tcp_filter()))
         if self.master.state.intercept_txt:
             r.append("[")
             r.append(("heading_key", "i"))
@@ -145,10 +153,6 @@ class StatusBar(common.WWrap):
             r.append("[")
             r.append(("heading_key", "u"))
             r.append(":%s]"%self.master.stickyauth_txt)
-        if self.master.server.config.reverse_proxy:
-            r.append("[")
-            r.append(("heading_key", "P"))
-            r.append(":%s]"%utils.unparse_url(*self.master.server.config.reverse_proxy))
         if self.master.state.default_body_view.name != "Auto":
             r.append("[")
             r.append(("heading_key", "M"))
@@ -169,14 +173,23 @@ class StatusBar(common.WWrap):
             opts.append("no-upstream-cert")
         if self.master.state.follow_focus:
             opts.append("following")
+        if self.master.stream_large_bodies:
+            opts.append("stream:%s" % utils.pretty_size(self.master.stream_large_bodies.max_size))
 
         if opts:
             r.append("[%s]"%(":".join(opts)))
 
+        if self.master.server.config.mode in ["reverse", "upstream"]:
+            dst = self.master.server.config.mode.dst
+            scheme = "https" if dst[0] else "http"
+            if dst[1] != dst[0]:
+                scheme += "2https" if dst[1] else "http"
+            r.append("[dest:%s]"%utils.unparse_url(scheme, *dst[2:]))
         if self.master.scripts:
-            r.append("[scripts:%s]"%len(self.master.scripts))
-        if self.master.debug:
-            r.append("[lt:%0.3f]"%self.master.looptime)
+            r.append("[")
+            r.append(("heading_key", "s"))
+            r.append("cripts:%s]"%len(self.master.scripts))
+        # r.append("[lt:%0.3f]"%self.master.looptime)
 
         if self.master.stream:
             r.append("[W:%s]"%self.master.stream_path)
@@ -197,7 +210,10 @@ class StatusBar(common.WWrap):
         ]
 
         if self.master.server.bound:
-            boundaddr = "[%s:%s]"%(self.master.server.address or "*", self.master.server.port)
+            host = self.master.server.address.host
+            if host == "0.0.0.0":
+                host = "*"
+            boundaddr = "[%s:%s]"%(host, self.master.server.address.port)
         else:
             boundaddr = ""
         t.extend(self.get_status())
@@ -261,8 +277,8 @@ class ConsoleState(flow.State):
         d = self.flowsettings.get(flow, {})
         return d.get(key, default)
 
-    def add_request(self, req):
-        f = flow.State.add_request(self, req)
+    def add_request(self, f):
+        flow.State.add_request(self, f)
         if self.focus is None:
             self.set_focus(0)
         elif self.follow_focus:
@@ -326,7 +342,6 @@ class Options(object):
         "anticache",
         "anticomp",
         "client_replay",
-        "debug",
         "eventlog",
         "keepserving",
         "kill",
@@ -342,6 +357,7 @@ class Options(object):
         "server_replay",
         "stickycookie",
         "stickyauth",
+        "stream_large_bodies",
         "verbosity",
         "wfile",
         "nopop",
@@ -390,6 +406,8 @@ class ConsoleMaster(flow.FlowMaster):
             print >> sys.stderr, "Sticky auth error:", r
             sys.exit(1)
 
+        self.set_stream_large_bodies(options.stream_large_bodies)
+
         self.refresh_server_playback = options.refresh_server_playback
         self.anticache = options.anticache
         self.anticomp = options.anticomp
@@ -407,8 +425,6 @@ class ConsoleMaster(flow.FlowMaster):
         if options.server_replay:
             self.server_playback_path(options.server_replay)
 
-        self.debug = options.debug
-
         if options.scripts:
             for i in options.scripts:
                 err = self.load_script(i)
@@ -423,7 +439,7 @@ class ConsoleMaster(flow.FlowMaster):
                 sys.exit(1)
 
         if options.app:
-            self.start_app(self.options.app_host, self.options.app_port, self.options.app_external)
+            self.start_app(self.options.app_host, self.options.app_port)
 
     def start_stream(self, path):
         path = os.path.expanduser(path)
@@ -438,20 +454,20 @@ class ConsoleMaster(flow.FlowMaster):
         status, val = s.run(method, f)
         if val:
             if status:
-                self.add_event("Method %s return: %s"%(method, val))
+                self.add_event("Method %s return: %s"%(method, val), "debug")
             else:
-                self.add_event("Method %s error: %s"%(method, val[1]))
+                self.add_event("Method %s error: %s"%(method, val[1]), "error")
 
     def run_script_once(self, command, f):
         if not command:
             return
-        self.add_event("Running script on flow: %s"%command)
+        self.add_event("Running script on flow: %s"%command, "debug")
 
         try:
             s = script.Script(command, self)
         except script.ScriptError, v:
             self.statusbar.message("Error loading script.")
-            self.add_event("Error loading script:\n%s"%v.args[0])
+            self.add_event("Error loading script:\n%s"%v.args[0], "error")
             return
 
         if f.request:
@@ -500,7 +516,8 @@ class ConsoleMaster(flow.FlowMaster):
             self.start_server_playback(
                 ret,
                 self.killextra, self.rheaders,
-                False, self.nopop
+                False, self.nopop,
+                self.options.replay_ignore_params, self.options.replay_ignore_content
             )
 
     def spawn_editor(self, data):
@@ -568,31 +585,35 @@ class ConsoleMaster(flow.FlowMaster):
         self.ui.set_terminal_properties(256)
         self.ui.register_palette(self.palette)
         self.flow_list_walker = flowlist.FlowListWalker(self, self.state)
-
         self.view = None
         self.statusbar = None
         self.header = None
         self.body = None
         self.help_context = None
-
         self.prompting = False
         self.onekey = False
 
         self.view_flowlist()
 
-        self.server.start_slave(controller.Slave, controller.Channel(self.masterq))
+        self.server.start_slave(controller.Slave, controller.Channel(self.masterq, self.should_exit))
 
         if self.options.rfile:
             ret = self.load_flows(self.options.rfile)
             if ret and self.state.flow_count():
-                self.add_event("File truncated or corrupted. Loaded as many flows as possible.")
+                self.add_event("File truncated or corrupted. Loaded as many flows as possible.","error")
             elif not self.state.flow_count():
                 self.shutdown()
                 print >> sys.stderr, "Could not load file:", ret
                 sys.exit(1)
 
-        self.ui.run_wrapper(self.loop)
-        # If True, quit just pops out to flow list view.
+        try:
+            self.ui.run_wrapper(self.loop)
+        except Exception:
+            self.ui.stop()
+            sys.stdout.flush()
+            print >> sys.stderr, traceback.format_exc()
+            print >> sys.stderr, "mitmproxy has crashed!"
+            print >> sys.stderr, "Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy"
         print >> sys.stderr, "Shutting down..."
         sys.stderr.flush()
         self.shutdown()
@@ -649,7 +670,6 @@ class ConsoleMaster(flow.FlowMaster):
         self.statusbar = StatusBar(self, flowview.footer)
         self.state.set_focus_flow(flow)
         self.state.view_mode = common.VIEW_FLOW
-
         self.make_view()
         self.help_context = flowview.help_context
 
@@ -762,15 +782,6 @@ class ConsoleMaster(flow.FlowMaster):
         self.state.default_body_view = v
         self.refresh_focus()
 
-    def set_reverse_proxy(self, txt):
-        if not txt:
-            self.server.config.reverse_proxy = None
-        else:
-            s = utils.parse_proxy_spec(txt)
-            if not s:
-                return "Invalid reverse proxy specification"
-            self.server.config.reverse_proxy = s
-
     def drawscreen(self):
         size = self.ui.get_cols_rows()
         canvas = self.view.render(size, focus=1)
@@ -783,19 +794,33 @@ class ConsoleMaster(flow.FlowMaster):
         else:
             self.view_flowlist()
 
-    def edit_scripts(self, *args, **kwargs):
-        pass
+    def edit_scripts(self, scripts):
+        commands = [x[0] for x in scripts]  # remove outer array
+        if commands == [s.command for s in self.scripts]:
+            return
+
+        self.unload_scripts()
+        for command in commands:
+            self.load_script(command)
+
+    def edit_ignore_filter(self, ignore):
+        patterns = (x[0] for x in ignore)
+        self.set_ignore_filter(patterns)
+
+    def edit_tcp_filter(self, tcp):
+        patterns = (x[0] for x in tcp)
+        self.set_tcp_filter(patterns)
 
     def loop(self):
         changed = True
         try:
-            while not controller.should_exit:
+            while not self.should_exit.is_set():
                 startloop = time.time()
                 if changed:
                     self.statusbar.redraw()
                     size = self.drawscreen()
-                changed = self.tick(self.masterq)
-                self.ui.set_input_timeouts(max_wait=0.1)
+                changed = self.tick(self.masterq, 0.01)
+                self.ui.set_input_timeouts(max_wait=0.01)
                 keys = self.ui.get_input()
                 if keys:
                     changed = True
@@ -842,6 +867,22 @@ class ConsoleMaster(flow.FlowMaster):
                                         self.setheaders.set
                                     )
                                 )
+                            elif k == "I":
+                                self.view_grideditor(
+                                    grideditor.HostPatternEditor(
+                                        self,
+                                        [[x] for x in self.get_ignore_filter()],
+                                        self.edit_ignore_filter
+                                    )
+                                )
+                            elif k == "T":
+                                self.view_grideditor(
+                                    grideditor.HostPatternEditor(
+                                        self,
+                                        [[x] for x in self.get_tcp_filter()],
+                                        self.edit_tcp_filter
+                                    )
+                                )
                             elif k == "i":
                                 self.prompt(
                                     "Intercept filter: ",
@@ -865,16 +906,6 @@ class ConsoleMaster(flow.FlowMaster):
                                     contentview.view_prompts,
                                     self.change_default_display_mode
                                 )
-                            elif k == "P":
-                                if self.server.config.reverse_proxy:
-                                    p = utils.unparse_url(*self.server.config.reverse_proxy)
-                                else:
-                                    p = ""
-                                self.prompt(
-                                    "Reverse proxy: ",
-                                    p,
-                                    self.set_reverse_proxy
-                                )
                             elif k == "R":
                                 self.view_grideditor(
                                     grideditor.ReplaceEditor(
@@ -887,7 +918,7 @@ class ConsoleMaster(flow.FlowMaster):
                                 self.view_grideditor(
                                     grideditor.ScriptEditor(
                                         self,
-                                        [[i.argv[0]] for i in self.scripts],
+                                        [[i.command] for i in self.scripts],
                                         self.edit_scripts
                                     )
                                 )
@@ -1007,11 +1038,11 @@ class ConsoleMaster(flow.FlowMaster):
         if hasattr(self.statusbar, "refresh_flow"):
             self.statusbar.refresh_flow(c)
 
-    def process_flow(self, f, r):
-        if self.state.intercept and f.match(self.state.intercept) and not f.request.is_replay():
+    def process_flow(self, f):
+        if self.state.intercept and f.match(self.state.intercept) and not f.request.is_replay:
             f.intercept()
         else:
-            r.reply()
+            f.reply()
         self.sync_list_view()
         self.refresh_flow(f)
 
@@ -1019,35 +1050,34 @@ class ConsoleMaster(flow.FlowMaster):
         self.eventlist[:] = []
 
     def add_event(self, e, level="info"):
-        if level == "info":
-            e = urwid.Text(str(e))
-        elif level == "error":
-            e = urwid.Text(("error", str(e)))
+        needed = dict(error=0, info=1, debug=2).get(level, 1)
+        if self.options.verbosity < needed:
+            return
 
+        if level == "error":
+            e = urwid.Text(("error", str(e)))
+        else:
+            e = urwid.Text(str(e))
         self.eventlist.append(e)
         if len(self.eventlist) > EVENTLOG_SIZE:
             self.eventlist.pop(0)
         self.eventlist.set_focus(len(self.eventlist)-1)
 
     # Handlers
-    def handle_log(self, l):
-        self.add_event(l.msg)
-        l.reply()
-
-    def handle_error(self, r):
-        f = flow.FlowMaster.handle_error(self, r)
+    def handle_error(self, f):
+        f = flow.FlowMaster.handle_error(self, f)
         if f:
-            self.process_flow(f, r)
+            self.process_flow(f)
         return f
 
-    def handle_request(self, r):
-        f = flow.FlowMaster.handle_request(self, r)
+    def handle_request(self, f):
+        f = flow.FlowMaster.handle_request(self, f)
         if f:
-            self.process_flow(f, r)
+            self.process_flow(f)
         return f
 
-    def handle_response(self, r):
-        f = flow.FlowMaster.handle_response(self, r)
+    def handle_response(self, f):
+        f = flow.FlowMaster.handle_response(self, f)
         if f:
-            self.process_flow(f, r)
+            self.process_flow(f)
         return f

libmproxy/console/common.py

@@ -1,6 +1,8 @@
+from __future__ import absolute_import
 import urwid
 import urwid.util
-from .. import utils, flow
+from .. import utils
+from ..protocol.http import CONTENT_MISSING
 
 
 VIEW_LIST = 0
@@ -106,7 +108,7 @@ def raw_format_flow(f, focus, extended, padding):
 
     preamble = sum(i[1] for i in req) + len(req) -1
 
-    if f["intercepting"] and not f["req_acked"]:
+    if f["intercepting"] and not f["acked"]:
         uc = "intercept"
     elif f["resp_code"] or f["err_msg"]:
         uc = "text"
@@ -136,7 +138,7 @@ def raw_format_flow(f, focus, extended, padding):
         if f["resp_is_replay"]:
             resp.append(fcol(SYMBOL_REPLAY, "replay"))
         resp.append(fcol(f["resp_code"], ccol))
-        if f["intercepting"] and f["resp_code"] and not f["resp_acked"]:
+        if f["intercepting"] and f["resp_code"] and not f["acked"]:
             rc = "intercept"
         else:
             rc = "text"
@@ -170,12 +172,12 @@ flowcache = FlowCache()
 def format_flow(f, focus, extended=False, hostheader=False, padding=2):
     d = dict(
         intercepting = f.intercepting,
+        acked = f.reply.acked,
 
         req_timestamp = f.request.timestamp_start,
-        req_is_replay = f.request.is_replay(),
+        req_is_replay = f.request.is_replay,
         req_method = f.request.method,
-        req_acked = f.request.reply.acked,
-        req_url = f.request.get_url(hostheader=hostheader),
+        req_url = f.request.pretty_url(hostheader=hostheader),
 
         err_msg = f.error.msg if f.error else None,
         resp_code = f.response.code if f.response else None,
@@ -183,19 +185,21 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
     if f.response:
         if f.response.content:
             contentdesc = utils.pretty_size(len(f.response.content))
-        elif f.response.content == flow.CONTENT_MISSING:
+        elif f.response.content == CONTENT_MISSING:
             contentdesc = "[content missing]"
         else:
             contentdesc = "[no content]"
 
-        delta = f.response.timestamp_end - f.response.timestamp_start
-        size = len(f.response.content) + f.response.get_header_size()
+        if f.response.timestamp_end:
+            delta = f.response.timestamp_end - f.response.timestamp_start
+        else:
+            delta = 0
+        size = f.response.size()
         rate = utils.pretty_size(size / ( delta if delta > 0 else 1 ) )
 
         d.update(dict(
             resp_code = f.response.code,
-            resp_is_replay = f.response.is_replay(),
-            resp_acked = f.response.reply.acked,
+            resp_is_replay = f.response.is_replay,
             resp_clen = contentdesc,
             resp_rate = "{0}/s".format(rate),
         ))

libmproxy/console/contentview.py

@@ -1,16 +1,14 @@
-import logging
-import re, cStringIO, traceback, json
-import urwid
-
+from __future__ import absolute_import
+import logging, subprocess, re, cStringIO, traceback, json, urwid
 from PIL import Image
 from PIL.ExifTags import TAGS
 
 import lxml.html, lxml.etree
 import netlib.utils
-import common
+from . import common
 from .. import utils, encoding, flow
 from ..contrib import jsbeautifier, html2text
-import subprocess
+from ..contrib.wbxml.ASCommandResponse import ASCommandResponse
 try:
     import pyamf
     from pyamf import remoting, flex
@@ -429,12 +427,31 @@ class ViewProtobuf:
         txt = _view_text(decoded[:limit], len(decoded), limit)
         return "Protobuf", txt
 
+class ViewWBXML:
+    name = "WBXML"
+    prompt = ("wbxml", "w")
+    content_types = [
+        "application/vnd.wap.wbxml",
+        "application/vnd.ms-sync.wbxml"
+    ]
+
+    def __call__(self, hdrs, content, limit):
+        
+        try:
+            parser = ASCommandResponse(content)
+            parsedContent = parser.xmlString
+            txt = _view_text(parsedContent, len(parsedContent), limit)
+            return "WBXML", txt
+        except:
+        	return None
+
 views = [
     ViewAuto(),
     ViewRaw(),
     ViewHex(),
     ViewJSON(),
     ViewXML(),
+    ViewWBXML(),
     ViewHTML(),
     ViewHTMLOutline(),
     ViewJavaScript(),
@@ -493,7 +510,7 @@ def get_content_view(viewmode, hdrItems, content, limit, logfunc):
     except Exception:
         s = traceback.format_exc()
         s = "Content viewer failed: \n"  + s
-        logfunc(s)
+        logfunc(s, "error")
         ret = None
     if not ret:
         ret = get("Raw")(hdrs, content, limit)

libmproxy/console/flowdetailview.py

@@ -1,5 +1,7 @@
+from __future__ import absolute_import
 import urwid
-import common
+from . import common
+from .. import utils
 
 footer = [
     ('heading_key', "q"), ":back ",
@@ -33,8 +35,17 @@ class FlowDetailsView(urwid.ListBox):
         title = urwid.AttrWrap(title, "heading")
         text.append(title)
 
-        if self.flow.response:
-            c = self.flow.response.cert
+        if self.flow.server_conn:
+            text.append(urwid.Text([("head", "Server Connection:")]))
+            sc = self.flow.server_conn
+            parts = [
+                ["Address", "%s:%s" % sc.address()],
+                ["Start time", utils.format_timestamp(sc.timestamp_start)],
+                ["End time", utils.format_timestamp(sc.timestamp_end) if sc.timestamp_end else "active"],
+            ]
+            text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
+
+            c = self.flow.server_conn.cert
             if c:
                 text.append(urwid.Text([("head", "Server Certificate:")]))
                 parts = [
@@ -43,19 +54,13 @@ class FlowDetailsView(urwid.ListBox):
                     ["Valid to", str(c.notafter)],
                     ["Valid from", str(c.notbefore)],
                     ["Serial", str(c.serial)],
-                ]
-
-                parts.append(
                     [
                         "Subject",
                         urwid.BoxAdapter(
                             urwid.ListBox(common.format_keyvals(c.subject, key="highlight", val="text")),
                             len(c.subject)
                         )
-                    ]
-                )
-
-                parts.append(
+                    ],
                     [
                         "Issuer",
                         urwid.BoxAdapter(
@@ -63,7 +68,7 @@ class FlowDetailsView(urwid.ListBox):
                             len(c.issuer)
                         )
                     ]
-                )
+                ]
 
                 if c.altnames:
                     parts.append(
@@ -74,13 +79,14 @@ class FlowDetailsView(urwid.ListBox):
                     )
                 text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
 
-        if self.flow.request.client_conn:
+        if self.flow.client_conn:
             text.append(urwid.Text([("head", "Client Connection:")]))
-            cc = self.flow.request.client_conn
+            cc = self.flow.client_conn
             parts = [
-                ["Address", "%s:%s"%tuple(cc.address)],
-                ["Requests", "%s"%cc.requestcount],
-                ["Closed", "%s"%cc.close],
+                ["Address", "%s:%s" % cc.address()],
+                ["Start time", utils.format_timestamp(cc.timestamp_start)],
+                # ["Requests", "%s"%cc.requestcount],
+                ["End time", utils.format_timestamp(cc.timestamp_end) if cc.timestamp_end else "active"],
             ]
             text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
 

libmproxy/console/flowlist.py

@@ -1,5 +1,6 @@
+from __future__ import absolute_import
 import urwid
-import common
+from . import common
 
 def _mkhelp():
     text = []
@@ -68,7 +69,7 @@ class BodyPile(urwid.Pile):
             else:
                 self.widget_list[1].header = self.inactive_header
             key = None
-        elif key == "v":
+        elif key == "e":
             self.master.toggle_eventlog()
             key = None
 
@@ -119,13 +120,15 @@ class ConnectionItem(common.WWrap):
             self.master.start_server_playback(
                 [i.copy() for i in self.master.state.view],
                 self.master.killextra, self.master.rheaders,
-                False, self.master.nopop
+                False, self.master.nopop,
+                self.master.options.replay_ignore_params, self.master.options.replay_ignore_content
             )
         elif k == "t":
             self.master.start_server_playback(
                 [self.flow.copy()],
                 self.master.killextra, self.master.rheaders,
-                False, self.master.nopop
+                False, self.master.nopop,
+                self.master.options.replay_ignore_params, self.master.options.replay_ignore_content
             )
         else:
             self.master.path_prompt(

libmproxy/console/flowview.py

@@ -1,7 +1,9 @@
+from __future__ import absolute_import
 import os, sys, copy
 import urwid
-import common, grideditor, contentview
+from . import common, grideditor, contentview
 from .. import utils, flow, controller
+from ..protocol.http import HTTPResponse, CONTENT_MISSING
 
 
 class SearchError(Exception): pass
@@ -68,7 +70,8 @@ def _mkhelp():
         ("space", "next flow"),
         ("|", "run script on this flow"),
         ("/", "search in response body (case sensitive)"),
-        ("n", "repeat previous search"),
+        ("n", "repeat search forward"),
+        ("N", "repeat search backwards"),
     ]
     text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
     return text
@@ -149,8 +152,8 @@ class FlowView(common.WWrap):
         return (description, text_objects)
 
     def cont_view_handle_missing(self, conn, viewmode):
-            if conn.content == flow.CONTENT_MISSING:
-                msg, body = "", [urwid.Text([("error", "[content missing]")])], 0
+            if conn.content == CONTENT_MISSING:
+                msg, body = "", [urwid.Text([("error", "[content missing]")])]
             else:
                 msg, body = self.content_view(viewmode, conn)
 
@@ -173,12 +176,9 @@ class FlowView(common.WWrap):
                 key = "header",
                 val = "text"
             )
-        if conn.content is not None:
-            override = self.override_get()
-            viewmode = self.viewmode_get(override)
-            msg, body = self.cont_view_handle_missing(conn, viewmode)
-        elif conn.content == flow.CONTENT_MISSING:
-            pass
+        override = self.override_get()
+        viewmode = self.viewmode_get(override)
+        msg, body = self.cont_view_handle_missing(conn, viewmode)
         return headers, msg, body
 
     def conn_text_merge(self, headers, msg, body):
@@ -230,7 +230,7 @@ class FlowView(common.WWrap):
     def wrap_body(self, active, body):
         parts = []
 
-        if self.flow.intercepting and not self.flow.request.reply.acked:
+        if self.flow.intercepting and not self.flow.reply.acked and not self.flow.response:
             qt = "Request intercepted"
         else:
             qt = "Request"
@@ -239,7 +239,7 @@ class FlowView(common.WWrap):
         else:
             parts.append(self._tab(qt, "heading_inactive"))
 
-        if self.flow.intercepting and self.flow.response and not self.flow.response.reply.acked:
+        if self.flow.intercepting and not self.flow.reply.acked and self.flow.response:
             st = "Response intercepted"
         else:
             st = "Response"
@@ -255,7 +255,7 @@ class FlowView(common.WWrap):
                 )
         return f
 
-    def search_wrapped_around(self, last_find_line, last_search_index):
+    def search_wrapped_around(self, last_find_line, last_search_index, backwards):
         """
             returns true if search wrapped around the bottom.
         """
@@ -265,24 +265,50 @@ class FlowView(common.WWrap):
         current_search_index = self.state.get_flow_setting(self.flow,
                 "last_search_index")
 
-        if current_find_line <= last_find_line:
-            return True
-        elif current_find_line == last_find_line:
-            if current_search_index <= last_search_index:
-                return True
+        if not backwards:
+            message = "search hit BOTTOM, continuing at TOP"
+            if current_find_line <= last_find_line:
+                return True, message
+            elif current_find_line == last_find_line:
+                if current_search_index <= last_search_index:
+                    return True, message
+        else:
+            message = "search hit TOP, continuing at BOTTOM"
+            if current_find_line >= last_find_line:
+                return True, message
+            elif current_find_line == last_find_line:
+                if current_search_index >= last_search_index:
+                    return True, message
 
-        return False
+        return False, ""
 
-    def search(self, search_string):
+    def search_again(self, backwards=False):
+        """
+            runs the previous search again, forwards or backwards.
+        """
+        last_search_string = self.state.get_flow_setting(self.flow, "last_search_string")
+        if last_search_string:
+            message = self.search(last_search_string, backwards)
+            if message:
+                self.master.statusbar.message(message)
+        else:
+            message = "no previous searches have been made"
+            self.master.statusbar.message(message)
+
+        return message
+
+    def search(self, search_string, backwards=False):
         """
             similar to view_response or view_request, but instead of just
             displaying the conn, it highlights a word that the user is
             searching for and handles all the logic surrounding that.
         """
 
-        if search_string == "":
+        if not search_string:
             search_string = self.state.get_flow_setting(self.flow,
                     "last_search_string")
+            if not search_string:
+                return
 
         if self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
             text = self.flow.request
@@ -301,7 +327,7 @@ class FlowView(common.WWrap):
         # generate the body, highlight the words and get focus
         headers, msg, body = self.conn_text_raw(text)
         try:
-            body, focus_position = self.search_highlight_text(body, search_string)
+            body, focus_position = self.search_highlight_text(body, search_string, backwards=backwards)
         except SearchError:
             return "Search not supported in this view."
 
@@ -318,8 +344,10 @@ class FlowView(common.WWrap):
 
         self.last_displayed_body = list_box
 
-        if self.search_wrapped_around(last_find_line, last_search_index):
-            return "search hit BOTTOM, continuing at TOP"
+        wrapped, wrapped_message = self.search_wrapped_around(last_find_line, last_search_index, backwards)
+
+        if wrapped:
+            return wrapped_message
 
     def search_get_start(self, search_string):
         start_line = 0
@@ -344,57 +372,94 @@ class FlowView(common.WWrap):
 
         return (start_line, start_index)
 
-    def search_highlight_text(self, text_objects, search_string, looping = False):
+    def search_get_range(self, len_text_objects, start_line, backwards):
+        if not backwards:
+            loop_range = xrange(start_line, len_text_objects)
+        else:
+            loop_range = xrange(start_line, -1, -1)
+
+        return loop_range
+
+    def search_find(self, text, search_string, start_index, backwards):
+            if backwards == False:
+                find_index = text.find(search_string, start_index)
+            else:
+                if start_index != 0:
+                    start_index -= len(search_string)
+                else:
+                    start_index = None
+
+                find_index = text.rfind(search_string, 0, start_index)
+
+            return find_index
+
+    def search_highlight_text(self, text_objects, search_string, looping = False, backwards = False):
         start_line, start_index = self.search_get_start(search_string)
         i = start_line
 
         found = False
         text_objects = copy.deepcopy(text_objects)
-        for text_object in text_objects[start_line:]:
-            if i != start_line:
-                start_index = 0
+        loop_range = self.search_get_range(len(text_objects), start_line, backwards)
+        for i in loop_range:
+            text_object = text_objects[i]
 
             try:
                 text, style = text_object.get_text()
             except AttributeError:
                 raise SearchError()
-            find_index = text.find(search_string, start_index)
-            if find_index != -1:
-                before = text[:find_index]
-                after = text[find_index+len(search_string):]
-                new_text = urwid.Text(
-                    [
-                        before,
-                        (self.highlight_color, search_string),
-                        after,
-                    ]
-                )
 
+            if i != start_line:
+                start_index = 0
+
+            find_index = self.search_find(text, search_string, start_index, backwards)
+
+            if find_index != -1:
+                new_text = self.search_highlight_object(text, find_index, search_string)
+                text_objects[i] = new_text
+
+                found = True
                 self.state.add_flow_setting(self.flow, "last_search_index",
                         find_index)
                 self.state.add_flow_setting(self.flow, "last_find_line", i)
 
-                text_objects[i] = new_text
-
-                found = True
-
                 break
 
-            i += 1
-
+        # handle search WRAP
         if found:
             focus_pos = i
         else :
-            # loop from the beginning, but not forever.
-            if (start_line == 0 and start_index == 0) or looping:
+            if looping:
                 focus_pos = None
             else:
-                self.state.add_flow_setting(self.flow, "last_search_index", 0)
-                self.state.add_flow_setting(self.flow, "last_find_line", 0)
-                text_objects, focus_pos = self.search_highlight_text(text_objects, search_string, True)
+                if not backwards:
+                    self.state.add_flow_setting(self.flow, "last_search_index", 0)
+                    self.state.add_flow_setting(self.flow, "last_find_line", 0)
+                else:
+                    self.state.add_flow_setting(self.flow, "last_search_index", None)
+                    self.state.add_flow_setting(self.flow, "last_find_line", len(text_objects) - 1)
+
+                text_objects, focus_pos = self.search_highlight_text(text_objects,
+                        search_string, looping=True, backwards=backwards)
 
         return text_objects, focus_pos
 
+    def search_highlight_object(self, text_object, find_index, search_string):
+        """
+            just a little abstraction
+        """
+        before = text_object[:find_index]
+        after = text_object[find_index+len(search_string):]
+
+        new_text = urwid.Text(
+            [
+                before,
+                (self.highlight_color, search_string),
+                after,
+            ]
+        )
+
+        return new_text
+
     def view_request(self):
         self.state.view_flow_mode = common.VIEW_FLOW_REQUEST
         body = self.conn_text(self.flow.request)
@@ -460,7 +525,9 @@ class FlowView(common.WWrap):
 
     def set_url(self, url):
         request = self.flow.request
-        if not request.set_url(str(url)):
+        try:
+            request.url = str(url)
+        except ValueError:
             return "Invalid URL."
         self.master.refresh_flow(self.flow)
 
@@ -506,7 +573,7 @@ class FlowView(common.WWrap):
             conn = self.flow.request
         else:
             if not self.flow.response:
-                self.flow.response = flow.Response(
+                self.flow.response = HTTPResponse(
                     self.flow.request,
                     self.flow.request.httpversion,
                     200, "OK", flow.ODictCaseless(), "", None
@@ -540,7 +607,7 @@ class FlowView(common.WWrap):
         elif part == "q":
             self.master.view_grideditor(grideditor.QueryEditor(self.master, conn.get_query().lst, self.set_query, conn))
         elif part == "u" and self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
-            self.master.prompt_edit("URL", conn.get_url(), self.set_url)
+            self.master.prompt_edit("URL", conn.url, self.set_url)
         elif part == "m" and self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
             self.master.prompt_onekey("Method", self.method_options, self.edit_method)
         elif part == "c" and self.state.view_flow_mode == common.VIEW_FLOW_RESPONSE:
@@ -579,7 +646,7 @@ class FlowView(common.WWrap):
 
     def delete_body(self, t):
         if t == "m":
-            val = flow.CONTENT_MISSING
+            val = CONTENT_MISSING
         else:
             val = None
         if self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
@@ -682,7 +749,7 @@ class FlowView(common.WWrap):
             self.master.statusbar.message("")
         elif key == "m":
             p = list(contentview.view_prompts)
-            p.insert(0, ("clear", "c"))
+            p.insert(0, ("Clear", "C"))
             self.master.prompt_onekey(
                 "Display mode",
                 p,
@@ -761,13 +828,9 @@ class FlowView(common.WWrap):
                     None,
                     self.search)
         elif key == "n":
-            last_search_string = self.state.get_flow_setting(self.flow, "last_search_string")
-            if last_search_string:
-                message = self.search(last_search_string)
-                if message:
-                    self.master.statusbar.message(message)
-            else:
-                self.master.statusbar.message("no previous searches have been made")
+            self.search_again(backwards=False)
+        elif key == "N":
+            self.search_again(backwards=True)
         else:
             return key
 

libmproxy/console/grideditor.py

@@ -1,6 +1,7 @@
+from __future__ import absolute_import
 import copy, re, os
 import urwid
-import common
+from . import common
 from .. import utils, filt, script
 from netlib import http_uastrings
 
@@ -128,6 +129,8 @@ class GridWalker(urwid.ListWalker):
         if emsg:
             self.editor.master.statusbar.message(emsg, 1000)
             errors.add(self.focus_col)
+        else:
+            errors.discard(self.focus_col)
 
         row = list(self.lst[self.focus][0])
         row[self.focus_col] = val
@@ -492,3 +495,15 @@ class ScriptEditor(GridEditor):
             script.Script.parse_command(val)
         except script.ScriptError, v:
             return str(v)
+
+
+class HostPatternEditor(GridEditor):
+    title = "Editing host patterns"
+    columns = 1
+    headings = ("Regex (matched on hostname:port / ip:port)",)
+
+    def is_error(self, col, val):
+        try:
+            re.compile(val, re.IGNORECASE)
+        except re.error as e:
+            return "Invalid regex: %s" % str(e)

libmproxy/console/help.py

@@ -1,5 +1,6 @@
+from __future__ import absolute_import
 import urwid
-import common
+from . import common
 from .. import filt, version
 
 footer = [
@@ -23,7 +24,7 @@ class HelpView(urwid.ListBox):
 
         text.append(urwid.Text([("head", "\n\nMovement:\n")]))
         keys = [
-            ("j, k", "up, down"),
+            ("j, k", "down, up"),
             ("h, l", "left, right (in some contexts)"),
             ("space", "page down"),
             ("pg up/down", "page up/down"),
@@ -35,6 +36,7 @@ class HelpView(urwid.ListBox):
         keys = [
             ("c", "client replay"),
             ("H", "edit global header set patterns"),
+            ("I", "set ignore pattern"),
             ("i", "set interception pattern"),
             ("M", "change global default display mode"),
                 (None,
@@ -77,6 +79,10 @@ class HelpView(urwid.ListBox):
                     common.highlight_key("xml", "x") +
                     [("text", ": XML")]
                 ),
+                (None,
+                    common.highlight_key("wbxml", "w") +
+                    [("text", ": WBXML")]
+                ),
                 (None,
                     common.highlight_key("amf", "f") +
                     [("text", ": AMF (requires PyAMF)")]
@@ -109,11 +115,11 @@ class HelpView(urwid.ListBox):
 
             ("q", "quit / return to flow list"),
             ("Q", "quit without confirm prompt"),
-            ("P", "set reverse proxy mode"),
             ("R", "edit replacement patterns"),
-            ("s", "set/unset script"),
+            ("s", "add/remove scripts"),
             ("S", "server replay"),
             ("t", "set sticky cookie expression"),
+            ("T", "set tcp proxying pattern"),
             ("u", "set sticky auth expression"),
         ]
         text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))

libmproxy/contrib/README

@@ -11,5 +11,4 @@ jsbeautifier, git checkout 25/03/12, MIT license
 
 html2text, git checkout 18/08/12, GPLv3
 
-md5crypt, PSF license, http://code.activestate.com/recipes/325204/
-
+WinDivert 1.1.4, LGPL license, http://reqrypt.org/windivert.html

libmproxy/contrib/jsbeautifier/unpackers/__init__.py

@@ -7,7 +7,7 @@
 
 import pkgutil
 import re
-from jsbeautifier.unpackers import evalbased
+from . import evalbased
 
 # NOTE: AT THE MOMENT, IT IS DEACTIVATED FOR YOUR SECURITY: it runs js!
 BLACKLIST = ['jsbeautifier.unpackers.evalbased']

libmproxy/contrib/jsbeautifier/unpackers/myobfuscate.py

@@ -40,7 +40,7 @@ try:
 except ImportError:
     from urllib.parse import unquote
 
-from jsbeautifier.unpackers import UnpackingError
+from . import UnpackingError
 
 PRIORITY = 1
 

libmproxy/contrib/jsbeautifier/unpackers/packer.py

@@ -14,7 +14,7 @@
 
 import re
 import string
-from jsbeautifier.unpackers import UnpackingError
+from . import UnpackingError
 
 PRIORITY = 1
 

libmproxy/contrib/wbxml/ASCommandResponse.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+'''
+@author: David Shaw, david.shaw.aw@gmail.com
+
+Inspired by EAS Inspector for Fiddler
+https://easinspectorforfiddler.codeplex.com
+
+----- The MIT License (MIT) ----- 
+Filename: ASCommandResponse.py
+Copyright (c) 2014, David P. Shaw
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+'''
+from ASWBXML import ASWBXML
+import logging
+
+class ASCommandResponse:
+
+	def __init__(self, response):
+		self.wbxmlBody = response
+		try:
+			if ( len(response) > 0):
+				self.xmlString = self.decodeWBXML(self.wbxmlBody)
+			else:
+				logging.error("Empty WBXML body passed")
+		except Exception as e:
+			logging.error("Error: {0}".format(e.message))
+			self.xmlString = None
+
+	def getWBXMLBytes(self):
+		return self.wbxmlBytes
+	
+	def getXMLString(self):
+		return self.xmlString
+	
+	def decodeWBXML(self, body):
+		self.instance = ASWBXML()
+		self.instance.loadBytes(body)
+		return self.instance.getXml()
+
+if __name__ == "__main__":
+	import os	
+	logging.basicConfig(level=logging.INFO)
+
+	projectDir = os.path.dirname(os.path.realpath("."))
+	samplesDir = os.path.join(projectDir, "Samples/")
+	listOfSamples = os.listdir(samplesDir)
+
+	for filename in listOfSamples:
+		byteWBXML = open(samplesDir + os.sep + filename, "rb").read()
+		
+		logging.info("-"*100)
+		logging.info(filename)
+		logging.info("-"*100)
+		instance = ASCommandResponse(byteWBXML)
+		logging.info(instance.xmlString)
+		

libmproxy/contrib/wbxml/ASWBXML.py

@@ -0,0 +1,903 @@
+#!/usr/bin/env python
+'''
+@author: David Shaw, david.shaw.aw@gmail.com
+
+Inspired by EAS Inspector for Fiddler
+https://easinspectorforfiddler.codeplex.com
+
+----- The MIT License (MIT) ----- 
+Filename: ASWBXML.py
+Copyright (c) 2014, David P. Shaw
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+'''
+import xml.dom.minidom
+import logging
+
+from ASWBXMLCodePage import ASWBXMLCodePage
+from ASWBXMLByteQueue import ASWBXMLByteQueue
+from GlobalTokens import GlobalTokens
+from InvalidDataException import InvalidDataException
+
+class ASWBXML:
+	versionByte = 0x03
+	publicIdentifierByte = 0x01
+	characterSetByte = 0x6A
+	stringTableLengthByte = 0x00
+	
+	def __init__(self):
+		
+		# empty on init
+		self.xmlDoc = xml.dom.minidom.Document()
+		self.currentCodePage = 0
+		self.defaultCodePage = -1
+		
+		# Load up code pages
+		# Currently there are 25 code pages as per MS-ASWBXML
+		self.codePages = []
+
+		# region Code Page Initialization
+		# Code Page 0: AirSync
+		# region AirSync Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "AirSync:"
+		page.xmlns = "airsync"
+
+		page.addToken(0x05, "Sync")
+		page.addToken(0x06, "Responses")
+		page.addToken(0x07, "Add")
+		page.addToken(0x08, "Change")
+		page.addToken(0x09, "Delete")
+		page.addToken(0x0A, "Fetch")
+		page.addToken(0x0B, "SyncKey")
+		page.addToken(0x0C, "ClientId")
+		page.addToken(0x0D, "ServerId")
+		page.addToken(0x0E, "Status")
+		page.addToken(0x0F, "Collection")
+		page.addToken(0x10, "Class")
+		page.addToken(0x12, "CollectionId")
+		page.addToken(0x13, "GetChanges")
+		page.addToken(0x14, "MoreAvailable")
+		page.addToken(0x15, "WindowSize")
+		page.addToken(0x16, "Commands")
+		page.addToken(0x17, "Options")
+		page.addToken(0x18, "FilterType")
+		page.addToken(0x1B, "Conflict")
+		page.addToken(0x1C, "Collections")
+		page.addToken(0x1D, "ApplicationData")
+		page.addToken(0x1E, "DeletesAsMoves")
+		page.addToken(0x20, "Supported")
+		page.addToken(0x21, "SoftDelete")
+		page.addToken(0x22, "MIMESupport")
+		page.addToken(0x23, "MIMETruncation")
+		page.addToken(0x24, "Wait")
+		page.addToken(0x25, "Limit")
+		page.addToken(0x26, "Partial")
+		page.addToken(0x27, "ConversationMode")
+		page.addToken(0x28, "MaxItems")
+		page.addToken(0x29, "HeartbeatInterval")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 1: Contacts
+		# region Contacts Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Contacts:"
+		page.xmlns = "contacts"
+
+		page.addToken(0x05, "Anniversary")
+		page.addToken(0x06, "AssistantName")
+		page.addToken(0x07, "AssistantTelephoneNumber")
+		page.addToken(0x08, "Birthday")
+		page.addToken(0x0C, "Business2PhoneNumber")
+		page.addToken(0x0D, "BusinessCity")
+		page.addToken(0x0E, "BusinessCountry")
+		page.addToken(0x0F, "BusinessPostalCode")
+		page.addToken(0x10, "BusinessState")
+		page.addToken(0x11, "BusinessStreet")
+		page.addToken(0x12, "BusinessFaxNumber")
+		page.addToken(0x13, "BusinessPhoneNumber")
+		page.addToken(0x14, "CarPhoneNumber")
+		page.addToken(0x15, "Categories")
+		page.addToken(0x16, "Category")
+		page.addToken(0x17, "Children")
+		page.addToken(0x18, "Child")
+		page.addToken(0x19, "CompanyName")
+		page.addToken(0x1A, "Department")
+		page.addToken(0x1B, "Email1Address")
+		page.addToken(0x1C, "Email2Address")
+		page.addToken(0x1D, "Email3Address")
+		page.addToken(0x1E, "FileAs")
+		page.addToken(0x1F, "FirstName")
+		page.addToken(0x20, "Home2PhoneNumber")
+		page.addToken(0x21, "HomeCity")
+		page.addToken(0x22, "HomeCountry")
+		page.addToken(0x23, "HomePostalCode")
+		page.addToken(0x24, "HomeState")
+		page.addToken(0x25, "HomeStreet")
+		page.addToken(0x26, "HomeFaxNumber")
+		page.addToken(0x27, "HomePhoneNumber")
+		page.addToken(0x28, "JobTitle")
+		page.addToken(0x29, "LastName")
+		page.addToken(0x2A, "MiddleName")
+		page.addToken(0x2B, "MobilePhoneNumber")
+		page.addToken(0x2C, "OfficeLocation")
+		page.addToken(0x2D, "OtherCity")
+		page.addToken(0x2E, "OtherCountry")
+		page.addToken(0x2F, "OtherPostalCode")
+		page.addToken(0x30, "OtherState")
+		page.addToken(0x31, "OtherStreet")
+		page.addToken(0x32, "PagerNumber")
+		page.addToken(0x33, "RadioPhoneNumber")
+		page.addToken(0x34, "Spouse")
+		page.addToken(0x35, "Suffix")
+		page.addToken(0x36, "Title")
+		page.addToken(0x37, "Webpage")
+		page.addToken(0x38, "YomiCompanyName")
+		page.addToken(0x39, "YomiFirstName")
+		page.addToken(0x3A, "YomiLastName")
+		page.addToken(0x3C, "Picture")
+		page.addToken(0x3D, "Alias")
+		page.addToken(0x3E, "WeightedRank")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 2: Email
+		# region Email Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Email:"
+		page.xmlns = "email"
+
+		page.addToken(0x0F, "DateReceived")
+		page.addToken(0x11, "DisplayTo")
+		page.addToken(0x12, "Importance")
+		page.addToken(0x13, "MessageClass")
+		page.addToken(0x14, "Subject")
+		page.addToken(0x15, "Read")
+		page.addToken(0x16, "To")
+		page.addToken(0x17, "CC")
+		page.addToken(0x18, "From")
+		page.addToken(0x19, "ReplyTo")
+		page.addToken(0x1A, "AllDayEvent")
+		page.addToken(0x1B, "Categories")
+		page.addToken(0x1C, "Category")
+		page.addToken(0x1D, "DTStamp")
+		page.addToken(0x1E, "EndTime")
+		page.addToken(0x1F, "InstanceType")
+		page.addToken(0x20, "BusyStatus")
+		page.addToken(0x21, "Location")
+		page.addToken(0x22, "MeetingRequest")
+		page.addToken(0x23, "Organizer")
+		page.addToken(0x24, "RecurrenceId")
+		page.addToken(0x25, "Reminder")
+		page.addToken(0x26, "ResponseRequested")
+		page.addToken(0x27, "Recurrences")
+		page.addToken(0x28, "Recurrence")
+		page.addToken(0x29, "Recurrence_Type")
+		page.addToken(0x2A, "Recurrence_Until")
+		page.addToken(0x2B, "Recurrence_Occurrences")
+		page.addToken(0x2C, "Recurrence_Interval")
+		page.addToken(0x2D, "Recurrence_DayOfWeek")
+		page.addToken(0x2E, "Recurrence_DayOfMonth")
+		page.addToken(0x2F, "Recurrence_WeekOfMonth")
+		page.addToken(0x30, "Recurrence_MonthOfYear")
+		page.addToken(0x31, "StartTime")
+		page.addToken(0x32, "Sensitivity")
+		page.addToken(0x33, "TimeZone")
+		page.addToken(0x34, "GlobalObjId")
+		page.addToken(0x35, "ThreadTopic")
+		page.addToken(0x39, "InternetCPID")
+		page.addToken(0x3A, "Flag")
+		page.addToken(0x3B, "FlagStatus")
+		page.addToken(0x3C, "ContentClass")
+		page.addToken(0x3D, "FlagType")
+		page.addToken(0x3E, "CompleteTime")
+		page.addToken(0x3F, "DisallowNewTimeProposal")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 3: AirNotify - retired
+		# region AirNotify Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = ""
+		page.xmlns = ""
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 4: Calendar
+		# region Calendar Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Calendar:"
+		page.xmlns = "calendar"
+
+		page.addToken(0x05, "TimeZone")
+		page.addToken(0x06, "AllDayEvent")
+		page.addToken(0x07, "Attendees")
+		page.addToken(0x08, "Attendee")
+		page.addToken(0x09, "Attendee_Email")
+		page.addToken(0x0A, "Attendee_Name")
+		page.addToken(0x0D, "BusyStatus")
+		page.addToken(0x0E, "Categories")
+		page.addToken(0x0F, "Category")
+		page.addToken(0x11, "DTStamp")
+		page.addToken(0x12, "EndTime")
+		page.addToken(0x13, "Exception")
+		page.addToken(0x14, "Exceptions")
+		page.addToken(0x15, "Exception_Deleted")
+		page.addToken(0x16, "Exception_StartTime")
+		page.addToken(0x17, "Location")
+		page.addToken(0x18, "MeetingStatus")
+		page.addToken(0x19, "Organizer_Email")
+		page.addToken(0x1A, "Organizer_Name")
+		page.addToken(0x1B, "Recurrence")
+		page.addToken(0x1C, "Recurrence_Type")
+		page.addToken(0x1D, "Recurrence_Until")
+		page.addToken(0x1E, "Recurrence_Occurrences")
+		page.addToken(0x1F, "Recurrence_Interval")
+		page.addToken(0x20, "Recurrence_DayOfWeek")
+		page.addToken(0x21, "Recurrence_DayOfMonth")
+		page.addToken(0x22, "Recurrence_WeekOfMonth")
+		page.addToken(0x23, "Recurrence_MonthOfYear")
+		page.addToken(0x24, "Reminder")
+		page.addToken(0x25, "Sensitivity")
+		page.addToken(0x26, "Subject")
+		page.addToken(0x27, "StartTime")
+		page.addToken(0x28, "UID")
+		page.addToken(0x29, "Attendee_Status")
+		page.addToken(0x2A, "Attendee_Type")
+		page.addToken(0x33, "DisallowNewTimeProposal")
+		page.addToken(0x34, "ResponseRequested")
+		page.addToken(0x35, "AppointmentReplyTime")
+		page.addToken(0x36, "ResponseType")
+		page.addToken(0x37, "CalendarType")
+		page.addToken(0x38, "IsLeapMonth")
+		page.addToken(0x39, "FirstDayOfWeek")
+		page.addToken(0x3A, "OnlineMeetingConfLink")
+		page.addToken(0x3B, "OnlineMeetingExternalLink")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 5: Move
+		# region Move Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Move:"
+		page.xmlns = "move"
+
+		page.addToken(0x05, "MoveItems")
+		page.addToken(0x06, "Move")
+		page.addToken(0x07, "SrcMsgId")
+		page.addToken(0x08, "SrcFldId")
+		page.addToken(0x09, "DstFldId")
+		page.addToken(0x0A, "Response")
+		page.addToken(0x0B, "Status")
+		page.addToken(0x0C, "DstMsgId")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 6: ItemEstimate
+		# region ItemEstimate Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "GetItemEstimate:"
+		page.xmlns = "getitemestimate"
+
+		page.addToken(0x05, "GetItemEstimate")
+		page.addToken(0x06, "Version")
+		page.addToken(0x07, "Collections")
+		page.addToken(0x08, "Collection")
+		page.addToken(0x09, "Class")
+		page.addToken(0x0A, "CollectionId")
+		page.addToken(0x0B, "DateTime")
+		page.addToken(0x0C, "Estimate")
+		page.addToken(0x0D, "Response")
+		page.addToken(0x0E, "Status")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 7: FolderHierarchy
+		# region FolderHierarchy Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "FolderHierarchy:"
+		page.xmlns = "folderhierarchy"
+
+		page.addToken(0x07, "DisplayName")
+		page.addToken(0x08, "ServerId")
+		page.addToken(0x09, "ParentId")
+		page.addToken(0x0A, "Type")
+		page.addToken(0x0C, "Status")
+		page.addToken(0x0E, "Changes")
+		page.addToken(0x0F, "Add")
+		page.addToken(0x10, "Delete")
+		page.addToken(0x11, "Update")
+		page.addToken(0x12, "SyncKey")
+		page.addToken(0x13, "FolderCreate")
+		page.addToken(0x14, "FolderDelete")
+		page.addToken(0x15, "FolderUpdate")
+		page.addToken(0x16, "FolderSync")
+		page.addToken(0x17, "Count")
+
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 8: MeetingResponse
+		# region MeetingResponse Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "MeetingResponse:"
+		page.xmlns = "meetingresponse"
+
+		page.addToken(0x05, "CalendarId")
+		page.addToken(0x06, "CollectionId")
+		page.addToken(0x07, "MeetingResponse")
+		page.addToken(0x08, "RequestId")
+		page.addToken(0x09, "Request")
+		page.addToken(0x0A, "Result")
+		page.addToken(0x0B, "Status")
+		page.addToken(0x0C, "UserResponse")
+		page.addToken(0x0E, "InstanceId")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 9: Tasks
+		# region Tasks Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Tasks:"
+		page.xmlns = "tasks"
+
+		page.addToken(0x08, "Categories")
+		page.addToken(0x09, "Category")
+		page.addToken(0x0A, "Complete")
+		page.addToken(0x0B, "DateCompleted")
+		page.addToken(0x0C, "DueDate")
+		page.addToken(0x0D, "UTCDueDate")
+		page.addToken(0x0E, "Importance")
+		page.addToken(0x0F, "Recurrence")
+		page.addToken(0x10, "Recurrence_Type")
+		page.addToken(0x11, "Recurrence_Start")
+		page.addToken(0x12, "Recurrence_Until")
+		page.addToken(0x13, "Recurrence_Occurrences")
+		page.addToken(0x14, "Recurrence_Interval")
+		page.addToken(0x15, "Recurrence_DayOfMonth")
+		page.addToken(0x16, "Recurrence_DayOfWeek")
+		page.addToken(0x17, "Recurrence_WeekOfMonth")
+		page.addToken(0x18, "Recurrence_MonthOfYear")
+		page.addToken(0x19, "Recurrence_Regenerate")
+		page.addToken(0x1A, "Recurrence_DeadOccur")
+		page.addToken(0x1B, "ReminderSet")
+		page.addToken(0x1C, "ReminderTime")
+		page.addToken(0x1D, "Sensitivity")
+		page.addToken(0x1E, "StartDate")
+		page.addToken(0x1F, "UTCStartDate")
+		page.addToken(0x20, "Subject")
+		page.addToken(0x22, "OrdinalDate")
+		page.addToken(0x23, "SubOrdinalDate")
+		page.addToken(0x24, "CalendarType")
+		page.addToken(0x25, "IsLeapMonth")
+		page.addToken(0x26, "FirstDayOfWeek")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 10: ResolveRecipients
+		# region ResolveRecipients Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "ResolveRecipients:"
+		page.xmlns = "resolverecipients"
+
+		page.addToken(0x05, "ResolveRecipients")
+		page.addToken(0x06, "Response")
+		page.addToken(0x07, "Status")
+		page.addToken(0x08, "Type")
+		page.addToken(0x09, "Recipient")
+		page.addToken(0x0A, "DisplayName")
+		page.addToken(0x0B, "EmailAddress")
+		page.addToken(0x0C, "Certificates")
+		page.addToken(0x0D, "Certificate")
+		page.addToken(0x0E, "MiniCertificate")
+		page.addToken(0x0F, "Options")
+		page.addToken(0x10, "To")
+		page.addToken(0x11, "CertificateRetrieval")
+		page.addToken(0x12, "RecipientCount")
+		page.addToken(0x13, "MaxCertificates")
+		page.addToken(0x14, "MaxAmbiguousRecipients")
+		page.addToken(0x15, "CertificateCount")
+		page.addToken(0x16, "Availability")
+		page.addToken(0x17, "StartTime")
+		page.addToken(0x18, "EndTime")
+		page.addToken(0x19, "MergedFreeBusy")
+		page.addToken(0x1A, "Picture")
+		page.addToken(0x1B, "MaxSize")
+		page.addToken(0x1C, "Data")
+		page.addToken(0x1D, "MaxPictures")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 11: ValidateCert
+		# region ValidateCert Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "ValidateCert:"
+		page.xmlns = "validatecert"
+
+		page.addToken(0x05, "ValidateCert")
+		page.addToken(0x06, "Certificates")
+		page.addToken(0x07, "Certificate")
+		page.addToken(0x08, "CertificateChain")
+		page.addToken(0x09, "CheckCRL")
+		page.addToken(0x0A, "Status")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 12: Contacts2
+		# region Contacts2 Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Contacts2:"
+		page.xmlns = "contacts2"
+
+		page.addToken(0x05, "CustomerId")
+		page.addToken(0x06, "GovernmentId")
+		page.addToken(0x07, "IMAddress")
+		page.addToken(0x08, "IMAddress2")
+		page.addToken(0x09, "IMAddress3")
+		page.addToken(0x0A, "ManagerName")
+		page.addToken(0x0B, "CompanyMainPhone")
+		page.addToken(0x0C, "AccountName")
+		page.addToken(0x0D, "NickName")
+		page.addToken(0x0E, "MMS")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 13: Ping
+		# region Ping Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Ping:"
+		page.xmlns = "ping"
+
+		page.addToken(0x05, "Ping")
+		page.addToken(0x06, "AutdState")  # Per MS-ASWBXML, this tag is not used by protocol
+		page.addToken(0x07, "Status")
+		page.addToken(0x08, "HeartbeatInterval")
+		page.addToken(0x09, "Folders")
+		page.addToken(0x0A, "Folder")
+		page.addToken(0x0B, "Id")
+		page.addToken(0x0C, "Class")
+		page.addToken(0x0D, "MaxFolders")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 14: Provision
+		# region Provision Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Provision:"
+		page.xmlns = "provision"
+
+		page.addToken(0x05, "Provision")
+		page.addToken(0x06, "Policies")
+		page.addToken(0x07, "Policy")
+		page.addToken(0x08, "PolicyType")
+		page.addToken(0x09, "PolicyKey")
+		page.addToken(0x0A, "Data")
+		page.addToken(0x0B, "Status")
+		page.addToken(0x0C, "RemoteWipe")
+		page.addToken(0x0D, "EASProvisionDoc")
+		page.addToken(0x0E, "DevicePasswordEnabled")
+		page.addToken(0x0F, "AlphanumericDevicePasswordRequired")
+		page.addToken(0x10, "RequireStorageCardEncryption")
+		page.addToken(0x11, "PasswordRecoveryEnabled")
+		page.addToken(0x13, "AttachmentsEnabled")
+		page.addToken(0x14, "MinDevicePasswordLength")
+		page.addToken(0x15, "MaxInactivityTimeDeviceLock")
+		page.addToken(0x16, "MaxDevicePasswordFailedAttempts")
+		page.addToken(0x17, "MaxAttachmentSize")
+		page.addToken(0x18, "AllowSimpleDevicePassword")
+		page.addToken(0x19, "DevicePasswordExpiration")
+		page.addToken(0x1A, "DevicePasswordHistory")
+		page.addToken(0x1B, "AllowStorageCard")
+		page.addToken(0x1C, "AllowCamera")
+		page.addToken(0x1D, "RequireDeviceEncryption")
+		page.addToken(0x1E, "AllowUnsignedApplications")
+		page.addToken(0x1F, "AllowUnsignedInstallationPackages")
+		page.addToken(0x20, "MinDevicePasswordComplexCharacters")
+		page.addToken(0x21, "AllowWiFi")
+		page.addToken(0x22, "AllowTextMessaging")
+		page.addToken(0x23, "AllowPOPIMAPEmail")
+		page.addToken(0x24, "AllowBluetooth")
+		page.addToken(0x25, "AllowIrDA")
+		page.addToken(0x26, "RequireManualSyncWhenRoaming")
+		page.addToken(0x27, "AllowDesktopSync")
+		page.addToken(0x28, "MaxCalendarAgeFilter")
+		page.addToken(0x29, "AllowHTMLEmail")
+		page.addToken(0x2A, "MaxEmailAgeFilter")
+		page.addToken(0x2B, "MaxEmailBodyTruncationSize")
+		page.addToken(0x2C, "MaxEmailHTMLBodyTruncationSize")
+		page.addToken(0x2D, "RequireSignedSMIMEMessages")
+		page.addToken(0x2E, "RequireEncryptedSMIMEMessages")
+		page.addToken(0x2F, "RequireSignedSMIMEAlgorithm")
+		page.addToken(0x30, "RequireEncryptionSMIMEAlgorithm")
+		page.addToken(0x31, "AllowSMIMEEncryptionAlgorithmNegotiation")
+		page.addToken(0x32, "AllowSMIMESoftCerts")
+		page.addToken(0x33, "AllowBrowser")
+		page.addToken(0x34, "AllowConsumerEmail")
+		page.addToken(0x35, "AllowRemoteDesktop")
+		page.addToken(0x36, "AllowInternetSharing")
+		page.addToken(0x37, "UnapprovedInROMApplicationList")
+		page.addToken(0x38, "ApplicationName")
+		page.addToken(0x39, "ApprovedApplicationList")
+		page.addToken(0x3A, "Hash")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 15: Search
+		# region Search Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Search:"
+		page.xmlns = "search"
+
+		page.addToken(0x05, "Search")
+		page.addToken(0x07, "Store")
+		page.addToken(0x08, "Name")
+		page.addToken(0x09, "Query")
+		page.addToken(0x0A, "Options")
+		page.addToken(0x0B, "Range")
+		page.addToken(0x0C, "Status")
+		page.addToken(0x0D, "Response")
+		page.addToken(0x0E, "Result")
+		page.addToken(0x0F, "Properties")
+		page.addToken(0x10, "Total")
+		page.addToken(0x11, "EqualTo")
+		page.addToken(0x12, "Value")
+		page.addToken(0x13, "And")
+		page.addToken(0x14, "Or")
+		page.addToken(0x15, "FreeText")
+		page.addToken(0x17, "DeepTraversal")
+		page.addToken(0x18, "LongId")
+		page.addToken(0x19, "RebuildResults")
+		page.addToken(0x1A, "LessThan")
+		page.addToken(0x1B, "GreaterThan")
+		page.addToken(0x1E, "UserName")
+		page.addToken(0x1F, "Password")
+		page.addToken(0x20, "ConversationId")
+		page.addToken(0x21, "Picture")
+		page.addToken(0x22, "MaxSize")
+		page.addToken(0x23, "MaxPictures")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 16: GAL
+		# region GAL Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "GAL:"
+		page.xmlns = "gal"
+
+		page.addToken(0x05, "DisplayName")
+		page.addToken(0x06, "Phone")
+		page.addToken(0x07, "Office")
+		page.addToken(0x08, "Title")
+		page.addToken(0x09, "Company")
+		page.addToken(0x0A, "Alias")
+		page.addToken(0x0B, "FirstName")
+		page.addToken(0x0C, "LastName")
+		page.addToken(0x0D, "HomePhone")
+		page.addToken(0x0E, "MobilePhone")
+		page.addToken(0x0F, "EmailAddress")
+		page.addToken(0x10, "Picture")
+		page.addToken(0x11, "Status")
+		page.addToken(0x12, "Data")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 17: AirSyncBase
+		# region AirSyncBase Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "AirSyncBase:"
+		page.xmlns = "airsyncbase"
+
+		page.addToken(0x05, "BodyPreference")
+		page.addToken(0x06, "Type")
+		page.addToken(0x07, "TruncationSize")
+		page.addToken(0x08, "AllOrNone")
+		page.addToken(0x0A, "Body")
+		page.addToken(0x0B, "Data")
+		page.addToken(0x0C, "EstimatedDataSize")
+		page.addToken(0x0D, "Truncated")
+		page.addToken(0x0E, "Attachments")
+		page.addToken(0x0F, "Attachment")
+		page.addToken(0x10, "DisplayName")
+		page.addToken(0x11, "FileReference")
+		page.addToken(0x12, "Method")
+		page.addToken(0x13, "ContentId")
+		page.addToken(0x14, "ContentLocation")
+		page.addToken(0x15, "IsInline")
+		page.addToken(0x16, "NativeBodyType")
+		page.addToken(0x17, "ContentType")
+		page.addToken(0x18, "Preview")
+		page.addToken(0x19, "BodyPartPreference")
+		page.addToken(0x1A, "BodyPart")
+		page.addToken(0x1B, "Status")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 18: Settings
+		# region Settings Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Settings:"
+		page.xmlns = "settings"
+
+		page.addToken(0x05, "Settings")
+		page.addToken(0x06, "Status")
+		page.addToken(0x07, "Get")
+		page.addToken(0x08, "Set")
+		page.addToken(0x09, "Oof")
+		page.addToken(0x0A, "OofState")
+		page.addToken(0x0B, "StartTime")
+		page.addToken(0x0C, "EndTime")
+		page.addToken(0x0D, "OofMessage")
+		page.addToken(0x0E, "AppliesToInternal")
+		page.addToken(0x0F, "AppliesToExternalKnown")
+		page.addToken(0x10, "AppliesToExternalUnknown")
+		page.addToken(0x11, "Enabled")
+		page.addToken(0x12, "ReplyMessage")
+		page.addToken(0x13, "BodyType")
+		page.addToken(0x14, "DevicePassword")
+		page.addToken(0x15, "Password")
+		page.addToken(0x16, "DeviceInformation")
+		page.addToken(0x17, "Model")
+		page.addToken(0x18, "IMEI")
+		page.addToken(0x19, "FriendlyName")
+		page.addToken(0x1A, "OS")
+		page.addToken(0x1B, "OSLanguage")
+		page.addToken(0x1C, "PhoneNumber")
+		page.addToken(0x1D, "UserInformation")
+		page.addToken(0x1E, "EmailAddresses")
+		page.addToken(0x1F, "SmtpAddress")
+		page.addToken(0x20, "UserAgent")
+		page.addToken(0x21, "EnableOutboundSMS")
+		page.addToken(0x22, "MobileOperator")
+		page.addToken(0x23, "PrimarySmtpAddress")
+		page.addToken(0x24, "Accounts")
+		page.addToken(0x25, "Account")
+		page.addToken(0x26, "AccountId")
+		page.addToken(0x27, "AccountName")
+		page.addToken(0x28, "UserDisplayName")
+		page.addToken(0x29, "SendDisabled")
+		page.addToken(0x2B, "RightsManagementInformation")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 19: DocumentLibrary
+		# region DocumentLibrary Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "DocumentLibrary:"
+		page.xmlns = "documentlibrary"
+
+		page.addToken(0x05, "LinkId")
+		page.addToken(0x06, "DisplayName")
+		page.addToken(0x07, "IsFolder")
+		page.addToken(0x08, "CreationDate")
+		page.addToken(0x09, "LastModifiedDate")
+		page.addToken(0x0A, "IsHidden")
+		page.addToken(0x0B, "ContentLength")
+		page.addToken(0x0C, "ContentType")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 20: ItemOperations
+		# region ItemOperations Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "ItemOperations:"
+		page.xmlns = "itemoperations"
+
+		page.addToken(0x05, "ItemOperations")
+		page.addToken(0x06, "Fetch")
+		page.addToken(0x07, "Store")
+		page.addToken(0x08, "Options")
+		page.addToken(0x09, "Range")
+		page.addToken(0x0A, "Total")
+		page.addToken(0x0B, "Properties")
+		page.addToken(0x0C, "Data")
+		page.addToken(0x0D, "Status")
+		page.addToken(0x0E, "Response")
+		page.addToken(0x0F, "Version")
+		page.addToken(0x10, "Schema")
+		page.addToken(0x11, "Part")
+		page.addToken(0x12, "EmptyFolderContents")
+		page.addToken(0x13, "DeleteSubFolders")
+		page.addToken(0x14, "UserName")
+		page.addToken(0x15, "Password")
+		page.addToken(0x16, "Move")
+		page.addToken(0x17, "DstFldId")
+		page.addToken(0x18, "ConversationId")
+		page.addToken(0x19, "MoveAlways")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 21: ComposeMail
+		# region ComposeMail Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "ComposeMail:"
+		page.xmlns = "composemail"
+
+		page.addToken(0x05, "SendMail")
+		page.addToken(0x06, "SmartForward")
+		page.addToken(0x07, "SmartReply")
+		page.addToken(0x08, "SaveInSentItems")
+		page.addToken(0x09, "ReplaceMime")
+		page.addToken(0x0B, "Source")
+		page.addToken(0x0C, "FolderId")
+		page.addToken(0x0D, "ItemId")
+		page.addToken(0x0E, "LongId")
+		page.addToken(0x0F, "InstanceId")
+		page.addToken(0x10, "MIME")
+		page.addToken(0x11, "ClientId")
+		page.addToken(0x12, "Status")
+		page.addToken(0x13, "AccountId")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 22: Email2
+		# region Email2 Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Email2:"
+		page.xmlns = "email2"
+
+		page.addToken(0x05, "UmCallerID")
+		page.addToken(0x06, "UmUserNotes")
+		page.addToken(0x07, "UmAttDuration")
+		page.addToken(0x08, "UmAttOrder")
+		page.addToken(0x09, "ConversationId")
+		page.addToken(0x0A, "ConversationIndex")
+		page.addToken(0x0B, "LastVerbExecuted")
+		page.addToken(0x0C, "LastVerbExecutionTime")
+		page.addToken(0x0D, "ReceivedAsBcc")
+		page.addToken(0x0E, "Sender")
+		page.addToken(0x0F, "CalendarType")
+		page.addToken(0x10, "IsLeapMonth")
+		page.addToken(0x11, "AccountId")
+		page.addToken(0x12, "FirstDayOfWeek")
+		page.addToken(0x13, "MeetingMessageType")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 23: Notes
+		# region Notes Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "Notes:"
+		page.xmlns = "notes"
+
+		page.addToken(0x05, "Subject")
+		page.addToken(0x06, "MessageClass")
+		page.addToken(0x07, "LastModifiedDate")
+		page.addToken(0x08, "Categories")
+		page.addToken(0x09, "Category")
+		self.codePages.append(page)
+		# endregion
+
+		# Code Page 24: RightsManagement
+		# region RightsManagement Code Page
+		page = ASWBXMLCodePage()
+		page.namespace = "RightsManagement:"
+		page.xmlns = "rightsmanagement"
+
+		page.addToken(0x05, "RightsManagementSupport")
+		page.addToken(0x06, "RightsManagementTemplates")
+		page.addToken(0x07, "RightsManagementTemplate")
+		page.addToken(0x08, "RightsManagementLicense")
+		page.addToken(0x09, "EditAllowed")
+		page.addToken(0x0A, "ReplyAllowed")
+		page.addToken(0x0B, "ReplyAllAllowed")
+		page.addToken(0x0C, "ForwardAllowed")
+		page.addToken(0x0D, "ModifyRecipientsAllowed")
+		page.addToken(0x0E, "ExtractAllowed")
+		page.addToken(0x0F, "PrintAllowed")
+		page.addToken(0x10, "ExportAllowed")
+		page.addToken(0x11, "ProgrammaticAccessAllowed")
+		page.addToken(0x12, "RMOwner")
+		page.addToken(0x13, "ContentExpiryDate")
+		page.addToken(0x14, "TemplateID")
+		page.addToken(0x15, "TemplateName")
+		page.addToken(0x16, "TemplateDescription")
+		page.addToken(0x17, "ContentOwner")
+		page.addToken(0x18, "RemoveRightsManagementDistribution")
+		self.codePages.append(page)
+		# endregion
+		# endregion
+	
+	def loadXml(self, strXML):
+		# note xmlDoc has .childNodes and .parentNode
+		self.xmlDoc = xml.dom.minidom.parseString(strXML)
+
+	def getXml(self):
+		if (self.xmlDoc != None):
+			try:
+				return self.xmlDoc.toprettyxml(indent="    ", newl="\n")
+			except:
+				return self.xmlDoc.toxml()
+	
+	def loadBytes(self, byteWBXML):
+		
+		currentNode = self.xmlDoc
+		
+		wbXMLBytes = ASWBXMLByteQueue(byteWBXML)
+		# Version is ignored
+		version = wbXMLBytes.dequeueAndLog()
+
+		# Public Identifier is ignored
+		publicId = wbXMLBytes.dequeueMultibyteInt()
+		
+		logging.debug("Version: %d, Public Identifier: %d" % (version, publicId))
+		
+		# Character set
+		# Currently only UTF-8 is supported, throw if something else
+		charset = wbXMLBytes.dequeueMultibyteInt()
+		if (charset != 0x6A):
+			raise InvalidDataException("ASWBXML only supports UTF-8 encoded XML.")
+
+		# String table length
+		# This should be 0, MS-ASWBXML does not use string tables
+		stringTableLength = wbXMLBytes.dequeueMultibyteInt()
+		if (stringTableLength != 0):
+			raise InvalidDataException("WBXML data contains a string table.")
+
+		# Now we should be at the body of the data.
+		# Add the declaration
+		unusedArray = [GlobalTokens.ENTITY, GlobalTokens.EXT_0, GlobalTokens.EXT_1, GlobalTokens.EXT_2, GlobalTokens.EXT_I_0, GlobalTokens.EXT_I_1, GlobalTokens.EXT_I_2, GlobalTokens.EXT_T_0, GlobalTokens.EXT_T_1, GlobalTokens.EXT_T_2, GlobalTokens.LITERAL, GlobalTokens.LITERAL_A, GlobalTokens.LITERAL_AC, GlobalTokens.LITERAL_C, GlobalTokens.PI, GlobalTokens.STR_T]
+		
+		while ( wbXMLBytes.qsize() > 0):
+			currentByte = wbXMLBytes.dequeueAndLog()
+			if ( currentByte == GlobalTokens.SWITCH_PAGE ):
+				newCodePage = wbXMLBytes.dequeueAndLog()
+				if (newCodePage >= 0 and newCodePage < 25):
+					self.currentCodePage = newCodePage
+				else:
+					raise InvalidDataException("Unknown code page ID 0x{0:X} encountered in WBXML".format(currentByte))
+			elif  ( currentByte == GlobalTokens.END ):
+				if (currentNode != None and currentNode.parentNode != None):
+					currentNode = currentNode.parentNode
+				else:
+					raise InvalidDataException("END global token encountered out of sequence")
+					break
+			elif  ( currentByte == GlobalTokens.OPAQUE ):
+				CDATALength = wbXMLBytes.dequeueMultibyteInt()
+				newOpaqueNode = self.xmlDoc.createCDATASection(wbXMLBytes.dequeueString(CDATALength))
+				currentNode.appendChild(newOpaqueNode)
+
+			elif  ( currentByte == GlobalTokens.STR_I ):
+				newTextNode = self.xmlDoc.createTextNode(wbXMLBytes.dequeueString())
+				currentNode.appendChild(newTextNode)
+
+			elif ( currentByte in unusedArray):
+				raise InvalidDataException("Encountered unknown global token 0x{0:X}.".format(currentByte))
+			else:
+				hasAttributes = (currentByte & 0x80) > 0
+				hasContent = (currentByte & 0x40) > 0
+
+				token = currentByte & 0x3F
+				if (hasAttributes):
+					raise InvalidDataException("Token 0x{0:X} has attributes.".format(token))
+
+				strTag = self.codePages[self.currentCodePage].getTag(token)
+				if (strTag == None):
+					strTag = "UNKNOWN_TAG_{0,2:X}".format(token)
+
+				newNode = self.xmlDoc.createElement(strTag)
+				# not sure if this should be set on every node or not
+				#newNode.setAttribute("xmlns", self.codePages[self.currentCodePage].xmlns)
+				
+				currentNode.appendChild(newNode)
+
+				if (hasContent):
+					currentNode = newNode
+
+		logging.debug("Total bytes dequeued: %d" % wbXMLBytes.bytesDequeued)

libmproxy/contrib/wbxml/ASWBXMLByteQueue.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+'''
+@author: David Shaw, david.shaw.aw@gmail.com
+
+Inspired by EAS Inspector for Fiddler
+https://easinspectorforfiddler.codeplex.com
+
+----- The MIT License (MIT) ----- 
+Filename: ASWBXMLByteQueue.py
+Copyright (c) 2014, David P. Shaw
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+'''
+from Queue import Queue
+import logging
+
+class ASWBXMLByteQueue(Queue):
+
+    def __init__(self, wbxmlBytes):
+        
+        self.bytesDequeued = 0
+        self.bytesEnqueued = 0
+        
+        Queue.__init__(self)
+
+        for byte in wbxmlBytes:
+            self.put(ord(byte))
+            self.bytesEnqueued += 1
+        
+        
+        logging.debug("Array byte count: %d, enqueued: %d" % (self.qsize(), self.bytesEnqueued))
+    
+    """
+    Created to debug the dequeueing of bytes
+    """
+    def dequeueAndLog(self):
+        singleByte = self.get()
+        self.bytesDequeued += 1
+        logging.debug("Dequeued byte 0x{0:X} ({1} total)".format(singleByte, self.bytesDequeued))
+        return singleByte
+    
+    """
+    Return true if the continuation bit is set in the byte
+    """
+    def checkContinuationBit(self, byteval):
+        continuationBitmask = 0x80
+        return (continuationBitmask & byteval) != 0
+    
+    def dequeueMultibyteInt(self):
+        iReturn = 0
+        singleByte = 0xFF
+         
+        while True:
+            iReturn <<= 7
+            if (self.qsize() == 0):
+                break
+            else:
+                singleByte = self.dequeueAndLog()
+            iReturn += int(singleByte & 0x7F)
+            if not self.checkContinuationBit(singleByte):
+                return iReturn
+
+    def dequeueString(self, length=None):
+        if ( length != None):
+            currentByte = 0x00
+            strReturn = ""
+            for i in range(0, length):
+                # TODO: Improve this handling. We are technically UTF-8, meaning
+                # that characters could be more than one byte long. This will fail if we have
+                # characters outside of the US-ASCII range
+                if ( self.qsize() == 0 ):
+                    break
+                currentByte = self.dequeueAndLog()
+                strReturn += chr(currentByte)
+
+        else:
+            currentByte = 0x00
+            strReturn = ""
+            while True:
+                currentByte = self.dequeueAndLog()
+                if (currentByte != 0x00):
+                    strReturn += chr(currentByte)
+                else:
+                    break
+
+        return strReturn
+

libmproxy/contrib/wbxml/ASWBXMLCodePage.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+'''
+@author: David Shaw, david.shaw.aw@gmail.com
+
+Inspired by EAS Inspector for Fiddler
+https://easinspectorforfiddler.codeplex.com
+
+----- The MIT License (MIT) ----- 
+Filename: ASWBXMLCodePage.py
+Copyright (c) 2014, David P. Shaw
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+'''
+class ASWBXMLCodePage:
+	def __init__(self):
+		self.namespace = ""
+		self.xmlns = ""
+		self.tokenLookup = {}
+		self.tagLookup = {}
+	
+	def addToken(self, token, tag):
+		self.tokenLookup[token] = tag
+		self.tagLookup[tag] = token
+	
+	def getToken(self, tag):
+		if self.tagLookup.has_key(tag):
+			return self.tagLookup[tag]
+		return 0xFF
+	
+	def getTag(self, token):
+		if self.tokenLookup.has_key(token):
+			return self.tokenLookup[token]
+		return None
+	
+	def __repr__(self):
+		return str(self.tokenLookup)

libmproxy/contrib/wbxml/GlobalTokens.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+'''
+@author: David Shaw, david.shaw.aw@gmail.com
+
+Inspired by EAS Inspector for Fiddler
+https://easinspectorforfiddler.codeplex.com
+
+----- The MIT License (MIT) ----- 
+Filename: GlobalTokens.py
+Copyright (c) 2014, David P. Shaw
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+'''
+class GlobalTokens:
+    SWITCH_PAGE = 0x00
+    END = 0x01
+    ENTITY = 0x02
+    STR_I = 0x03
+    LITERAL = 0x04
+    EXT_I_0 = 0x40
+    EXT_I_1 = 0x41
+    EXT_I_2 = 0x42
+    PI = 0x43
+    LITERAL_C = 0x44
+    EXT_T_0 = 0x80
+    EXT_T_1 = 0x81
+    EXT_T_2 = 0x82
+    STR_T = 0x83
+    LITERAL_A = 0x84
+    EXT_0 = 0xC0
+    EXT_1 = 0xC1
+    EXT_2 = 0xC2
+    OPAQUE = 0xC3
+    LITERAL_AC = 0xC4

libmproxy/contrib/wbxml/InvalidDataException.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+'''
+@author: David Shaw, david.shaw.aw@gmail.com
+
+Inspired by EAS Inspector for Fiddler
+https://easinspectorforfiddler.codeplex.com
+
+----- The MIT License (MIT) ----- 
+Filename: InvalidDataException.py
+Copyright (c) 2014, David P. Shaw
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+'''
+class InvalidDataException(Exception):
+    pass

libmproxy/controller.py

@@ -1,8 +1,6 @@
+from __future__ import absolute_import
 import Queue, threading
 
-should_exit = False
-
-
 class DummyReply:
     """
         A reply object that does nothing. Useful when we need an object to seem
@@ -36,8 +34,9 @@ class Reply:
 
 
 class Channel:
-    def __init__(self, q):
+    def __init__(self, q, should_exit):
         self.q = q
+        self.should_exit = should_exit
 
     def ask(self, mtype, m):
         """
@@ -46,11 +45,11 @@ class Channel:
         """
         m.reply = Reply(m)
         self.q.put((mtype, m))
-        while not should_exit:
+        while not self.should_exit.is_set():
             try:
                 # The timeout is here so we can handle a should_exit event.
                 g = m.reply.q.get(timeout=0.5)
-            except Queue.Empty: # pragma: nocover
+            except Queue.Empty:  # pragma: nocover
                 continue
             return g
 
@@ -72,12 +71,13 @@ class Slave(threading.Thread):
         self.channel, self.server = channel, server
         self.server.set_channel(channel)
         threading.Thread.__init__(self)
+        self.name = "SlaveThread (%s:%s)" % (self.server.address.host, self.server.address.port)
 
     def run(self):
         self.server.serve_forever()
 
 
-class Master:
+class Master(object):
     """
         Masters get and respond to messages from slaves.
     """
@@ -87,8 +87,9 @@ class Master:
         """
         self.server = server
         self.masterq = Queue.Queue()
+        self.should_exit = threading.Event()
 
-    def tick(self, q):
+    def tick(self, q, timeout):
         changed = False
         try:
             # This endless loop runs until the 'Queue.Empty'
@@ -96,8 +97,7 @@ class Master:
             # the queue, this speeds up every request by 0.1 seconds,
             # because get_input(..) function is not blocking.
             while True:
-                # Small timeout to prevent pegging the CPU
-                msg = q.get(timeout=0.01)
+                msg = q.get(timeout=timeout)
                 self.handle(*msg)
                 changed = True
         except Queue.Empty:
@@ -105,11 +105,10 @@ class Master:
         return changed
 
     def run(self):
-        global should_exit
-        should_exit = False
-        self.server.start_slave(Slave, Channel(self.masterq))
-        while not should_exit:
-            self.tick(self.masterq)
+        self.should_exit.clear()
+        self.server.start_slave(Slave, Channel(self.masterq, self.should_exit))
+        while not self.should_exit.is_set():
+            self.tick(self.masterq, 0.01)
         self.shutdown()
 
     def handle(self, mtype, obj):
@@ -121,8 +120,7 @@ class Master:
             obj.reply()
 
     def shutdown(self):
-        global should_exit
-        if not should_exit:
-            should_exit = True
+        if not self.should_exit.is_set():
+            self.should_exit.set()
             if self.server:
                 self.server.shutdown()

libmproxy/dump.py

@@ -1,20 +1,25 @@
-import sys, os
+from __future__ import absolute_import
+import sys
+import os
 import netlib.utils
-import flow, filt, utils
+from . import flow, filt, utils
+from .protocol import http
 
-class DumpError(Exception): pass
+
+class DumpError(Exception):
+    pass
 
 
 class Options(object):
     attributes = [
         "app",
-        "app_external",
         "app_host",
         "app_port",
         "anticache",
         "anticomp",
         "client_replay",
-        "eventlog",
+        "filtstr",
+        "flow_detail",
         "keepserving",
         "kill",
         "no_server",
@@ -29,9 +34,13 @@ class Options(object):
         "showhost",
         "stickycookie",
         "stickyauth",
+        "stream_large_bodies",
         "verbosity",
         "wfile",
+        "replay_ignore_content",
+        "replay_ignore_params",
     ]
+
     def __init__(self, **kwargs):
         for k, v in kwargs.items():
             setattr(self, k, v)
@@ -42,35 +51,38 @@ class Options(object):
 
 def str_response(resp):
     r = "%s %s"%(resp.code, resp.msg)
-    if resp.is_replay():
+    if resp.is_replay:
         r = "[replay] " + r
     return r
 
 
-def str_request(req, showhost):
-    if req.client_conn:
-        c = req.client_conn.address[0]
+def str_request(f, showhost):
+    if f.client_conn:
+        c = f.client_conn.address.host
     else:
         c = "[replay]"
-    r = "%s %s %s"%(c, req.method, req.get_url(showhost))
-    if req.stickycookie:
+    r = "%s %s %s"%(c, f.request.method, f.request.pretty_url(showhost))
+    if f.request.stickycookie:
         r = "[stickycookie] " + r
     return r
 
 
 class DumpMaster(flow.FlowMaster):
-    def __init__(self, server, options, filtstr, outfile=sys.stdout):
+    def __init__(self, server, options, outfile=sys.stdout):
         flow.FlowMaster.__init__(self, server, flow.State())
         self.outfile = outfile
         self.o = options
         self.anticache = options.anticache
         self.anticomp = options.anticomp
-        self.eventlog = options.eventlog
         self.showhost = options.showhost
+        self.replay_ignore_params = options.replay_ignore_params
+        self.replay_ignore_content = options.replay_ignore_content
         self.refresh_server_playback = options.refresh_server_playback
 
-        if filtstr:
-            self.filt = filt.parse(filtstr)
+        self.set_stream_large_bodies(options.stream_large_bodies)
+
+        if options.filtstr:
+            self.filt = filt.parse(options.filtstr)
         else:
             self.filt = None
 
@@ -101,7 +113,9 @@ class DumpMaster(flow.FlowMaster):
                 self._readflow(options.server_replay),
                 options.kill, options.rheaders,
                 not options.keepserving,
-                options.nopop
+                options.nopop,
+                options.replay_ignore_params,
+                options.replay_ignore_content
             )
 
         if options.client_replay:
@@ -126,10 +140,10 @@ class DumpMaster(flow.FlowMaster):
             try:
                 self.load_flows(freader)
             except flow.FlowReadError, v:
-                self.add_event("Flow file corrupted. Stopped loading.")
+                self.add_event("Flow file corrupted. Stopped loading.", "error")
 
         if self.o.app:
-            self.start_app(self.o.app_host, self.o.app_port, self.o.app_external)
+            self.start_app(self.o.app_host, self.o.app_port)
 
     def _readflow(self, path):
         path = os.path.expanduser(path)
@@ -141,7 +155,8 @@ class DumpMaster(flow.FlowMaster):
         return flows
 
     def add_event(self, e, level="info"):
-        if self.eventlog:
+        needed = dict(error=0, info=1, debug=2).get(level, 1)
+        if self.o.verbosity >= needed:
             print >> self.outfile, e
             self.outfile.flush()
 
@@ -155,13 +170,18 @@ class DumpMaster(flow.FlowMaster):
             return
 
         if f.response:
-            sz = utils.pretty_size(len(f.response.content))
-            if self.o.verbosity > 0:
+            if self.o.flow_detail > 0:
+                if f.response.content == http.CONTENT_MISSING:
+                    sz = "(content missing)"
+                else:
+                    sz = utils.pretty_size(len(f.response.content))
                 result = " << %s %s"%(str_response(f.response), sz)
-            if self.o.verbosity > 1:
+            if self.o.flow_detail > 1:
                 result = result + "\n\n" + self.indent(4, f.response.headers)
-            if self.o.verbosity > 2:
-                if utils.isBin(f.response.content):
+            if self.o.flow_detail > 2:
+                if f.response.content == http.CONTENT_MISSING:
+                    cont = self.indent(4, "(content missing)")
+                elif utils.isBin(f.response.content):
                     d = netlib.utils.hexdump(f.response.content)
                     d = "\n".join("%s\t%s %s"%i for i in d)
                     cont = self.indent(4, d)
@@ -173,47 +193,45 @@ class DumpMaster(flow.FlowMaster):
         elif f.error:
             result = " << %s"%f.error.msg
 
-        if self.o.verbosity == 1:
-            print >> self.outfile, str_request(f.request, self.showhost)
+        if self.o.flow_detail == 1:
+            print >> self.outfile, str_request(f, self.showhost)
             print >> self.outfile, result
-        elif self.o.verbosity == 2:
-            print >> self.outfile, str_request(f.request, self.showhost)
+        elif self.o.flow_detail == 2:
+            print >> self.outfile, str_request(f, self.showhost)
             print >> self.outfile, self.indent(4, f.request.headers)
             print >> self.outfile
             print >> self.outfile, result
             print >> self.outfile, "\n"
-        elif self.o.verbosity >= 3:
-            print >> self.outfile, str_request(f.request, self.showhost)
+        elif self.o.flow_detail >= 3:
+            print >> self.outfile, str_request(f, self.showhost)
             print >> self.outfile, self.indent(4, f.request.headers)
-            if utils.isBin(f.request.content):
-                print >> self.outfile, self.indent(4, netlib.utils.hexdump(f.request.content))
+            if f.request.content != http.CONTENT_MISSING and utils.isBin(f.request.content):
+                d = netlib.utils.hexdump(f.request.content)
+                d = "\n".join("%s\t%s %s"%i for i in d)
+                print >> self.outfile, self.indent(4, d)
             elif f.request.content:
                 print >> self.outfile, self.indent(4, f.request.content)
             print >> self.outfile
             print >> self.outfile, result
             print >> self.outfile, "\n"
-        if self.o.verbosity:
+        if self.o.flow_detail:
             self.outfile.flush()
 
-    def handle_log(self, l):
-        self.add_event(l.msg)
-        l.reply()
-
-    def handle_request(self, r):
-        f = flow.FlowMaster.handle_request(self, r)
+    def handle_request(self, f):
+        flow.FlowMaster.handle_request(self, f)
         if f:
-            r.reply()
+            f.reply()
         return f
 
-    def handle_response(self, msg):
-        f = flow.FlowMaster.handle_response(self, msg)
+    def handle_response(self, f):
+        flow.FlowMaster.handle_response(self, f)
         if f:
-            msg.reply()
+            f.reply()
             self._process_flow(f)
         return f
 
-    def handle_error(self, msg):
-        f = flow.FlowMaster.handle_error(self, msg)
+    def handle_error(self, f):
+        flow.FlowMaster.handle_error(self, f)
         if f:
             self._process_flow(f)
         return f
@@ -223,8 +241,7 @@ class DumpMaster(flow.FlowMaster):
 
     def run(self):  # pragma: no cover
         if self.o.rfile and not self.o.keepserving:
-            for script in self.scripts:
-                self.unload_script(script)
+            self.shutdown()
             return
         try:
             return flow.FlowMaster.run(self)

libmproxy/encoding.py

@@ -1,6 +1,7 @@
 """
     Utility functions for decoding response bodies.
 """
+from __future__ import absolute_import
 import cStringIO
 import gzip, zlib
 

libmproxy/filt.py

@@ -31,8 +31,10 @@
         ~c CODE     Response code.
         rex         Equivalent to ~u rex
 """
+from __future__ import absolute_import
 import re, sys
-import contrib.pyparsing as pp
+from .contrib import pyparsing as pp
+from .protocol.http import decoded
 
 
 class _Token:
@@ -164,10 +166,14 @@ class FBod(_Rex):
     code = "b"
     help = "Body"
     def __call__(self, f):
-        if f.request.content and re.search(self.expr, f.request.content):
-            return True
-        elif f.response and f.response.content and re.search(self.expr, f.response.content):
-            return True
+        if f.request and f.request.content:
+            with decoded(f.request):
+                if re.search(self.expr, f.request.content):
+                    return True
+        if f.response and f.response.content:
+            with decoded(f.response):
+                if re.search(self.expr, f.response.content):
+                    return True
         return False
 
 
@@ -175,16 +181,20 @@ class FBodRequest(_Rex):
     code = "bq"
     help = "Request body"
     def __call__(self, f):
-        if f.request.content and re.search(self.expr, f.request.content):
-            return True
+        if f.request and f.request.content:
+            with decoded(f.request):
+                if re.search(self.expr, f.request.content):
+                    return True
 
 
 class FBodResponse(_Rex):
     code = "bs"
     help = "Response body"
     def __call__(self, f):
-        if f.response and f.response.content and re.search(self.expr, f.response.content):
-            return True
+        if f.response and f.response.content:
+            with decoded(f.response):
+                if re.search(self.expr, f.response.content):
+                    return True
 
 
 class FMethod(_Rex):
@@ -212,7 +222,7 @@ class FUrl(_Rex):
         return klass(*toks)
 
     def __call__(self, f):
-        return re.search(self.expr, f.request.get_url())
+        return re.search(self.expr, f.request.url)
 
 
 class _Int(_Action):

libmproxy/main.py

@@ -0,0 +1,230 @@
+from __future__ import print_function, absolute_import
+import argparse
+import os
+import signal
+import sys
+import netlib.version
+from . import version, cmdline
+from .proxy import process_proxy_options, ProxyServerError
+from .proxy.server import DummyServer, ProxyServer
+
+
+def check_versions():
+    """
+    Having installed a wrong version of pyOpenSSL or netlib is unfortunately a very common source of error.
+    Check before every start that both versions are somewhat okay.
+    """
+    # We don't introduce backward-incompatible changes in patch versions. Only consider major and minor version.
+    if netlib.version.IVERSION[:2] != version.IVERSION[:2]:
+        print(
+            "Warning: You are using mitmdump %s with netlib %s. "
+            "Most likely, that doesn't work - please upgrade!" % (version.VERSION, netlib.version.VERSION),
+            file=sys.stderr)
+    import OpenSSL, inspect
+
+    v = tuple([int(x) for x in OpenSSL.__version__.split(".")][:2])
+    if v < (0, 14):
+        print("You are using an outdated version of pyOpenSSL: mitmproxy requires pyOpenSSL 0.14 or greater.",
+              file=sys.stderr)
+        # Some users apparently have multiple versions of pyOpenSSL installed. Report which one we got.
+        pyopenssl_path = os.path.dirname(inspect.getfile(OpenSSL))
+        print("Your pyOpenSSL %s installation is located at %s" % (OpenSSL.__version__, pyopenssl_path),
+              file=sys.stderr)
+        sys.exit(1)
+
+
+def assert_utf8_env():
+    spec = ""
+    for i in ["LANG", "LC_CTYPE", "LC_ALL"]:
+        spec += os.environ.get(i, "").lower()
+    if "utf" not in spec:
+        print("Error: mitmproxy requires a UTF console environment.", file=sys.stderr)
+        print("Set your LANG enviroment variable to something like en_US.UTF-8", file=sys.stderr)
+        sys.exit(1)
+
+
+def get_server(dummy_server, options):
+    if dummy_server:
+        return DummyServer(options)
+    else:
+        try:
+            return ProxyServer(options)
+        except ProxyServerError, v:
+            print(str(v), file=sys.stderr)
+            sys.exit(1)
+
+
+def mitmproxy_cmdline():
+    # Don't import libmproxy.console for mitmdump, urwid is not available on all platforms.
+    from . import console
+    from .console import palettes
+
+    parser = argparse.ArgumentParser(usage="%(prog)s [options]")
+    parser.add_argument('--version', action='version', version=version.NAMEVERSION)
+    cmdline.common_options(parser)
+    parser.add_argument(
+        "--palette", type=str, default="dark",
+        action="store", dest="palette",
+        help="Select color palette: " + ", ".join(palettes.palettes.keys())
+    )
+    parser.add_argument(
+        "-e",
+        action="store_true", dest="eventlog",
+        help="Show event log."
+    )
+    group = parser.add_argument_group(
+        "Filters",
+        "See help in mitmproxy for filter expression syntax."
+    )
+    group.add_argument(
+        "-i", "--intercept", action="store",
+        type=str, dest="intercept", default=None,
+        help="Intercept filter expression."
+    )
+
+    options = parser.parse_args()
+    if options.quiet:
+        options.verbose = 0
+
+    proxy_config = process_proxy_options(parser, options)
+    console_options = console.Options(**cmdline.get_common_options(options))
+    console_options.palette = options.palette
+    console_options.eventlog = options.eventlog
+    console_options.intercept = options.intercept
+
+    return console_options, proxy_config
+
+
+def mitmproxy():  # pragma: nocover
+    from . import console
+
+    check_versions()
+    assert_utf8_env()
+    console_options, proxy_config = mitmproxy_cmdline()
+    server = get_server(console_options.no_server, proxy_config)
+
+    m = console.ConsoleMaster(server, console_options)
+    try:
+        m.run()
+    except KeyboardInterrupt:
+        pass
+
+
+def mitmdump_cmdline():
+    from . import dump
+
+    parser = argparse.ArgumentParser(usage="%(prog)s [options] [filter]")
+    parser.add_argument('--version', action='version', version="mitmdump" + " " + version.VERSION)
+    cmdline.common_options(parser)
+    parser.add_argument(
+        "--keepserving",
+        action="store_true", dest="keepserving", default=False,
+        help="Continue serving after client playback or file read. We exit by default."
+    )
+    parser.add_argument(
+        "-d",
+        action="count", dest="flow_detail", default=1,
+        help="Increase flow detail display level. Can be passed multiple times."
+    )
+    parser.add_argument('args', nargs=argparse.REMAINDER)
+
+    options = parser.parse_args()
+    if options.quiet:
+        options.verbose = 0
+        options.flow_detail = 0
+
+    proxy_config = process_proxy_options(parser, options)
+    dump_options = dump.Options(**cmdline.get_common_options(options))
+    dump_options.flow_detail = options.flow_detail
+    dump_options.keepserving = options.keepserving
+    dump_options.filtstr = " ".join(options.args) if options.args else None
+
+    return dump_options, proxy_config
+
+
+def mitmdump():  # pragma: nocover
+    from . import dump
+
+    check_versions()
+    dump_options, proxy_config = mitmdump_cmdline()
+    server = get_server(dump_options.no_server, proxy_config)
+
+    try:
+        master = dump.DumpMaster(server, dump_options)
+
+        def cleankill(*args, **kwargs):
+            master.shutdown()
+
+        signal.signal(signal.SIGTERM, cleankill)
+        master.run()
+    except dump.DumpError as e:
+        print("mitmdump: %s" % e, file=sys.stderr)
+        sys.exit(1)
+    except KeyboardInterrupt:
+        pass
+
+
+def mitmweb_cmdline():
+    from . import web
+    parser = argparse.ArgumentParser(usage="%(prog)s [options]")
+    parser.add_argument(
+        '--version',
+        action='version',
+        version="mitmweb" + " " + version.VERSION
+    )
+
+    group = parser.add_argument_group("Mitmweb")
+    group.add_argument(
+        "--wport",
+        action="store", type=int, dest="wport", default=8081,
+        metavar="PORT",
+        help="Mitmweb port."
+    )
+    group.add_argument(
+        "--wiface",
+        action="store", dest="wiface", default="127.0.0.1",
+        metavar="IFACE",
+        help="Mitmweb interface."
+    )
+    group.add_argument(
+        "--wdebug",
+        action="store_true", dest="wdebug",
+        help="Turn on mitmweb debugging"
+    )
+
+    cmdline.common_options(parser)
+    group = parser.add_argument_group(
+        "Filters",
+        "See help in mitmproxy for filter expression syntax."
+    )
+    group.add_argument(
+        "-i", "--intercept", action="store",
+        type=str, dest="intercept", default=None,
+        help="Intercept filter expression."
+    )
+
+    options = parser.parse_args()
+    if options.quiet:
+        options.verbose = 0
+
+    proxy_config = process_proxy_options(parser, options)
+    web_options = web.Options(**cmdline.get_common_options(options))
+    web_options.intercept = options.intercept
+    web_options.wdebug = options.wdebug
+    web_options.wiface = options.wiface
+    web_options.wport = options.wport
+    return web_options, proxy_config
+
+
+def mitmweb():  # pragma: nocover
+    from . import web
+
+    check_versions()
+    web_options, proxy_config = mitmweb_cmdline()
+    server = get_server(web_options.no_server, proxy_config)
+
+    m = web.WebMaster(server, web_options)
+    try:
+        m.run()
+    except KeyboardInterrupt:
+        pass

libmproxy/onboarding/app.py

@@ -0,0 +1,29 @@
+from __future__ import absolute_import
+import flask
+import os
+from ..proxy import config
+
+mapp = flask.Flask(__name__)
+mapp.debug = True
+
+
+def master():
+    return flask.request.environ["mitmproxy.master"]
+
+
+@mapp.route("/")
+def index():
+    return flask.render_template("index.html", section="home")
+
+
+@mapp.route("/cert/pem")
+def certs_pem():
+    p = os.path.join(master().server.config.confdir, config.CONF_BASENAME + "-ca-cert.pem")
+    return flask.Response(open(p, "rb").read(), mimetype='application/x-x509-ca-cert')
+
+
+@mapp.route("/cert/p12")
+def certs_p12():
+    p = os.path.join(master().server.config.confdir, config.CONF_BASENAME + "-ca-cert.p12")
+    return flask.Response(open(p, "rb").read(), mimetype='application/x-pkcs12')
+