Minor tweaks to release checlist and binary builds

These are commonly symlinked during dev

.coveragerc

@@ -2,5 +2,5 @@
 branch = True
 
 [report]
-omit = *contrib*, *tnetstring*, *platform*, *console*
+omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py
 include = *libmproxy*

.gitattributes

@@ -0,0 +1,2 @@
+libmproxy/web/static/**/* -diff
+web/src/js/filt/filt.js -diff

.gitignore

@@ -1,14 +1,27 @@
+.DS_Store
 MANIFEST
 /build
 /dist
 /tmp
 /doc
 /venv
+/libmproxy/gui
+/release/build
 *.py[cdo]
 *.swp
 *.swo
+mitmproxy.egg-info/
 mitmproxyc
 mitmdumpc
 .coverage
+.idea
 netlib
+pathod
 libpathod
+
+# UI
+
+node_modules
+bower_components
+*.compiled.js
+*.map

.travis.yml

@@ -0,0 +1,23 @@
+language: python
+sudo: false
+python:
+  - "2.7"
+  - pypy
+# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
+install:
+  - "pip install --upgrade --src . -r requirements.txt"
+# command to run tests, e.g. python setup.py test
+script: 
+  - "nosetests --with-cov --cov-report term-missing"
+after_success: 
+  - coveralls
+notifications:
+  irc:
+    channels:
+      - "irc.oftc.net#mitmproxy"
+    on_success: change
+    on_failure: always
+cache:
+  directories:
+  - /home/travis/virtualenv/python2.7.9/lib/python2.7/site-packages
+  - /home/travis/virtualenv/pypy-2.4.0/site-packages

CHANGELOG

@@ -1,3 +1,100 @@
+29 Dec 2014: mitmproxy 0.11.2:
+
+    * Configuration files - mitmproxy.conf, mitmdump.conf, common.conf in the
+      .mitmproxy directory.
+    * Better handling of servers that reject connections that are not SNI.
+    * Many other small bugfixes and improvements.
+
+
+15 November 2014: mitmproxy 0.11.1:
+
+    * Bug fixes: connection leaks some crashes 
+
+
+7 November 2014: mitmproxy 0.11:
+    
+    * Performance improvements for mitmproxy console
+
+    * SOCKS5 proxy mode allows mitmproxy to act as a SOCKS5 proxy server
+
+    * Data streaming for response bodies exceeding a threshold
+      (bradpeabody@gmail.com)
+
+    * Ignore hosts or IP addresses, forwarding both HTTP and HTTPS traffic
+      untouched
+
+    * Finer-grained control of traffic replay, including options to ignore
+      contents or parameters when matching flows (marcelo.glezer@gmail.com)
+
+    * Pass arguments to inline scripts 
+
+    * Configurable size limit on HTTP request and response bodies
+
+    * Per-domain specification of interception certificates and keys (see
+      --cert option)
+
+    * Certificate forwarding, relaying upstream SSL certificates verbatim (see
+      --cert-forward)
+
+    * Search and highlighting for HTTP request and response bodies in
+      mitmproxy console (pedro@worcel.com)
+
+    * Transparent proxy support on Windows
+
+    * Improved error messages and logging
+
+    * Support for FreeBSD in transparent mode, using pf (zbrdge@gmail.com)
+
+    * Content view mode for WBXML (davidshaw835@air-watch.com)
+
+    * Better documentation, with a new section on proxy modes
+
+    * Generic TCP proxy mode
+
+    * Countless bugfixes and other small improvements
+
+
+
+28 January 2014: mitmproxy 0.10:
+	
+    * Support for multiple scripts and multiple script arguments
+
+    * Easy certificate install through the in-proxy web app, which is now
+      enabled by default
+
+    * Forward proxy mode, that forwards proxy requests to an upstream HTTP server
+
+    * Reverse proxy now works with SSL
+
+    * Search within a request/response using the "/" and "n" shortcut keys
+
+    * A view that beatifies CSS files if cssutils is available
+
+    * Bug fix, documentation improvements, and more.
+
+
+25 August 2013: mitmproxy 0.9.2:
+
+    * Improvements to the mitmproxywrapper.py helper script for OSX.
+    
+    * Don't take minor version into account when checking for serialized file
+      compatibility.
+
+    * Fix a bug causing resource exhaustion under some circumstances for SSL
+      connections.
+
+    * Revamp the way we store interception certificates. We used to store these
+      on disk, they're now in-memory. This fixes a race condition related to
+      cert handling, and improves compatibility with Windows, where the rules
+      governing permitted file names are weird, resulting in errors for some
+      valid IDNA-encoded names.
+
+    * Display transfer rates for responses in the flow list.
+    
+    * Many other small bugfixes and improvements.
+    
+
+
 
 16 June 2013: mitmproxy 0.9.1:
 

CONTRIBUTORS

@@ -1,39 +1,65 @@
-   777	Aldo Cortesi
+   902	Aldo Cortesi
+   323	Maximilian Hils
     18	Henrik Nordstrom
     13	Thomas Roth
+    12	Pedro Worcel
     11	Stephen Altamirano
     10	András Veres-Szentkirályi
      8	Rouli
      8	Jason A. Novak
      7	Alexis Hildebrandt
-     6	Maximilian Hils
+     5	Tomaz Muraus
+     5	Brad Peabody
+     5	Matthias Urlichs
+     4	root
+     4	Marc Liyanage
      4	Valtteri Virtanen
      4	Bryan Bishop
      3	Chris Neasbitt
-     2	Heikki Hannikainen
-     2	Jim Lloyd
-     2	Mark E. Haase
+     3	Zack B
+     3	Eli Shvartsman
+     3	Kyle Manna
      2	Michael Frister
+     2	Bennett Blodinger
+     2	Jim Lloyd
      2	Rob Wills
-     2	alts
      2	israel
-     1	Mathieu Mitchell
-     1	Michael Bisbjerg
-     1	capt8bit
-     1	Nicolas Esteves
-     1	Paul
+     2	Jaime Soriano Pastor
+     2	Heikki Hannikainen
+     2	Mark E. Haase
+     2	alts
+     1	davidpshaw
+     1	deployable
+     1	joebowbeer
+     1	meeee
      1	phil plante
-     1	Rory McCann
+     1	Michael Bisbjerg
+     1	Andy Smith
+     1	Dan Wilbraham
+     1	David Shaw
+     1	Eric Entzel
+     1	Felix Wolfsteller
+     1	Henrik Nordström
+     1	Ivaylo Popov
+     1	JC
      1	Jakub Nawalaniec
+     1	James Billingham
+     1	Jean Regisser
+     1	Kit Randel
+     1	Marcelo Glezer
+     1	Mathieu Mitchell
+     1	Mikhail Korobov
+     1	Nicolas Esteves
+     1	Oleksandr Sheremet
+     1	Paul
+     1	Rich Somerfield
+     1	Rory McCann
      1	Rune Halvorsen
      1	Sahn Lam
-     1	Ivaylo Popov
-     1	Andy Smith
+     1	Seppo Yli-Olli
+     1	Sergey Chipiga
+     1	Steven Van Acker
      1	Ulrich Petri
-     1	Henrik Nordström
-     1	Felix Wolfsteller
+     1	Vyacheslav Bakhmutov
      1	Yuangxuan Wang
-     1	Kit Randel
-     1	Marc Liyanage
-     1	meeee
-     1	Eric Entzel
+     1	capt8bit

MANIFEST.in

@@ -1,11 +1,10 @@
-include LICENSE
-include CHANGELOG
-include CONTRIBUTORS
-include README.txt
-include setup.py
+include mitmproxy mitmdump
+include LICENSE CHANGELOG CONTRIBUTORS README.txt
 exclude README.mkd
 recursive-include examples *
 recursive-include doc *
 recursive-include test *
-recursive-include libmproxy/resources *
-recursive-exclude test *.swo *.swp *.pyc
+recursive-include libmproxy *
+recursive-exclude * *.pyc *.pyo *.swo *.swp
+recursive-exclude netlib *
+recursive-exclude libpathod *

README.mkd

@@ -1,3 +1,5 @@
+[![Build Status](https://travis-ci.org/mitmproxy/mitmproxy.png?branch=master)](https://travis-ci.org/mitmproxy/mitmproxy) [![Coverage Status](https://coveralls.io/repos/mitmproxy/mitmproxy/badge.png?branch=master)](https://coveralls.io/r/mitmproxy/mitmproxy)
+
 __mitmproxy__ is an interactive, SSL-capable man-in-the-middle proxy for HTTP
 with a console interface.
 
@@ -11,6 +13,9 @@ mitmproxy.org website:
 [mitmproxy.org](http://mitmproxy.org).
 
 
+You can find complete directions for installing mitmproxy [here](http://mitmproxy.org/doc/install.html). 
+
+
 Features
 --------
 
@@ -24,39 +29,58 @@ Features
 - SSL certificates for interception are generated on the fly.
 - And much, much more.
 
+__mitmproxy__ is tested and developed on OSX, Linux and OpenBSD. On Windows,
+only mitmdump is supported, which does not have a graphical user interface.
 
-Requirements
-------------
-
-* [Python](http://www.python.org) 2.7.x.
-* [netlib](http://pypi.python.org/pypi/netlib), version matching mitmproxy.
-* [PyOpenSSL](http://pypi.python.org/pypi/pyOpenSSL) 0.13 or newer.
-* [pyasn1](http://pypi.python.org/pypi/pyasn1) 0.1.2 or newer.
-* [urwid](http://excess.org/urwid/) version 1.1 or newer.
-* [PIL](http://www.pythonware.com/products/pil/) version 1.1 or newer.
-* [lxml](http://lxml.de/) version 2.3 or newer.
-* [flask](http://flask.pocoo.org/) version 0.9 or newer.
-
-Optional, for extended content decoding:
-
-* [PyAMF](http://www.pyamf.org/) version 0.6.1 or newer.
-* [protobuf](https://code.google.com/p/protobuf/) version 2.5.0 or newer.
-
-__mitmproxy__ is tested and developed on OSX, Linux and OpenBSD. Windows is not
-officially supported at the moment.
 
 
 Hacking
 -------
 
-The following components are needed if you plan to hack on mitmproxy:
 
-* The test suite uses the [nose](http://readthedocs.org/docs/nose/en/latest/) unit testing
-  framework and requires [pathod](http://pathod.org) and [flask](http://flask.pocoo.org/).
-* Rendering the documentation requires [countershape](http://github.com/cortesi/countershape).
+### Requirements
+
+
+* [Python](http://www.python.org) 2.7.x.
+* [netlib](http://pypi.python.org/pypi/netlib), version matching mitmproxy.
+* Third-party packages listed in [setup.py](https://github.com/mitmproxy/mitmproxy/blob/master/setup.py)
+
+Optional packages for extended content decoding:
+
+* [PyAMF](http://www.pyamf.org/) version 0.6.1 or newer.
+* [protobuf](https://code.google.com/p/protobuf/) version 2.5.0 or newer.
+* [cssutils](http://cthedot.de/cssutils/) version 1.0 or newer.
+
+For convenience, all optional dependencies can be installed with 
+
+`pip install "mitmproxy[contentviews]"`
+
+### Setting up a dev environment
+
+The following procedure is recommended to set up your dev environment:
+
+```
+$ git clone https://github.com/mitmproxy/mitmproxy.git
+$ cd mitmproxy
+$ pip install --src . -r requirements.txt
+```
+
+This installs the latest GitHub versions of mitmproxy, netlib and pathod into `mitmproxy/`. All other development dependencies save countershape are installed into their usual locations. 
+
+
+### Testing
+
+The test suite requires the `dev` extra requirements listed in [setup.py](https://github.com/mitmproxy/mitmproxy/blob/master/setup.py) and [pathod](http://pathod.net), version matching mitmproxy. Install these with:
+
+`pip install "mitmproxy[dev]"`
 
-For convenience, all dependencies save countershape, can be installed from pypi to a virtualenv with 'pip install -r requirements.txt'.
 
 Please ensure that all patches are accompanied by matching changes in the test
 suite. The project maintains 100% test coverage.
 
+
+### Docs
+
+Rendering the documentation requires [countershape](http://github.com/cortesi/countershape). After installation, you can render the documentation to the doc like this:
+
+`cshape doc-src doc`

doc-src/02-docstyle.css

@@ -10,3 +10,11 @@ body {
 .nowrap {
      white-space: nowrap;
 }
+
+h1 {
+    line-height: 1.1;
+}
+
+.page-header {
+    margin: 0px 0 22px;
+}

doc-src/_layout.html

@@ -6,7 +6,7 @@
         <span class="icon-bar"></span>
         <span class="icon-bar"></span>
       </a>
-      <a class="brand" href="@!urlTo(idxpath)!@">mitmproxy 0.9 docs</a>
+      <a class="brand" href="@!urlTo(idxpath)!@">mitmproxy $!VERSION!$ docs</a>
       </div><!--/.nav-collapse -->
     </div>
   </div>
@@ -16,53 +16,7 @@
   <div class="row">
     <div class="span3">
       <div class="well sidebar-nav">
-        <ul class="nav nav-list">
-            $!nav(idxpath, this, state)!$
-            $!nav("install.html", this, state)!$
-            $!nav("howmitmproxy.html", this, state)!$
-
-            <li class="nav-header">Tools</li>
-                $!nav("mitmproxy.html", this, state)!$
-                $!nav("mitmdump.html", this, state)!$
-
-            <li class="nav-header">Features</li>
-                $!nav("anticache.html", this, state)!$
-                $!nav("clientreplay.html", this, state)!$
-                $!nav("filters.html", this, state)!$
-                $!nav("proxyauth.html", this, state)!$
-                $!nav("replacements.html", this, state)!$
-                $!nav("serverreplay.html", this, state)!$
-                $!nav("setheaders.html", this, state)!$
-                $!nav("sticky.html", this, state)!$
-                $!nav("reverseproxy.html", this, state)!$
-                $!nav("upstreamcerts.html", this, state)!$
-
-            <li class="nav-header">Installing Certificates</li>
-                $!nav("ssl.html", this, state)!$
-                $!nav("certinstall/firefox.html", this, state)!$
-                $!nav("certinstall/osx.html", this, state)!$
-                $!nav("certinstall/windows7.html", this, state)!$
-                $!nav("certinstall/ios.html", this, state)!$
-                $!nav("certinstall/ios-simulator.html", this, state)!$
-                $!nav("certinstall/android.html", this, state)!$
-
-            <li class="nav-header">Transparent Proxying</li>
-                $!nav("transparent.html", this, state)!$
-                $!nav("transparent/linux.html", this, state)!$
-                $!nav("transparent/osx.html", this, state)!$
-
-             <li class="nav-header">Tutorials</li>
-                $!nav("tutorials/30second.html", this, state)!$
-                $!nav("tutorials/gamecenter.html", this, state)!$
-
-            <li class="nav-header">Scripting mitmproxy</li>
-                $!nav("scripting/inlinescripts.html", this, state)!$
-                $!nav("scripting/libmproxy.html", this, state)!$
-
-            <li class="nav-header">Hacking</li>
-                $!nav("dev/testing.html", this, state)!$
-
-        </ul>
+            $!navbar!$
       </div>
     </div>
 

doc-src/_nav.html

@@ -0,0 +1,59 @@
+<ul class="nav nav-list">
+    $!nav(idxpath, this, state)!$
+    $!nav("install.html", this, state)!$
+    $!nav("howmitmproxy.html", this, state)!$
+    $!nav("modes.html", this, state)!$
+
+    <li class="nav-header">Tools</li>
+        $!nav("mitmproxy.html", this, state)!$
+        $!nav("mitmdump.html", this, state)!$
+        $!nav("config.html", this, state)!$
+
+    <li class="nav-header">Features</li>
+        $!nav("anticache.html", this, state)!$
+
+        $!nav("filters.html", this, state)!$
+        $!nav("replacements.html", this, state)!$
+        $!nav("clientreplay.html", this, state)!$
+        $!nav("serverreplay.html", this, state)!$
+        $!nav("setheaders.html", this, state)!$
+        $!nav("passthrough.html", this, state)!$
+        $!nav("proxyauth.html", this, state)!$
+        $!nav("reverseproxy.html", this, state)!$
+        $!nav("responsestreaming.html", this, state)!$
+        $!nav("socksproxy.html", this, state)!$
+        $!nav("sticky.html", this, state)!$
+        $!nav("tcpproxy.html", this, state)!$
+        $!nav("upstreamproxy.html", this, state)!$
+        $!nav("upstreamcerts.html", this, state)!$
+
+
+    <li class="nav-header">Installing Certificates</li>
+        $!nav("ssl.html", this, state)!$
+        $!nav("certinstall/webapp.html", this, state)!$
+        $!nav("certinstall/android.html", this, state)!$
+        $!nav("certinstall/firefox.html", this, state)!$
+        $!nav("certinstall/ios.html", this, state)!$
+        $!nav("certinstall/ios-simulator.html", this, state)!$
+        $!nav("certinstall/java.html", this, state)!$
+        $!nav("certinstall/osx.html", this, state)!$
+        $!nav("certinstall/windows7.html", this, state)!$
+
+    <li class="nav-header">Transparent Proxying</li>
+        $!nav("transparent.html", this, state)!$
+        $!nav("transparent/linux.html", this, state)!$
+        $!nav("transparent/osx.html", this, state)!$
+
+    <li class="nav-header">Scripting mitmproxy</li>
+        $!nav("scripting/inlinescripts.html", this, state)!$
+        $!nav("scripting/libmproxy.html", this, state)!$
+
+    <li class="nav-header">Tutorials</li>
+        $!nav("tutorials/30second.html", this, state)!$
+        $!nav("tutorials/gamecenter.html", this, state)!$
+        $!nav("tutorials/transparent-dhcp.html", this, state)!$
+
+    <li class="nav-header">Hacking</li>
+        $!nav("dev/architecture.html", this, state)!$
+        $!nav("dev/testing.html", this, state)!$
+</ul>

doc-src/_websitelayout.html

@@ -6,7 +6,7 @@
         <span class="icon-bar"></span>
         <span class="icon-bar"></span>
       </a>
-      <a class="brand" href="@!urlTo("/index.html")!@">mitmproxy</a>
+      <a class="brand" href="@!urlTo(idxpath)!@">mitmproxy</a>
       <div class="nav">
         <ul class="nav">
           <li $!'class="active"' if this.match("/index.html", True) else ""!$> <a href="@!top!@/index.html">home</a> </li>
@@ -23,48 +23,7 @@
 
     <div class="span3">
       <div class="well sidebar-nav">
-        <ul class="nav nav-list">
-            $!nav(idxpath, this, state)!$
-            $!nav("install.html", this, state)!$
-            $!nav("howmitmproxy.html", this, state)!$
-
-            <li class="nav-header">Tools</li>
-                $!nav("mitmproxy.html", this, state)!$
-                $!nav("mitmdump.html", this, state)!$
-
-            <li class="nav-header">Features</li>
-                $!nav("anticache.html", this, state)!$
-                $!nav("clientreplay.html", this, state)!$
-                $!nav("filters.html", this, state)!$
-                $!nav("proxyauth.html", this, state)!$
-                $!nav("replacements.html", this, state)!$
-                $!nav("serverreplay.html", this, state)!$
-                $!nav("setheaders.html", this, state)!$
-                $!nav("sticky.html", this, state)!$
-                $!nav("reverseproxy.html", this, state)!$
-                $!nav("upstreamcerts.html", this, state)!$
-
-            <li class="nav-header">SSL interception</li>
-                $!nav("ssl.html", this, state)!$
-                $!nav("certinstall/firefox.html", this, state)!$
-                $!nav("certinstall/osx.html", this, state)!$
-                $!nav("certinstall/windows7.html", this, state)!$
-                $!nav("certinstall/ios.html", this, state)!$
-                $!nav("certinstall/android.html", this, state)!$
-
-            <li class="nav-header">Transparent Proxying</li>
-                $!nav("transparent.html", this, state)!$
-                $!nav("transparent/linux.html", this, state)!$
-                $!nav("transparent/osx.html", this, state)!$
-
-             <li class="nav-header">Tutorials</li>
-                $!nav("tutorials/30second.html", this, state)!$
-                $!nav("tutorials/gamecenter.html", this, state)!$
-
-            <li class="nav-header">Scripting mitmproxy</li>
-                $!nav("scripting/inlinescripts.html", this, state)!$
-                $!nav("scripting/libmproxy.html", this, state)!$
-        </ul>
+            $!navbar!$
       </div>
     </div>
     <div class="span9">

doc-src/certinstall/android.html

@@ -1,4 +1,3 @@
-
 The proxy situation on Android is [an
 embarrasment](http://code.google.com/p/android/issues/detail?id=1273).  It's
 scarcely credible, but Android didn't have a global proxy setting at all until
@@ -8,24 +7,26 @@ necessity, and many apps merrily ignore it even if it's there. This situation
 is improving, but in many circumstances using [transparent
 mode](@!urlTo("transparent.html")!@) is mandatory for testing Android apps.
 
-We used an Asus Transformer Prime TF201 with Android 4.0.3 in the examples
-below - your device may differ, but the broad process should be similar. 
+We used both an Asus Transformer Prime TF201 (Android 4.0.3) and a Nexus 4
+(Android 4.4.4) in the examples below - your device may differ, but the broad
+process should be similar.  On **emulated devices**, there are some [additional
+quirks](https://github.com/mitmproxy/mitmproxy/issues/204#issuecomment-32837093)
+to consider.
 
 
 ## Getting the certificate onto the device
 
-First we need to get the __mitmproxy-ca-cert.cer__ file into the
-__/sdcard/Downloads__ folder on the device. There are a number of ways to do
-this. If you have the Android Developer Tools installed, you can use [__adb
-push__](http://developer.android.com/tools/help/adb.html) to accomplish this.
-Depending on your device, you could also transfer the file using external media
-like an SD Card. In this example, we're using wget from within a terminal
-emulator  to transfer the certificate from a local HTTP server: 
+The easiest way to get the certificate to the device is to use [the web
+app](@!urlTo("webapp.html")!@). In the rare cases where the web app doesn't
+work, you will need to get the __mitmproxy-ca-cert.cer__ file into the
+__/sdcard__ folder on the device (/sdcard/Download on older devices). This can
+be accomplished in a number of ways:
 
-<img src="android-shellwgetmitmproxyca.png"/>
-
-
-## Installing the certificate
+- If you have the Android Developer Tools installed, you can use [__adb
+push__](http://developer.android.com/tools/help/adb.html).
+- Using a file transfer program like wget (installed on the Android device) to
+copy the file over.
+- Transfer the file using external media like an SD Card. 
 
 Once we have the certificate on the local disk, we need to import it into the
 list of trusted CAs. Go to Settings -&gt; Security -&gt; Credential Storage,
@@ -33,14 +34,20 @@ and select "Install from storage":
 
 <img src="android-settingssecuritymenu.png"/>
 
-The certificate in /sdcard/Downloads is automatically located and offered for
+The certificate in /sdcard is automatically located and offered for
 installation. Installing the cert will delete the download file from the local
-disk:
+disk.
+
+
+## Installing the certificate
+
+You should now see something like this (you may have to explicitly name the
+certificate):
 
 <img src="android-settingssecurityinstallca.png"/>
 
-Afterwards, you should see the certificate listed in the Trusted Credentials
-store:
+Click OK, and you should then see the certificate listed in the Trusted
+Credentials store:
 
 <img src="android-settingssecurityuserinstalledca.png"/>
 

doc-src/certinstall/firefox.html

@@ -1,10 +1,13 @@
+## Get the certificate to the browser
+
+The easiest way to get the certificate to the browser is to use [the web
+app](@!urlTo("webapp.html")!@). If this fails, do the following:
 
-How to install the __mitmproxy__ certificate authority in Firefox:
 
 <ol class="tlist">
     <li> If needed, copy the ~/.mitmproxy/mitmproxy-ca-cert.pem file to the target. </li>
 
-    <li>Open preferences, click on "Advanced", then select"Encryption":
+    <li>Open preferences, click on "Advanced", then select"Certificates":
         <img src="@!urlTo('firefox3.jpg')!@"/>
     </li>
 
@@ -12,12 +15,17 @@ How to install the __mitmproxy__ certificate authority in Firefox:
         <img src="@!urlTo('firefox3-import.jpg')!@"/>
     </li>
 
-    <li>Tick "Trust this CS to identify web sites", and click "Ok":
+</ol>
+
+
+## Installing the certificate
+
+<ol class="tlist">
+    <li>Tick "Trust this CA to identify web sites", and click "Ok":
         <img src="@!urlTo('firefox3-trust.jpg')!@"/>
     </li>
 
     <li> You should now see the mitmproxy certificate listed in the Authorities
     tab.</li>
-
 </ol>
 

doc-src/certinstall/index.py

@@ -1,10 +1,13 @@
 from countershape import Page
 
 pages = [
+    Page("webapp.html", "Using the Web App"),
     Page("firefox.html", "Firefox"),
     Page("osx.html", "OSX"),
     Page("windows7.html", "Windows 7"),
     Page("ios.html", "IOS"),
     Page("ios-simulator.html", "IOS Simulator"),
     Page("android.html", "Android"),
+    Page("java.html", "Java"),
+    Page("mitm.it-error.html", "Error: No proxy configured"),
 ]

doc-src/certinstall/ios.html

@@ -1,11 +1,17 @@
 
-How to install the __mitmproxy__ certificate authority on IOS devices:
+## Getting the certificate onto the device
+
+The easiest way to get the certificate to the device is to use [the web
+app](@!urlTo("webapp.html")!@). In the rare cases where the web app doesn't
+work, you will need to get the __mitmproxy-ca-cert.pem__ file to the device to
+install it. The easiest way to accomplish this is to set up the Mail app on the
+device, and to email it over as an attachment. Open the email, tap on the
+attachment, then proceed with the install.
+
+
+## Installing the certificate
 
 <ol class="tlist">
-    <li>Set up the Mail app on the device to receive email.</li>
-
-    <li>Mail the mitmproxy-ca-cert.pem file to the device, and tap on the attachment.</li>
-
     <li>You will be prompted to install a profile. Click "Install":
 
     <img src="@!urlTo('ios-profile.png')!@"/></li>

doc-src/certinstall/java.html

@@ -0,0 +1,13 @@
+
+You can add the mitmproxy certificates to the Java trust store using
+[keytool](http://docs.oracle.com/javase/6/docs/technotes/tools/solaris/keytool.html).
+On OSX, the required command looks like this:
+
+<pre class="terminal">
+sudo keytool -importcert -alias mitmproxy -storepass "password" \
+-keystore /System/Library/Java/Support/CoreDeploy.bundle/Contents/Home/lib/security/cacerts \
+-trustcacerts -file ~/.mitmproxy/mitmproxy-ca-cert.pem
+</pre>
+
+Note that your store password will (hopefully) be different from the one above.
+

doc-src/certinstall/mitm.it-error.html

@@ -0,0 +1,5 @@
+**Looks like you wanted to install the mitmproxy CA using the web app?**
+
+Unfortunately, there's been no mitmproxy instance on the wire that could have intercepted your request.
+Please configure your client to use mitmproxy and try again.<br>
+The request to <a href="http://mitm.it/">http://mitm.it/</a> must go through your mitmproxy instance.

doc-src/certinstall/webapp.html

@@ -0,0 +1,13 @@
+
+By far the easiest way to install the mitmproxy certs is to use the built-in
+web app. To do this, start mitmproxy and configure your target device with the
+correct proxy settings. Now start a browser on the device, and visit the magic
+domain **mitm.it**. You should see something like this:
+
+<img src="@!urlTo("webapp.png")!@"></img>
+
+Just click on the relevant icon, and then follow the setup instructions
+for the platform you're on.
+
+Make sure you aren't using a bandwith optimizer (like Google's Data Compression
+Proxy on Chrome for Android) or the page will not load.

doc-src/certinstall/windows7.html

@@ -3,10 +3,13 @@ How to install the __mitmproxy__ certificate authority in Windows 7:
 
 <ol class="tlist">
 
-    <li> Copy the ~/.mitmproxy/mitmproxy-ca-cert.p12 file to the target system. </li>
+    <li> The easiest way to get the certificate to the device is to use <a
+        href="@!urlTo("webapp.html")!@">the web app</a>. If this fails for some
+    reason, simply copy the ~/.mitmproxy/mitmproxy-ca-cert.p12 file to the
+    target system and double-click it.  </li>
 
     <li> 
-        Double-click the certificate file. You should see a certificate import wizard: 
+        You should see a certificate import wizard: 
 
         <img src="@!urlTo('win7-wizard.png')!@"/>
     </li>

doc-src/config.html

@@ -0,0 +1,86 @@
+
+
+Mitmproxy is configured through a set of files in the users ~/.mitmproxy
+directory. 
+
+<table class="table">
+	<tbody>
+		<tr>
+			<th>mitmproxy.conf</th> 
+			<td>Settings for the <b>mitmproxy</b>. This file can contain any options supported by mitmproxy.</td>
+		</tr>
+		<tr>
+			<th>mitmdump.conf</th> 
+			<td>Settings for the <b>mitmdump</b>. This file can contain any options supported by mitmdump.</td>
+		</tr>
+		<tr>
+			<th>common.conf</th>
+
+			<td>Settings shared between all command-line tools. Settings in
+			this file are over-ridden by those in the tool-specific
+			files. Only options shared by mitmproxy and mitmdump should be used in this file. </td>
+		</tr>
+	</tbody>
+</table>
+
+# Syntax
+
+## Comments
+
+<pre>
+# this is a comment
+; this is also a comment (.ini style)
+--- and this is a comment too (yaml style)
+</pre>
+
+## Key/Value pairs
+
+- Keys and values are case-sensitive
+- Whitespace is ignored
+- Lists are comma-delimited, and enclosed in square brackets
+
+<pre>
+name = value   # (.ini style) 
+name: value    # (yaml style)
+--name value   # (command-line option style)
+
+fruit = [apple, orange, lemon]
+indexes = [1, 12, 35 , 40]
+</pre>
+
+## Flags
+
+These are boolean options that take no value but true/false.
+
+<pre>
+name = true    # (.ini style)
+name
+--name 	 	   # (command-line option style)
+</pre>
+
+# Options
+
+The options available in the config files are precisely those available as
+command-line flags, with the key being the option's long name. To get a
+complete list of these, use the __--help__ option on each of the tools. Be
+careful to only specify common options in the __common.conf__ file -
+unsupported options in this file will be detected as an error on startup.
+
+# Examples
+
+## common.conf
+
+Note that __port__ is an option supported by all tools. 
+
+<pre class="code">
+port = 8080
+</pre>
+
+
+## mitmproxy.conf
+
+<pre class="code">
+palette = light
+</pre>
+
+

doc-src/custom-routing.txt

@@ -0,0 +1,24 @@
+# Adapted from http://tldp.org/HOWTO/TransparentProxy-6.html (6.2 Second method)
+# Note that the choice of firewall mark (3) and routing table (2) was fairly arbitrary.
+# If you are already using policy routing or firewall marking for some other purpose,
+# make sure you choose unique numbers here. Otherwise, don't worry about it.
+
+
+
+# On the router, run
+
+PROXY_IP=192.168.1.100
+TARGET_IP=192.168.1.110
+
+iptables -t mangle -A PREROUTING -j ACCEPT -p tcp -m multiport --dports 80,443 -s ! $TARGET_IP
+# Alternative to MITM the whole network:
+# iptables -t mangle -A PREROUTING -j ACCEPT -p tcp -m multiport --dports 80,443 -s  $PROXY_IP
+iptables -t mangle -A PREROUTING -j MARK --set-mark 3 -p tcp -m multiport --dports 80,443
+ip rule add fwmark 3 table 2
+ip route add default via $PROXY_IP dev br0 table 2
+
+
+
+# On the proxy machine, run
+
+iptables -A PREROUTING -t nat -i eth0 -p tcp -m multiport --dports 80,443 -j REDIRECT --to-port 8080

doc-src/dev/architecture.html

@@ -0,0 +1,8 @@
+To give you a better understanding of how mitmproxy works, mitmproxy's high-level architecture is detailed
+in the following graphic:
+
+<img src="@!urlTo('schematics/architecture.png')!@">
+
+<a href="@!urlTo('schematics/architecture.pdf')!@">(architecture.pdf)</a>
+<p>Please don't refrain from asking any further
+questions on the mailing list, the IRC channel or the GitHub issue tracker.</p>

doc-src/dev/index.py

@@ -2,5 +2,6 @@ from countershape import Page
 
 pages = [
      Page("testing.html", "Testing"),
+     Page("architecture.html", "Architecture"),
 #    Page("addingviews.html", "Writing Content Views"),
 ]

doc-src/features/index.py

@@ -4,11 +4,16 @@ pages = [
     Page("anticache.html", "Anticache"),
     Page("clientreplay.html", "Client-side replay"),
     Page("filters.html", "Filter expressions"),
+    Page("passthrough.html", "Ignore Domains"),
+    Page("proxyauth.html", "Proxy Authentication"),
+    Page("replacements.html", "Replacements"),
+    Page("responsestreaming.html", "Response Streaming"),
+    Page("reverseproxy.html", "Reverse proxy mode"),
+    Page("socksproxy.html", "SOCKS Mode"),
     Page("setheaders.html", "Set Headers"),
     Page("serverreplay.html", "Server-side replay"),
     Page("sticky.html", "Sticky cookies and auth"),
-    Page("proxyauth.html", "Proxy Authentication"),
-    Page("replacements.html", "Replacements"),
-    Page("reverseproxy.html", "Reverse proxy mode"),
+    Page("tcpproxy.html", "TCP Proxy"),
     Page("upstreamcerts.html", "Upstream Certs"),
-]
+    Page("upstreamproxy.html", "Upstream proxy mode"),
+]

doc-src/features/passthrough.html

@@ -0,0 +1,81 @@
+There are two main reasons why you may want to exempt some traffic from mitmproxy's interception mechanism:
+
+- **Certificate pinning:** Some traffic is is protected using
+  [certificate pinning](https://security.stackexchange.com/questions/29988/what-is-certificate-pinning) and mitmproxy's
+  interception leads to errors. For example, Windows Update or the Apple App Store fail to work if mitmproxy is active.
+- **Convenience:** You really don't care about some parts of the traffic and just want them to go away.
+
+If you want to peek into (SSL-protected) non-HTTP connections, check out the [tcp proxy](@!urlTo("tcpproxy.html")!@) feature.
+If you want to ignore traffic from mitmproxy's processing because of large response bodies, take a look at the
+[response streaming](@!urlTo("responsestreaming.html")!@) feature.
+
+## How it works
+
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>--ignore regex</td>
+        </tr>
+        <tr>
+            <th>mitmproxy shortcut</th> <td><b>I</b></td>
+        </tr>
+    </tbody>
+</table>
+
+
+mitmproxy allows you to specify a regex which is matched against a <code>host:port</code> string (e.g. "example.com:443")
+to determine hosts that should be excluded. 
+
+There are two important quirks to consider:
+
+- **In transparent mode, the ignore pattern is matched against the IP.** While we usually infer the hostname from the 
+  Host header if the --host argument is passed to mitmproxy, we do not have access to this information before the SSL
+  handshake.
+- In regular mode, explicit HTTP requests are never ignored.[^explicithttp] The ignore pattern is applied on CONNECT
+  requests, which initiate HTTPS or clear-text WebSocket connections.
+
+
+### Tutorial
+
+If you just want to ignore one specific domain, there's usually a bulletproof method to do so:
+
+1. Run mitmproxy or mitmdump in verbose mode (-v) and observe the host:port information in the serverconnect 
+messages. mitmproxy will filter on these.
+2. Take the host:port string, surround it with ^ and $, escape all dots (. becomes \\.) 
+and use this as your ignore pattern:
+
+<pre class="terminal">
+$ mitmdump -v
+127.0.0.1:50588: clientconnect
+127.0.0.1:50588: request
+  -> CONNECT example.com:443 HTTP/1.1
+127.0.0.1:50588: Set new server address: example.com:443
+<span style="color: white">127.0.0.1:50588: serverconnect
+  -> example.com:443</span>
+^C
+$ <span style="color: white">mitmproxy --ignore ^example\.com:443$</span>    
+</pre>
+
+Here are some other examples for ignore patterns:
+<pre>
+# Exempt traffic from the iOS App Store (usually just works):
+--ignore apple.com:443  
+# "Correct" version without false-positives:
+--ignore ^(.+\.)?apple\.com:443$ 
+    
+# Ignore example.com on all ports, but no subdomains:
+--ignore ^example.com:
+    
+# Transparent mode:
+--ignore 17\.178\.96\.59:443
+# IP address range:
+--ignore 17\.178\.\d+\.\d+:443
+</pre>
+
+### See Also
+
+- [TCP Proxy](@!urlTo("tcpproxy.html")!@)
+- [Response Streaming](@!urlTo("responsestreaming.html")!@)
+
+[^explicithttp]: This stems from an limitation of explicit HTTP proxying: A single connection can be re-used for multiple target domains - a <code>GET http://example.com/</code> request may be followed by a <code>GET http://evil.com/</code> request on the same connection. If we start to ignore the connection after the first request, we would miss the relevant second one.

doc-src/features/responsestreaming.html

@@ -0,0 +1,54 @@
+By using mitmproxy's streaming feature, response contents can be passed to the client incrementally before they have been fully received by the proxy.
+This is especially useful for large binary files such as videos, where buffering the whole file slows down the client's browser.
+
+By default, mitmproxy will read the entire response, perform any indicated
+manipulations on it and then send the (possibly modified) response to
+the client. In some cases this is undesirable and you may wish to "stream"
+the reponse back to the client. When streaming is enabled, the response is
+not buffered on the proxy but directly sent back to the client instead.
+
+<h2>On the command-line</h2>
+
+Streaming can be enabled on the command line for all response bodies exceeding a certain size.  The SIZE argument understands
+k/m/g suffixes, e.g. 3m for 3 megabytes.
+
+<table class="table">
+    <tbody>
+    <tr>
+        <th width="20%">command-line</th>
+        <td>
+            --stream SIZE
+        </td>
+    </tr>
+    </tbody>
+</table>
+
+
+<h2>Caveats</h2>
+
+When response streaming is enabled, <strong>streamed response contents will not be
+    recorded or preserved in any way.</strong>
+
+When response streaming is enabled, the response body cannot be modified.
+
+<h2>Customizing Response Streaming</h2>
+
+You can also use an <a href="@!urlTo("scripting/inlinescripts.html")!@">inline script</a> to customize exactly
+which responses are streamed.
+
+Responses that should be tagged for streaming by setting their respective .stream attribute to True:
+
+$!example("examples/stream.py")!$
+
+
+<h2>Implementation Details</h2>
+
+When response streaming is enabled, portions of the code which would have otherwise performed changes
+on the response body will see an empty response body instead (<code>libmproxy.protocol.http.CONTENT_MISSING</code>). Any modifications will be ignored.
+
+Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a <code>Transfer-Encoding:
+    chunked</code> header, the response will be streamed one chunk at a time.
+
+### See Also
+
+- [Ignore Domains](@!urlTo("passthrough.html")!@)

doc-src/features/reverseproxy.html

@@ -1,17 +1,52 @@
 
-In reverse proxy mode, mitmproxy acts as a standard HTTP server and forwards
-all requests to the specified upstream server. Note that the displayed URL for
-flows in this mode will use the value of the __Host__ header field from the
-request, not the reverse proxy server.
-
+In reverse proxy mode, mitmproxy accepts standard HTTP requests and forwards
+them to the specified upstream server. This is in contrast to
+<a href="@!urlTo("upstreamproxy.html")!@">upstream proxy mode</a>, in which
+mitmproxy forwards HTTP proxy requests to an upstream proxy server.
 
 <table class="table">
     <tbody>
         <tr>
-            <th width="20%">command-line</th> <td>-P http[s]://hostname[:port]</td>
-        </tr>
-        <tr>
-            <th>mitmproxy shortcut</th> <td><b>P</b></td>
+            <th width="20%">command-line</th> <td>-R <i>schema</i>://hostname[:port]</td>
         </tr>
     </tbody>
 </table>
+
+Here, **schema** is one of http, https, http2https or https2http. The latter
+two extended schema specifications control the use of HTTP and HTTPS on
+mitmproxy and the upstream server. You can indicate that mitmproxy should use
+HTTP, and the upstream server uses HTTPS like this:
+
+	http2https://hostname:port
+
+And you can indicate that mitmproxy should use HTTPS while the upstream
+service uses HTTP like this:
+
+	https2http://hostname:port
+
+
+### Host Header
+
+In reverse proxy mode, mitmproxy does not rewrite the host header. While often useful, this
+may lead to issues with public web servers. For example, consider the following scenario:
+
+    $ python mitmdump -d -R http://example.com/ &
+    $ curl http://localhost:8080/
+
+    >> GET https://example.com/
+        Host: localhost:8080
+        User-Agent: curl/7.35.0
+        [...]
+
+    << 404 Not Found 345B
+
+Since the Host header doesn't match <samp>example.com</samp>, an error is returned.<br>
+There are two ways to solve this:
+<ol>
+    <li>Modify the hosts file of your OS so that example.com resolves to 127.0.0.1.</li>
+    <li>
+        Instruct mitmproxy to rewrite the host header by passing <kbd>&#8209;&#8209;setheader&nbsp;:~q:Host:example.com</kbd>.
+        However, keep in mind that absolute URLs within the returned document or HTTP redirects will cause the client application
+        to bypass the proxy.
+    </li>
+</ol>

doc-src/features/socksproxy.html

@@ -0,0 +1,10 @@
+
+In this mode, mitmproxy acts as a SOCKS5 proxy server.
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>--socks</td>
+        </tr>
+    </tbody>
+</table>

doc-src/features/tcpproxy.html

@@ -0,0 +1,30 @@
+WebSockets or other non-HTTP protocols are not supported by mitmproxy yet. However, you can exempt hostnames from
+processing, so that mitmproxy acts as a generic TCP forwarder. This feature is closely related to the
+[ignore domains](@!urlTo("passthrough.html")!@) functionality, but differs in two important aspects:
+
+- The raw TCP messages are printed to the event log.
+- SSL connections will be intercepted.
+
+Please note that message interception or modification are not possible yet.
+If you are not interested in the raw TCP messages, you should use the ignore domains feature.
+
+## How it works
+
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>--tcp HOST</td>
+        </tr>
+        <tr>
+            <th>mitmproxy shortcut</th> <td><b>T</b></td>
+        </tr>
+    </tbody>
+</table>
+
+For a detailed description on the structure of the hostname pattern, please refer to the [Ignore Domains](@!urlTo("passthrough.html")!@) feature.
+
+### See Also
+
+- [Ignore Domains](@!urlTo("passthrough.html")!@)
+- [Response Streaming](@!urlTo("responsestreaming.html")!@)

doc-src/features/upstreamproxy.html

@@ -0,0 +1,27 @@
+
+In this mode, mitmproxy accepts proxy requests and unconditionally forwards all
+requests to a specified upstream proxy server. This is in contrast to <a
+href="@!urlTo("reverseproxy.html")!@">reverse proxy mode</a>, in which
+mitmproxy forwards ordinary HTTP requests to an upstream server.
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>-U http://hostname[:port]</td>
+        </tr>
+    </tbody>
+</table>
+
+Here, **schema** is one of http, https, http2https or https2http. The latter
+two extended schema specifications control the use of HTTP and HTTPS on
+mitmproxy and the upstream server. You can indicate that mitmproxy should use
+HTTP, and the upstream server uses HTTPS like this:
+
+	http2https://hostname:port
+
+And you can indicate that mitmproxy should use HTTPS while the upstream
+service uses HTTP like this:
+
+	https2http://hostname:port
+
+

doc-src/index.html

@@ -1,4 +1,27 @@
 
-@!index_contents!@
+__mitmproxy__ is an interactive, SSL-capable man-in-the-middle proxy for HTTP
+with a console interface.
+
+__mitmdump__ is the command-line version of mitmproxy. Think tcpdump for HTTP.
+
+__libmproxy__ is the library that mitmproxy and mitmdump are built on.
+
+Documentation, tutorials and distribution packages can be found on the
+mitmproxy.org website:
+
+[mitmproxy.org](http://mitmproxy.org).
 
 
+Features
+--------
+
+- Intercept HTTP requests and responses and modify them on the fly.
+- Save complete HTTP conversations for later replay and analysis.
+- Replay the client-side of an HTTP conversations.
+- Replay HTTP responses of a previously recorded server.
+- Reverse proxy mode to forward traffic to a specified server.
+- Transparent proxy mode on OSX and Linux.
+- Make scripted changes to HTTP traffic using Python.
+- SSL certificates for interception are generated on the fly.
+- And much, much more.
+

doc-src/index.py

@@ -1,11 +1,17 @@
-import os, sys
+import os
+import sys
+import datetime
 import countershape
-from countershape import Page, Directory, PythonModule, markup, model
+from countershape import Page, Directory, markup, model
 import countershape.template
-sys.path.insert(0, "..")
-from libmproxy import filt
 
-MITMPROXY_SRC = "~/mitmproxy/mitmproxy"
+MITMPROXY_SRC = os.path.abspath(
+    os.path.expanduser(os.environ.get("MITMPROXY_SRC", ".."))
+)
+sys.path.insert(0, MITMPROXY_SRC)
+from libmproxy import filt, version
+
+ns.VERSION = version.VERSION
 
 if ns.options.website:
     ns.idxpath = "doc/index.html"
@@ -14,51 +20,30 @@ else:
     ns.idxpath = "index.html"
     this.layout = countershape.Layout("_layout.html")
 
-
 ns.title = countershape.template.Template(None, "<h1>@!this.title!@</h1>")
-this.titlePrefix = "mitmproxy 0.9 - "
+this.titlePrefix = "%s - " % version.NAMEVERSION
 this.markup = markup.Markdown(extras=["footnotes"])
 
 ns.docMaintainer = "Aldo Cortesi"
 ns.docMaintainerEmail = "aldo@corte.si"
-ns.copyright = u"\u00a9 mitmproxy project, 2013"
+ns.copyright = u"\u00a9 mitmproxy project, %s" % datetime.date.today().year
+
 
 def mpath(p):
     p = os.path.join(MITMPROXY_SRC, p)
     return os.path.expanduser(p)
 
-ns.index_contents = file(mpath("README.mkd")).read()
 
 def example(s):
     d = file(mpath(s)).read().rstrip()
     extemp = """<div class="example">%s<div class="example_legend">(%s)</div></div>"""
     return extemp%(countershape.template.Syntax("py")(d), s)
+
+
 ns.example = example
 
 
-filt_help = []
-for i in filt.filt_unary:
-    filt_help.append(
-        ("~%s"%i.code, i.help)
-    )
-for i in filt.filt_rex:
-    filt_help.append(
-        ("~%s regex"%i.code, i.help)
-    )
-for i in filt.filt_int:
-    filt_help.append(
-        ("~%s int"%i.code, i.help)
-    )
-filt_help.sort()
-filt_help.extend(
-    [
-        ("!", "unary not"),
-        ("&", "and"),
-        ("|", "or"),
-        ("(...)", "grouping"),
-    ]
-)
-ns.filt_help = filt_help
+ns.filt_help = filt.help
 
 
 def nav(page, current, state):
@@ -69,13 +54,18 @@ def nav(page, current, state):
     p = state.application.getPage(page)
     return pre + '<a href="%s">%s</a></li>'%(model.UrlTo(page), p.title)
 ns.nav = nav
+ns.navbar = countershape.template.File(None, "_nav.html")
+
 
 pages = [
     Page("index.html", "Introduction"),
     Page("install.html", "Installation"),
+    Page("howmitmproxy.html", "How mitmproxy works"),
+    Page("modes.html", "Modes of Operation"),
+
     Page("mitmproxy.html", "mitmproxy"),
     Page("mitmdump.html", "mitmdump"),
-    Page("howmitmproxy.html", "How mitmproxy works"),
+    Page("config.html", "configuration"),
 
     Page("ssl.html", "Overview"),
     Directory("certinstall"),

doc-src/install.html

@@ -1,40 +1,33 @@
 
+## Installing from source
+
 The preferred way to install mitmproxy - whether you're installing the latest
 release or from source - is to use [pip](http://www.pip-installer.org/). If you
 don't already have pip on your system, you can find installation instructions
 [here](http://www.pip-installer.org/en/latest/installing.html).
 
-
-## Installing the latest release
-
-A single command will download and install the latest release of mitmproxy,
-along with all its dependencies:
-
 <pre class="terminal">
 pip install mitmproxy
 </pre>
 
+If you also want to install the optional packages AMF, protobuf and CSS
+content views, do this:
 
-## Installing from source
-
-When installing from source, the easiest method is still to use pip. In this
-case run:
-    
 <pre class="terminal">
-pip install /path/to/source
+pip install "mitmproxy[contentviews]"
 </pre>
 
-Note that if you're installing current git master, you will also have to
-install the current git master of [netlib](http://github.com/mitmproxy/netlib) by
-hand.
 
 ## OSX
 
+The easiest way to get up and running on OSX is to download the pre-built
+binary packages from [mitmproxy.org](http://mitmproxy.org). If you still want
+to install using pip, there are a few things to keep in mind:
+
 - If you're running a Python interpreter installed with homebrew (or similar),
 you may have to install some dependencies by hand. 
 - Make sure that XCode is installed from the App Store, and that the
 command-line tools have been downloaded (XCode/Preferences/Downloads).
-- Now use __pip__ to do the installation, as above.
 
 There are a few bits of customization you might want to do to make mitmproxy
 comfortable to use on OSX. The default color scheme is optimized for a dark
@@ -50,3 +43,17 @@ image/*; /usr/bin/open -Wn %s
 video/*; /usr/bin/open -Wn %s
 </pre>
 
+
+## Ubuntu
+
+On Ubuntu, you will need the following native packages to install mitmproxy
+from source:
+
+- build-essential
+- python-dev
+- libffi-dev
+- libssl-dev
+- libxml2-dev
+- libxslt1-dev
+
+

doc-src/mitmproxy.html

@@ -9,7 +9,7 @@ documentation from any __mitmproxy__ screen.
 
 The flow list shows an index of captured flows in chronological order. 
 
-<img src="@!urlTo("screenshots/mitmproxy.png")!@"/>
+<img src="@!urlTo('screenshots/mitmproxy.png')!@"/>
 
 - __1__: A GET request, returning a 302 Redirect response.
 - __2__: A GET request, returning 16.75kb of text/html data.
@@ -32,7 +32,7 @@ interfaces.
 
 The __Flow View__ lets you inspect and manipulate a single flow:
 
-<img src="@!urlTo("screenshots/mitmproxy-flowview.png")!@"/>
+<img src="@!urlTo('screenshots/mitmproxy-flowview.png')!@"/>
 
 - __1__: Flow summary.
 - __2__: The Request/Response tabs, showing you which part of the flow you are
@@ -65,13 +65,13 @@ At the moment, the Grid Editor is used in four parts of mitmproxy:
 If there is is no data, an empty editor will be started to let you add some.
 Here is the editor showing the headers from a request:
 
-<img src="@!urlTo("screenshots/mitmproxy-kveditor.png")!@"/>
+<img src="@!urlTo('screenshots/mitmproxy-kveditor.png')!@"/>
 
 To edit, navigate to the key or value you want to modify using the arrow or vi
 navigation keys, and press enter. The background color will change to show that
 you are in edit mode for the specified field:
 
-<img src="@!urlTo("screenshots/mitmproxy-kveditor-editmode.png")!@"/>
+<img src="@!urlTo('screenshots/mitmproxy-kveditor-editmode.png')!@"/>
 
 Modify the field as desired, then press escape to exit edit mode when you're
 done. You can also add a row (_a_ key), delete a row (_d_ key), spawn an

doc-src/modes.html

@@ -0,0 +1,222 @@
+
+Mitmproxy has four modes of operation that allow you to use mitmproxy in a
+variety of scenarios:
+
+- **Regular** (the default)
+- **Transparent**
+- **Reverse Proxy**
+- **Upstream Proxy**
+
+Now, which one should you pick? Use this flow chart:
+
+<img src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"/>
+
+<div class="page-header">
+    <h1>Regular Proxy</h1>
+</div>
+
+Mitmproxy's regular mode is the simplest and the easiest to set up.
+
+1. Start mitmproxy.
+2. Configure your client to use mitmproxy. For instance on IOS, the settings might look like <a href="@!urlTo('screenshots/ios-manual.png')!@">this</a>.
+3. Quick Check: You should already be able to visit an unencrypted HTTP site
+through the proxy.
+4. Open the magic domain <strong>mitm.it</strong> and install the certificate for your device.
+
+<div class="well">
+<strong>Heads Up:</strong> Unfortunately, some applications bypass the
+system HTTP proxy settings - Android applications are a common example. In
+these cases, you need to use mitmproxy's transparent mode.
+</div>
+
+If you are proxying an external device, your network will probably look like this:
+
+<img src="@!urlTo('schematics/proxy-modes-regular.png')!@">
+
+The square brackets signify the source and destination IP addresses. Your
+client explicitly connects to mitmproxy and mitmproxy explicitly connects
+to the target server.
+
+<div class="page-header">
+    <h1>Transparent Proxy</h1>
+</div>
+
+In transparent mode, traffic is directed into a proxy at the network layer,
+without any client configuration required. This makes transparent proxying
+ideal for situations where you can't change client behaviour. In the graphic
+below, a machine running mitmproxy has been inserted between the router and
+the internet:
+
+<a href="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
+</a>
+
+The square brackets signify the source and destination IP addresses. Round
+brackets mark the next hop on the *Ethernet/data link* layer. This distinction
+is important: when the packet arrives at the mitmproxy machine, it must still
+be addressed to the target server. This means that Network Address Translation
+should not be applied before the traffic reaches mitmproxy, since this would
+remove the target information, leaving mitmproxy unable to determine the real
+destination.
+
+<a href="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@"></a>
+
+<h2>Common Configurations</h2>
+
+There are many ways to configure your network for transparent proxying. We'll
+look at three common scenarios:
+
+1. Configuring the client to use a custom gateway/router/"next hop"
+2. Implementing custom routing on the router
+
+In most cases, the first option is recommended due to its ease of use.
+
+<h3>(a) Custom Gateway</h3>
+
+One simple way to get traffic to the mitmproxy machine with the destination IP
+intact, is to simply configure the client with the mitmproxy box as the
+default gateway.
+
+<a href="@!urlTo('schematics/proxy-modes-transparent-2.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-transparent-2.png')!@"></a>
+
+In this scenario, we would:
+
+- Configure the proxy machine for transparent mode. You can find instructions
+in the <em>Transparent Proxying</em> section of the mitmproxy docs.
+
+- Configure the client to use the proxy machine's IP as the default gateway.
+<a href="@!urlTo('screenshots/ios-gateway.png')!@">Here</a> is what this would
+look like on IOS.
+
+- Quick Check: At this point, you should already be able to visit an
+unencrypted HTTP site over the proxy.
+
+- Open the magic domain <strong>mitm.it</strong> and install the certificate
+for your device.
+
+Setting the custom gateway on clients can be automated by serving the settings
+out to clients over DHCP. This lets set up an interception network where all
+clients are proxied automatically, which can save time and effort. 
+
+
+<div class="well">
+    <strong style="text-align: center; display: block">Troubleshooting Transparent Mode</strong>
+
+    <p>Incorrect transparent mode configurations are a frequent source of
+    error. If it doesn't work for you, try the following things:</p>
+
+    <ul>
+        <li>
+            Open mitmproxy's event log (press `e`) - do you see clientconnect
+            messages? If not, the packets are not arriving at the proxy. One common
+            cause is the occurrence of ICMP redirects, which means that your
+            machine is telling the client that there's a faster way to the
+            internet by contacting your router directly (see the
+            <em>Transparent Proxying</em> section on how to disable them). If in
+            doubt, <a href="https://wireshark.org/">Wireshark</a> may help you
+            to see whether something arrives at your machine or not.
+        </li>
+        <li>
+            Make sure you have not explicitly configured an HTTP proxy on the
+            client. This is not needed in transparent mode.
+        </li>
+        <li>
+            Re-check the instructions in the <em>Transparent Proxying</em> section. Anything you missed?
+        </li>
+    </ul>
+
+    If you encounter any other pitfalls that should be listed here, please let us know!
+
+</div>
+
+<h3>(b) Custom Routing</h3>
+
+In some cases, you may need more fine-grained control of which traffic reaches
+the mitmproxy instance, and which doesn't. You may, for instance, choose only
+to divert traffic to some hosts into the transparent proxy. There are a huge
+number of ways to accomplish this, and much will depend on the router or
+packet filter you're using. In most cases, the configuration will look like
+this:
+
+<a href="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
+</a>
+
+
+<div class="page-header">
+    <h1>Reverse Proxy</h1>
+</div>
+
+Mitmproxy is usually used with a client that uses the proxy to access the
+Internet. Using reverse proxy mode, you can use mitmproxy to act like a normal
+HTTP server:
+
+<a href="@!urlTo('schematics/proxy-modes-reverse.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-reverse.png')!@">
+</a>
+
+There are various use-cases:
+
+- Say you have an internal API running at http://example.local/. You could now
+set up mitmproxy in reverse proxy mode at http://debug.example.local/ and
+dynamically point clients to this new API endpoint, which provides clients
+with the same data and you with debug information. Similarly, you could move
+your real server to a different IP/port and set up mitmproxy at the original
+place to debug all sessions.
+
+- Say you're a web developer working on example.com (with a development
+version running on localhost:8000). You can modify your hosts file so that
+example.com points to 127.0.0.1 and then run mitmproxy in reverse proxy mode
+on port 80. You can test your app on the example.com domain and get all
+requests recorded in mitmproxy.
+
+- Say you have some toy project that should get SSL support. Simply set up
+mitmproxy with SSL termination and you're done (<code>mitmdump -p 443 -R
+https2http://localhost:80/</code>). There are better tools for this specific
+task, but mitmproxy is very quick and simple way to set up an SSL-speaking
+server.
+
+- Want to add a non-SSL-capable compression proxy in front of your server? You
+could even spawn a mitmproxy instance that terminates SSL (https2http://...),
+point it to the compression proxy and let the compression proxy point to a
+SSL-initiating mitmproxy (http2https://...), which then points to the real
+server. As you see, it's a fairly flexible thing.
+
+Note that mitmproxy supports either an HTTP or an HTTPS upstream server, not
+both at the same time. You can work around this by spawning a second mitmproxy
+instance.
+
+<div class="well">
+    <strong style="text-align: center; display: block">Caveat: Interactive Use</strong>
+
+
+One caveat is that reverse proxy mode is often not sufficient for interactive
+browsing. Consider trying to clone Google by using:
+
+<code>mitmproxy -R http://google.com/</code>
+
+This works for the initial request, but the HTML served to the client remains
+unchanged. As soon as the user clicks on an non-relative URL (or downloads a
+non-relative image resource), traffic no longer passes through mitmproxy, and
+the client connects to Google directly again.
+
+</div>
+
+
+
+<div class="page-header">
+    <h1>Upstream Proxy</h1>
+</div>
+
+If you want to chain proxies by adding mitmproxy in front of a different proxy
+appliance, you can use mitmproxy's upstream mode. In upstream mode, all
+requests are unconditionally transferred to an upstream proxy of your choice.
+
+<a href="@!urlTo('schematics/proxy-modes-upstream.png')!@">
+    <img src="@!urlTo('schematics/proxy-modes-upstream.png')!@"></a>
+
+mitmproxy supports both explicit HTTP and explicit HTTPS in upstream proxy
+mode. You could in theory chain multiple mitmproxy instances in a row, but
+that doesn't make any sense in practice (i.e. outside of our tests). 

doc-src/scripting/inlinescripts.html

@@ -1,5 +1,5 @@
 __mitmproxy__ has a powerful scripting API that allows you to modify flows
-on-the-fly or rewrite previously saved flows locally. 
+on-the-fly or rewrite previously saved flows locally.
 
 The mitmproxy scripting API is event driven - a script is simply a Python
 module that exposes a set of event methods. Here's a complete mitmproxy script
@@ -21,42 +21,59 @@ We can now run this script using mitmdump or mitmproxy as follows:
 
 The new header will be added to all responses passing through the proxy.
 
+## Example Scripts
+
+mitmproxy comes with a variety of example inline scripts, which demonstrate
+many basic tasks. We encourage you to either browse them locally or in our
+[GitHub repo](https://github.com/mitmproxy/mitmproxy/tree/master/examples).
 
 
 ## Events
 
-### start(ScriptContext)
+### start(ScriptContext, argv)
 
 Called once on startup, before any other events.
 
 
-### clientconnect(ScriptContext, ClientConnect)
+### clientconnect(ScriptContext, ConnectionHandler)
 
 Called when a client initiates a connection to the proxy. Note that
 a connection can correspond to multiple HTTP requests.
 
+### serverconnect(ScriptContext, ConnectionHandler)
 
-### request(ScriptContext, Flow)
+Called when the proxy initiates a connection to the target server. Note that
+a connection can correspond to multiple HTTP requests.
 
-Called when a client request has been received. The __Flow__ object is
+### request(ScriptContext, HTTPFlow)
+
+Called when a client request has been received. The __HTTPFlow__ object is
 guaranteed to have a non-None __request__ attribute.
 
+### responseheaders(ScriptContext, HTTPFlow)
 
-### response(ScriptContext, Flow)
+Called when the headers of a server response have been received.
+This will always be called before the response hook.
+The __HTTPFlow__ object is guaranteed to have non-None __request__ and
+__response__ attributes.  __response.content__ will be None,
+as the response body has not been read yet.
 
-Called when a server response has been received. The __Flow__ object is
+### response(ScriptContext, HTTPFlow)
+
+Called when a server response has been received. The __HTTPFlow__ object is
 guaranteed to have non-None __request__ and __response__ attributes.
+Note that if response streaming is enabled for this response,
+__response.content__ will not contain the response body.
 
-
-### error(ScriptContext, Flow)
+### error(ScriptContext, HTTPFlow)
 
 Called when a flow error has occurred, e.g. invalid server responses, or
 interrupted connections. This is distinct from a valid server HTTP error
-response, which is simply a response with an HTTP error code. The __Flow__
+response, which is simply a response with an HTTP error code. The __HTTPFlow__
 object is guaranteed to have non-None __request__ and __error__ attributes.
 
 
-### clientdisconnect(ScriptContext, ClientDisconnect)
+### clientdisconnect(ScriptContext, ConnectionHandler)
 
 Called when a client disconnects from the proxy.
 
@@ -71,43 +88,46 @@ The main classes you will deal with in writing mitmproxy scripts are:
 
 <table class="table">
     <tr>
-        <th>libmproxy.flow.ClientConnection</th>
+        <th>libmproxy.proxy.server.ConnectionHandler</th>
+        <td>Describes a proxy client connection session. Always has a client_conn attribute, might have a server_conn
+            attribute.
+        </td>
+    </tr>
+    <tr>
+        <th>libmproxy.proxy.connection.ClientConnection</th>
         <td>Describes a client connection.</td>
     </tr>
     <tr>
-        <th>libmproxy.flow.ClientDisconnection</th>
-        <td>Describes a client disconnection.</td>
+        <th>libmproxy.proxy.connection.ServerConnection</th>
+        <td>Describes a server connection.</td>
     </tr>
     <tr>
-        <th>libmproxy.flow.Error</th>
-        <td>A communications error.</td>
-    </tr>
-    <tr>
-        <th>libmproxy.flow.Flow</th>
+        <th>libmproxy.protocol.http.HTTPFlow</th>
         <td>A collection of objects representing a single HTTP transaction.</td>
     </tr>
     <tr>
-        <th>libmproxy.flow.Headers</th>
-        <td>HTTP headers for a request or response.</td>
+        <th>libmproxy.protocol.http.HTTPResponse</th>
+        <td>An HTTP response.</td>
+    </tr>
+    <tr>
+        <th>libmproxy.protocol.http.HTTPRequest</th>
+        <td>An HTTP request.</td>
+    </tr>
+    <tr>
+        <th>libmproxy.protocol.primitives.Error</th>
+        <td>A communications error.</td>
+    </tr>
+    <tr>
+        <th>libmproxy.script.ScriptContext</th>
+        <td> A handle for interacting with mitmproxy's from within scripts.</td>
     </tr>
     <tr>
         <th>libmproxy.flow.ODict</th>
 
         <td>A dictionary-like object for managing sets of key/value data. There
-        is also a variant called CaselessODict that ignores key case for some
-        calls (used mainly for headers).</td>
-    </tr>
-    <tr>
-        <th>libmproxy.flow.Response</th>
-        <td>An HTTP response.</td>
-    </tr>
-    <tr>
-        <th>libmproxy.flow.Request</th>
-        <td>An HTTP request.</td>
-    </tr>
-    <tr>
-        <th>libmproxy.flow.ScriptContext</th>
-        <td> A handle for interacting with mitmproxy's from within scripts.  </td>
+            is also a variant called CaselessODict that ignores key case for some
+            calls (used mainly for headers).
+        </td>
     </tr>
     <tr>
         <th>libmproxy.certutils.SSLCert</th>
@@ -115,14 +135,29 @@ The main classes you will deal with in writing mitmproxy scripts are:
     </tr>
 </table>
 
-The canonical API documentation is the code. You can view the API documentation
-using pydoc (which is installed with Python by default), like this:
+The canonical API documentation is the code, which you can browse locally or in our
+[GitHub repo](https://github.com/mitmproxy/mitmproxy).
+You can view the API documentation using pydoc (which is installed with Python by default), like this:
 
 <pre class="terminal">
-> pydoc libmproxy.flow.Request
+> pydoc libmproxy.protocol.http.HTTPRequest
 </pre>
 
 
+## Running scripts in parallel
+
+We have a single flow primitive, so when a script is handling something, other requests block.
+While that's a very desirable behaviour under some circumstances, scripts can be run threaded by using the <code>libmproxy.script.concurrent</code> decorator.
+
+$!example("examples/nonblocking.py")!$
+
+## Make scripts configurable with arguments
+
+Sometimes, you want to pass runtime arguments to the inline script. This can be simply done by surrounding the script call with quotes, e.g.
+<code>mitmdump -s "script.py --foo 42"</code>. The arguments are then exposed in the start event:
+
+$!example("examples/modify_response_body.py")!$
+
 ## Running scripts on saved flows
 
 Sometimes, we want to run a script on __Flow__ objects that are already
@@ -132,6 +167,11 @@ flows from a file (see the "scripted data transformation" example on the
 one-shot script on a single flow through the _|_ (pipe) shortcut in mitmproxy.
 
 In this case, there are no client connections, and the events are run in the
-following order: __start__, __request__, __response__, __error__, __done__.  If
+following order: __start__, __request__, __responseheaders__, __response__, __error__, __done__.  If
 the flow doesn't have a __response__ or __error__ associated with it, the
-matching event will be skipped. 
+matching events will be skipped.
+
+## Spaces in the script path
+By default, spaces are interpreted as separator between the inline script and its arguments (e.g. <code>-s "foo.py
+    42"</code>). Consequently, the script path needs to be wrapped in a separate pair of quotes if it contains spaces:
+<code>-s "'./foo bar/baz.py' 42"</code>.

doc-src/scripting/libmproxy.html

@@ -1,3 +1,13 @@
+<div class="well">
+    <strong>Heads up!</strong> We strongly encourage you to use <a href="@!urlTo("scripting/inlinescripts.html")!@">inline scripts</a> rather than libmproxy
+    directly.<br><br>
+    <ul>
+        <li>Inline Scripts are equally powerful and provide an easier syntax.</li>
+        <li>Most examples are written as inline scripts.</li>
+        <li>Multiple inline scripts can be combined and used together.</li>
+        <li>Inline Scripts can either be executed headless with mitmdump or within the mitmproxy UI.</li>
+    </ul>
+</div>
 
 All of mitmproxy's basic functionality is exposed through the __libmproxy__
 library. The example below shows a simple implementation of the "sticky cookie"

doc-src/ssl.html

@@ -1,7 +1,20 @@
 
 The first time __mitmproxy__ or __mitmdump__ is run, a set of certificate files
 for the mitmproxy Certificate Authority are created in the config directory
-(~/.mitmproxy by default). The files are as follows: 
+(~/.mitmproxy by default). This CA is used for on-the-fly generation of dummy
+certificates for SSL interception. Since your browser won't trust the
+__mitmproxy__ CA out of the box (and rightly so), you will see an SSL cert
+warning every time you visit a new SSL domain through __mitmproxy__. When
+you're testing a single site through a browser, just accepting the bogus SSL
+cert manually is not too much trouble, but there are a many circumstances where
+you will want to configure your testing system or browser to trust the
+__mitmproxy__ CA as a signing root authority.
+
+
+CA and cert files
+-----------------
+
+The files created by mitmproxy in the .mitmproxy directory are as follows: 
 
 <table class="table">
     <tr>
@@ -24,15 +37,48 @@ for the mitmproxy Certificate Authority are created in the config directory
     </tr>
 </table>
     
-This CA is used for on-the-fly generation of dummy certificates for SSL
-interception. Since your browser won't trust the __mitmproxy__ CA out of the
-box (and rightly so), you will see an SSL cert warning every time you visit a
-new SSL domain through __mitmproxy__. When you're testing a single site through
-a browser, just accepting the bogus SSL cert manually is not too much trouble,
-but there are a many circumstances where you will want to configure your
-testing system or browser to trust the __mitmproxy__ CA as a signing root
-authority.
 
+Using a custom certificate
+--------------------------
+
+You can use your own certificate by passing the <kbd>--cert</kbd> option to mitmproxy. mitmproxy then uses the provided
+certificate for interception of the specified domains instead of generating a cert signed by its own CA.
+
+The certificate file is expected to be in the PEM format.
+You can include intermediary certificates right below your leaf certificate, so that you PEM file roughly looks like
+this:
+
+<pre>
+-----BEGIN PRIVATE KEY-----
+&lt;private key&gt;
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+&lt;cert&gt;
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+&lt;intermediary cert (optional)&gt;
+-----END CERTIFICATE-----
+</pre>
+
+For example, you can generate a certificate in this format using these instructions:
+
+<pre class="terminal">
+> openssl genrsa -out cert.key 8192
+> openssl req -new -x509 -key cert.key -out cert.crt
+    (Specify the mitm domain as Common Name, e.g. *.google.com)
+> cat cert.key cert.crt > cert.pem
+> mitmproxy --cert=cert.pem
+</pre>
+
+
+Using a custom certificate authority
+------------------------------------
+
+By default, mitmproxy will (generate and) use <samp>~/.mitmproxy/mitmproxy-ca.pem</samp> as the default certificate
+authority to generate certificates for all domains for which no custom certificate is provided (see above).
+You can use your own certificate authority by passing the <kbd>--confdir</kbd> option to mitmproxy.
+mitmproxy will then look for <samp>mitmproxy-ca.pem</samp> in the specified directory. If no such file exists,
+it will be generated automatically.
 
 Installing the mitmproxy CA
 ---------------------------

doc-src/transparent/linux.html

@@ -3,7 +3,7 @@ achieve transparent mode.
 
 <ol class="tlist">
 
-    <li> <a href="@!urlTo("ssl.html")!@">Install the mitmproxy
+    <li> <a href="@!urlTo('ssl.html')!@">Install the mitmproxy
     certificates on the test device</a>. </li>
 
     <li> Enable IP forwarding:
@@ -15,6 +15,16 @@ achieve transparent mode.
 
     </li>
 
+    <li> If your target machine is on the same physical network and you configured it to use a custom gateway,
+    disable ICMP redirects:
+
+    <pre class="terminal">echo 0 | sudo tee /proc/sys/net/ipv4/conf/*/send_redirects</pre>
+
+    You may also want to consider enabling this permanently in
+    <b>/etc/sysctl.conf</b> as demonstrated <a href="http://unix.stackexchange.com/a/58081">here</a>.
+
+    </li>
+
     <li> Create an iptables ruleset that redirects the desired traffic to the
     mitmproxy port. Details will differ according to your setup, but the
     ruleset should look something like this:
@@ -38,3 +48,6 @@ iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8
     running as the default gateway.</li>
 
 </ol>
+
+
+For a detailed walkthrough, have a look at the <a href="@!urlTo('tutorials/transparent-dhcp.html')!@"><i>Transparently proxify virtual machines</i></a> tutorial.

doc-src/transparent/osx.html

@@ -7,7 +7,7 @@ OSX.
 
 <ol class="tlist">
 
-    <li> <a href="@!urlTo("ssl.html")!@">Install the mitmproxy
+    <li> <a href="@!urlTo('ssl.html')!@">Install the mitmproxy
     certificates on the test device</a>. </li>
 
     <li> Enable IP forwarding:
@@ -67,3 +67,15 @@ rdr on en2 inet proto tcp to any port 443 -&gt; 127.0.0.1 port 8080
 
 
 </ol>
+
+Note that the **rdr** rules in the pf.conf given above only apply to inbound
+traffic. This means that they will NOT redirect traffic coming from the box
+running pf itself. We can't distinguish between an outbound connection from a
+non-mitmproxy app, and an outbound connection from mitmproxy itself - if you
+want to intercept your OSX traffic, you should use an external host to run
+mitmproxy. None the less, pf is flexible to cater for a range of creative
+possibilities, like intercepting traffic emanating from VMs.  See the
+**pf.conf** man page for more.
+
+
+

doc-src/tutorials/gamecenter.html

@@ -17,14 +17,14 @@ Worm](http://itunes.apple.com/us/app/super-mega-worm/id388541990?mt=8) - a
 great little retro-apocalyptic sidescroller for the iPhone: 
 
 <center>
-    <img src="@!urlTo("tutorials/supermega.png")!@"/>
+    <img src="@!urlTo('tutorials/supermega.png')!@"/>
 </center>
 
 After finishing a game (take your time), watch the traffic flowing through
 mitmproxy:
 
 <center>
-    <img src="@!urlTo("tutorials/one.png")!@"/>
+    <img src="@!urlTo('tutorials/one.png')!@"/>
 </center>
 
 We see a bunch of things we might expect - initialisation, the retrieval of
@@ -67,8 +67,8 @@ timestamp. Looks pretty simple to mess with.
 Lets edit the score submission. First, select it in mitmproxy, then press
 __enter__ to view it. Make sure you're viewing the request, not the response -
 you can use __tab__ to flick between the two. Now press __e__ for edit. You'll
-be prompted for the part of the request you want to change - press __b__ for
-body.  Your preferred editor (taken from the EDITOR environment variable) will
+be prompted for the part of the request you want to change - press __r__ for
+raw body.  Your preferred editor (taken from the EDITOR environment variable) will
 now fire up. Lets bump the score up to something a bit more ambitious:
 
 <!--(block|syntax("xml"))-->
@@ -99,7 +99,7 @@ replay.
 ## The glorious result and some intrigue
 
 <center>
-    <img src="@!urlTo("tutorials/leaderboard.png")!@"/>
+    <img src="@!urlTo('tutorials/leaderboard.png')!@"/>
 </center>
 
 And that's it - according to the records, I am the greatest Super Mega Worm

doc-src/tutorials/index.py

@@ -3,4 +3,5 @@ from countershape import Page
 pages = [
     Page("30second.html", "Client playback: a 30 second example"),
     Page("gamecenter.html", "Setting highscores on Apple's GameCenter"),
-]
+    Page("transparent-dhcp.html", "Transparently proxify virtual machines")
+]

doc-src/tutorials/transparent-dhcp.html

@@ -0,0 +1,54 @@
+This walkthrough illustrates how to set up transparent proxying with mitmproxy. We use VirtualBox VMs with an Ubuntu proxy machine in this example, but the general principle can be applied to other setups.
+
+1.  **Configure VirtualBox Network Adapters for the proxy machine**  
+    The network setup is simple:  `internet <--> proxy vm <--> (virtual) internal network`.  
+    For the proxy machine, *eth0* represents the outgoing network. *eth1* is connected to the internal network that will be proxified, using a static ip (192.168.3.1).  
+    <hr>VirtualBox configuration:
+    <img src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth0.png')!@"/><br><br>
+    <img src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth1.png')!@"/>  
+    <br>Proxy VM:
+    <img src="@!urlTo('tutorials/transparent-dhcp/step1_proxy.png')!@"/>  
+    <hr>
+2.  **Configure DHCP and DNS**  
+    We use dnsmasq to provide DHCP and DNS in our internal network. 
+    Dnsmasq is a lightweight server designed to provide DNS (and optionally DHCP and TFTP) services to a small-scale 
+    network.  
+
+    -   Before we get to that, we need to fix some Ubuntu quirks: 
+        **Ubuntu >12.04** runs an internal dnsmasq instance (listening on loopback only) by default 
+        <a href="https://www.stgraber.org/2012/02/24/dns-in-ubuntu-12-04/">[1]</a>. For our use case, this needs to be
+        disabled by changing <br>`dns=dnsmasq` to `#dns=dnsmasq` in */etc/NetworkManager/NetworkManager.conf*  
+        and running `sudo restart network-manager` afterwards.
+    -   Now, dnsmasq can be be installed and configured:  
+        `sudo apt-get install dnsmasq`  
+        Replace */etc/dnsmasq.conf* with the following configuration:  
+        <pre>\# Listen for DNS requests on the internal network
+        interface=eth1
+        \# Act as a DHCP server, assign IP addresses to clients
+        dhcp-range=192.168.3.10,192.168.3.100,96h
+        \# Broadcast gateway and dns server information
+        dhcp-option=option:router,192.168.3.1
+        dhcp-option=option:dns-server,192.168.3.1
+        </pre>
+        Apply changes:  
+        `sudo service dnsmasq restart`
+        <hr>
+        Your proxied machine's network settings should now look similar to this:
+        <img src="@!urlTo('tutorials/transparent-dhcp/step2_proxied_vm.png')!@"/>
+        <hr>
+
+3.  **Set up traffic redirection to mitmproxy**  
+    To redirect traffic to mitmproxy, we need to add two iptables rules:  
+    <pre class="terminal">
+    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 \
+        -j REDIRECT --to-port 8080
+    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 \
+        -j REDIRECT --to-port 8080
+    </pre>
+
+4.  If required, <a href="@!urlTo('ssl.html')!@">install the mitmproxy
+    certificates on the test device</a>.
+
+5.  Finally, we can run <code>mitmproxy -T</code>.  
+    The proxied machine cannot to leak any data outside of HTTP or DNS requests.
+

examples/README

@@ -1,10 +1,23 @@
-add_header.py           Simple script that just adds a header to every request.
-dup_and_replay.py       Duplicates each request, changes it, and then replays the modified request.
-flowbasic               Basic use of mitmproxy as a library.
-modify_form.py          Modify all form submissions to add a parameter.
-modify_querystring.py   Modify all query strings to add a parameters.
-proxapp                 How to embed a WSGI app in a mitmproxy server
-stub.py                 Script stub with a method definition for every event.
-stickycookies           An example of writing a custom proxy with libmproxy.
-upsidedownternet.py     Rewrites traffic to turn PNGs upside down.
-mitmproxywrapper.py     Bracket mitmproxy run with proxy enable/disable on OS X
+# inline script examples
+add_header.py             Simple script that just adds a header to every request.
+change_upstream_proxy.py  Dynamically change the upstream proxy
+dup_and_replay.py         Duplicates each request, changes it, and then replays the modified request.
+iframe_injector.py        Inject configurable iframe into pages.
+modify_form.py            Modify all form submissions to add a parameter.
+modify_querystring.py     Modify all query strings to add a parameters.
+modify_response_body.py   Replace arbitrary strings in all responses
+nonblocking.py            Demonstrate parallel processing with a blocking script.
+proxapp.py                How to embed a WSGI app in a mitmproxy server
+redirect_requests.py      Redirect requests or directly reply to them.
+stub.py                   Script stub with a method definition for every event.
+upsidedownternet.py       Rewrites traffic to turn images upside down.
+
+
+# libmproxy examples
+flowbasic                 Basic use of mitmproxy as a library.
+stickycookies             An example of writing a custom proxy with libmproxy.
+
+
+# misc
+read_dumpfile             Read a dumpfile generated by mitmproxy.
+mitmproxywrapper.py       Bracket mitmproxy run with proxy enable/disable on OS X

examples/add_header.py

@@ -1,2 +1,2 @@
 def response(context, flow):
-    flow.response.headers["newheader"] = ["foo"]
+    flow.response.headers["newheader"] = ["foo"]

examples/change_upstream_proxy.py

@@ -0,0 +1,21 @@
+# This scripts demonstrates how mitmproxy can switch to a second/different upstream proxy
+# in upstream proxy mode.
+#
+# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s "change_upstream_proxy.py host"
+from libmproxy.protocol.http import send_connect_request
+
+alternative_upstream_proxy = ("localhost", 8082)
+def should_redirect(flow):
+    return flow.request.host == "example.com"
+
+
+def request(context, flow):
+    if flow.live and should_redirect(flow):
+
+        # If you want to change the target server, you should modify flow.request.host and flow.request.port
+        # flow.live.change_server should only be used by inline scripts to change the upstream proxy,
+        # unless you are sure that you know what you are doing.
+        server_changed = flow.live.change_server(alternative_upstream_proxy, persistent_change=True)
+        if flow.request.scheme == "https" and server_changed:
+            send_connect_request(flow.live.c.server_conn, flow.request.host, flow.request.port)
+            flow.live.c.establish_ssl(server=True)

examples/dup_and_replay.py

@@ -1,4 +1,4 @@
-def request(ctx, flow):
-   f = ctx.duplicate_flow(flow)
-   f.request.path = "/changed"
-   ctx.replay_request(f)
+def request(context, flow):
+    f = context.duplicate_flow(flow)
+    f.request.path = "/changed"
+    context.replay_request(f)

examples/flowbasic

@@ -3,11 +3,15 @@
     This example shows how to build a proxy based on mitmproxy's Flow
     primitives.
 
+    Heads Up: In the majority of cases, you want to use inline scripts.
+
     Note that request and response messages are not automatically replied to,
     so we need to implement handlers to do this.
 """
 import os
-from libmproxy import proxy, flow
+from libmproxy import flow, proxy
+from libmproxy.proxy.server import ProxyServer
+
 
 class MyMaster(flow.FlowMaster):
     def run(self):
@@ -16,24 +20,25 @@ class MyMaster(flow.FlowMaster):
         except KeyboardInterrupt:
             self.shutdown()
 
-    def handle_request(self, r):
-        f = flow.FlowMaster.handle_request(self, r)
+    def handle_request(self, f):
+        f = flow.FlowMaster.handle_request(self, f)
         if f:
-            r.reply()
+            f.reply()
         return f
 
-    def handle_response(self, r):
-        f = flow.FlowMaster.handle_response(self, r)
+    def handle_response(self, f):
+        f = flow.FlowMaster.handle_response(self, f)
         if f:
-            r.reply()
+            f.reply()
         print f
         return f
 
 
 config = proxy.ProxyConfig(
-    cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
+    port=8080,
+    cadir="~/.mitmproxy/"  # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file.
 )
 state = flow.State()
-server = proxy.ProxyServer(config, 8080)
+server = ProxyServer(config)
 m = MyMaster(server, state)
 m.run()

examples/har_extractor.py

@@ -0,0 +1,212 @@
+"""
+    This inline script utilizes harparser.HAR from https://github.com/JustusW/harparser
+    to generate a HAR log object.
+"""
+try:
+    from harparser import HAR
+    from pytz import UTC
+except ImportError as e:
+    import sys
+    print >> sys.stderr, "\r\nMissing dependencies: please run `pip install mitmproxy[examples]`.\r\n"
+    raise
+
+from datetime import datetime, timedelta, tzinfo
+
+
+class _HARLog(HAR.log):
+    # The attributes need to be registered here for them to actually be available later via self. This is
+    # due to HAREncodable linking __getattr__ to __getitem__. Anything that is set only in __init__ will
+    # just be added as key/value pair to self.__classes__.
+    __page_list__ = []
+    __page_count__ = 0
+    __page_ref__ = {}
+
+    def __init__(self, page_list):
+        self.__page_list__ = page_list
+        self.__page_count__ = 0
+        self.__page_ref__ = {}
+
+        HAR.log.__init__(self, {"version": "1.2",
+                                "creator": {"name": "MITMPROXY HARExtractor",
+                                            "version": "0.1",
+                                            "comment": ""},
+                                "pages": [],
+                                "entries": []})
+
+    def reset(self):
+        self.__init__(self.__page_list__)
+
+    def add(self, obj):
+        if isinstance(obj, HAR.pages):
+            self['pages'].append(obj)
+        if isinstance(obj, HAR.entries):
+            self['entries'].append(obj)
+
+    def create_page_id(self):
+        self.__page_count__ += 1
+        return "autopage_%s" % str(self.__page_count__)
+
+    def set_page_ref(self, page, ref):
+        self.__page_ref__[page] = ref
+
+    def get_page_ref(self, page):
+        return self.__page_ref__.get(page, None)
+
+    def get_page_list(self):
+        return self.__page_list__
+
+
+def start(context, argv):
+    """
+        On start we create a HARLog instance. You will have to adapt this to suit your actual needs
+        of HAR generation. As it will probably be necessary to cluster logs by IPs or reset them
+        from time to time.
+    """
+    context.dump_file = None
+    if len(argv) > 1:
+        context.dump_file = argv[1]
+    else:
+        raise ValueError('Usage: -s "har_extractor.py filename" '
+                         '(- will output to stdout, filenames ending with .zhar will result in compressed har)')
+    context.HARLog = _HARLog(['https://github.com'])
+    context.seen_server = set()
+
+
+def response(context, flow):
+    """
+       Called when a server response has been received. At the time of this message both
+       a request and a response are present and completely done.
+    """
+    # Values are converted from float seconds to int milliseconds later.
+    ssl_time = -.001
+    connect_time = -.001
+    if flow.server_conn not in context.seen_server:
+        # Calculate the connect_time for this server_conn. Afterwards add it to seen list, in
+        # order to avoid the connect_time being present in entries that use an existing connection.
+        connect_time = flow.server_conn.timestamp_tcp_setup - flow.server_conn.timestamp_start
+        context.seen_server.add(flow.server_conn)
+
+        if flow.server_conn.timestamp_ssl_setup is not None:
+            # Get the ssl_time for this server_conn as the difference between the start of the successful
+            # tcp setup and the successful ssl setup. If  no ssl setup has been made it is left as -1 since
+            # it doesn't apply to this connection.
+            ssl_time = flow.server_conn.timestamp_ssl_setup - flow.server_conn.timestamp_tcp_setup
+
+    # Calculate the raw timings from the different timestamps present in the request and response object.
+    # For lack of a way to measure it dns timings can not be calculated. The same goes for HAR blocked:
+    # MITMProxy will open a server connection as soon as it receives the host and port from the client
+    # connection. So the time spent waiting is actually spent waiting between request.timestamp_end and
+    # response.timestamp_start thus it correlates to HAR wait instead.
+    timings_raw = {'send': flow.request.timestamp_end - flow.request.timestamp_start,
+                   'wait': flow.response.timestamp_start - flow.request.timestamp_end,
+                   'receive': flow.response.timestamp_end - flow.response.timestamp_start,
+                   'connect': connect_time,
+                   'ssl': ssl_time}
+
+    # HAR timings are integers in ms, so we have to re-encode the raw timings to that format.
+    timings = dict([(key, int(1000 * value)) for key, value in timings_raw.iteritems()])
+
+    # The full_time is the sum of all timings. Timings set to -1 will be ignored as per spec.
+    full_time = 0
+    for item in timings.values():
+        if item > -1:
+            full_time += item
+
+    started_date_time = datetime.fromtimestamp(flow.request.timestamp_start, tz=utc).isoformat()
+
+    request_query_string = [{"name": k, "value": v} for k, v in flow.request.get_query()]
+    request_http_version = ".".join([str(v) for v in flow.request.httpversion])
+    # Cookies are shaped as tuples by MITMProxy.
+    request_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.request.get_cookies() or {}).iteritems()]
+    request_headers = [{"name": k, "value": v} for k, v in flow.request.headers]
+    request_headers_size = len(str(flow.request.headers))
+    request_body_size = len(flow.request.content)
+
+    response_http_version = ".".join([str(v) for v in flow.response.httpversion])
+    # Cookies are shaped as tuples by MITMProxy.
+    response_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.response.get_cookies() or {}).iteritems()]
+    response_headers = [{"name": k, "value": v} for k, v in flow.response.headers]
+    response_headers_size = len(str(flow.response.headers))
+    response_body_size = len(flow.response.content)
+    response_body_decoded_size = len(flow.response.get_decoded_content())
+    response_body_compression = response_body_decoded_size - response_body_size
+    response_mime_type = flow.response.headers.get_first('Content-Type', '')
+    response_redirect_url = flow.response.headers.get_first('Location', '')
+
+    entry = HAR.entries({"startedDateTime": started_date_time,
+                         "time": full_time,
+                         "request": {"method": flow.request.method,
+                                     "url": flow.request.url,
+                                     "httpVersion": request_http_version,
+                                     "cookies": request_cookies,
+                                     "headers": request_headers,
+                                     "queryString": request_query_string,
+                                     "headersSize": request_headers_size,
+                                     "bodySize": request_body_size, },
+                         "response": {"status": flow.response.code,
+                                      "statusText": flow.response.msg,
+                                      "httpVersion": response_http_version,
+                                      "cookies": response_cookies,
+                                      "headers": response_headers,
+                                      "content": {"size": response_body_size,
+                                                  "compression": response_body_compression,
+                                                  "mimeType": response_mime_type},
+                                      "redirectURL": response_redirect_url,
+                                      "headersSize": response_headers_size,
+                                      "bodySize": response_body_size, },
+                         "cache": {},
+                         "timings": timings, })
+
+    # If the current url is in the page list of context.HARLog or does not have a referrer we add it as a new
+    # pages object.
+    if flow.request.url in context.HARLog.get_page_list() or flow.request.headers.get('Referer', None) is None:
+        page_id = context.HARLog.create_page_id()
+        context.HARLog.add(HAR.pages({"startedDateTime": entry['startedDateTime'],
+                                      "id": page_id,
+                                      "title": flow.request.url, }))
+        context.HARLog.set_page_ref(flow.request.url, page_id)
+        entry['pageref'] = page_id
+
+    # Lookup the referer in the page_ref of context.HARLog to point this entries pageref attribute to the right
+    # pages object, then set it as a new reference to build a reference tree.
+    elif context.HARLog.get_page_ref(flow.request.headers.get('Referer', (None, ))[0]) is not None:
+        entry['pageref'] = context.HARLog.get_page_ref(flow.request.headers['Referer'][0])
+        context.HARLog.set_page_ref(flow.request.headers['Referer'][0], entry['pageref'])
+
+    context.HARLog.add(entry)
+
+
+def done(context):
+    """
+        Called once on script shutdown, after any other events.
+    """
+    from pprint import pprint
+    import json
+
+    json_dump = context.HARLog.json()
+    compressed_json_dump = context.HARLog.compress()
+
+    print "=" * 100
+    if context.dump_file == '-':
+        pprint(json.loads(json_dump))
+    elif context.dump_file.endswith('.zhar'):
+        file(context.dump_file, "w").write(compressed_json_dump)
+    else:
+        file(context.dump_file, "w").write(json_dump)
+    print "=" * 100
+    print "HAR log finished with %s bytes (%s bytes compressed)" % (len(json_dump), len(compressed_json_dump))
+    print "Compression rate is %s%%" % str(100. * len(compressed_json_dump) / len(json_dump))
+    print "=" * 100
+
+
+def print_attributes(obj, filter_string=None, hide_privates=False):
+    """
+        Useful helper method to quickly get all attributes of an object and its values.
+    """
+    for attr in dir(obj):
+        if hide_privates and "__" in attr:
+            continue
+        if filter_string is not None and filter_string not in attr:
+            continue
+        value = getattr(obj, attr)
+        print "%s.%s" % ('obj', attr), value, type(value)

examples/iframe_injector

@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-"""
-    Zap encoding in requests and inject iframe after body tag in html responses.
-    Usage:
-        iframe_injector http://someurl/somefile.html
-"""
-from libmproxy import controller, proxy
-import os
-import sys
-
-
-class InjectingMaster(controller.Master):
-    def __init__(self, server, iframe_url):
-        controller.Master.__init__(self, server)
-        self._iframe_url = iframe_url
-
-    def run(self):
-        try:
-            return controller.Master.run(self)
-        except KeyboardInterrupt:
-            self.shutdown()
-
-    def handle_request(self, msg):
-        if 'Accept-Encoding' in msg.headers:
-            msg.headers["Accept-Encoding"] = 'none'
-        msg.reply()
-
-    def handle_response(self, msg):
-        if msg.content:
-            c = msg.replace('<body>', '<body><iframe src="%s" frameborder="0" height="0" width="0"></iframe>' % self._iframe_url)
-            if c > 0:
-                print 'Iframe injected!'
-        msg.reply()
-
-
-def main(argv):
-    if len(argv) != 2:
-        print "Usage: %s IFRAME_URL" % argv[0]
-        sys.exit(1)
-    iframe_url = argv[1]
-    config = proxy.ProxyConfig(
-        cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
-    )
-    server = proxy.ProxyServer(config, 8080)
-    print 'Starting proxy...'
-    m = InjectingMaster(server, iframe_url)
-    m.run()
-
-if __name__ == '__main__':
-    main(sys.argv)

examples/iframe_injector.py

@@ -0,0 +1,22 @@
+# Usage: mitmdump -s "iframe_injector.py url"
+# (this script works best with --anticache)
+from bs4 import BeautifulSoup
+from libmproxy.protocol.http import decoded
+
+
+def start(context, argv):
+    if len(argv) != 2:
+        raise ValueError('Usage: -s "iframe_injector.py url"')
+    context.iframe_url = argv[1]
+
+
+def response(context, flow):
+    if flow.request.host in context.iframe_url:
+        return
+    with decoded(flow.response):  # Remove content encoding (gzip, ...)
+        html = BeautifulSoup(flow.response.content)
+        if html.body:
+            iframe = html.new_tag("iframe", src=context.iframe_url, frameborder=0, height=0, width=0)
+            html.body.insert(0, iframe)
+            flow.response.content = str(html)
+            context.log("Iframe inserted.")

examples/ignore_websocket.py

@@ -0,0 +1,34 @@
+# This script makes mitmproxy switch to passthrough mode for all HTTP
+# responses with "Connection: Upgrade" header. This is useful to make
+# WebSockets work in untrusted environments.
+#
+# Note: Chrome (and possibly other browsers), when explicitly configured
+# to use a proxy (i.e. mitmproxy's regular mode), send a CONNECT request
+# to the proxy before they initiate the websocket connection.
+# To make WebSockets work in these cases, supply
+# `--ignore :80$` as an additional parameter.
+# (see http://mitmproxy.org/doc/features/passthrough.html)
+
+from libmproxy.protocol.http import HTTPRequest
+from libmproxy.protocol.tcp import TCPHandler
+from libmproxy.protocol import KILL
+from libmproxy.script import concurrent
+
+
+def start(context, argv):
+    HTTPRequest._headers_to_strip_off.remove("Connection")
+    HTTPRequest._headers_to_strip_off.remove("Upgrade")
+
+
+def done(context):
+    HTTPRequest._headers_to_strip_off.append("Connection")
+    HTTPRequest._headers_to_strip_off.append("Upgrade")
+
+@concurrent
+def response(context, flow):
+    if flow.response.headers.get_first("Connection", None) == "Upgrade":
+        # We need to send the response manually now...
+        flow.client_conn.send(flow.response.assemble())
+        # ...and then delegate to tcp passthrough.
+        TCPHandler(flow.live.c, log=False).handle_messages()
+        flow.reply(KILL)

examples/mitmproxywrapper.py

@@ -10,12 +10,15 @@
 import subprocess
 import re
 import argparse
+import contextlib
+import os
+import sys
 
 class Wrapper(object):
     
-    def __init__(self, port):
+    def __init__(self, port, extra_arguments=None):
         self.port = port
-        self.primary_service_name = self.find_primary_service_name()
+        self.extra_arguments = extra_arguments
 
     def run_networksetup_command(self, *arguments):
         return subprocess.check_output(['sudo', 'networksetup'] + list(arguments))
@@ -50,40 +53,90 @@ class Wrapper(object):
         interface, = re.findall(r'PrimaryInterface\s*:\s*(.+)', stdout)
         return interface
 
-    def find_primary_service_name(self):
+    def primary_service_name(self):
         return self.interface_name_to_service_name_map()[self.primary_interace_name()]
 
     def proxy_enabled_for_service(self, service):
         return self.proxy_state_for_service(service)['Enabled'] == 'Yes'
 
     def toggle_proxy(self):
-        if self.proxy_enabled_for_service(self.primary_service_name):
-            self.disable_proxy_for_service(self.primary_service_name)
-        else:
-            self.enable_proxy_for_service(self.primary_service_name)
+        new_state = not self.proxy_enabled_for_service(self.primary_service_name())
+        for service_name in self.connected_service_names():
+            if self.proxy_enabled_for_service(service_name) and not new_state:
+                self.disable_proxy_for_service(service_name)
+            elif not self.proxy_enabled_for_service(service_name) and new_state:
+                self.enable_proxy_for_service(service_name)
+
+    def connected_service_names(self):
+        scutil_script = 'list\n'
+        stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script)
+        service_ids = re.findall(r'State:/Network/Service/(.+)/IPv4', stdout)
+
+        service_names = []
+        for service_id in service_ids:
+            scutil_script = 'show Setup:/Network/Service/{}\n'.format(service_id)
+            stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script)
+            service_name, = re.findall(r'UserDefinedName\s*:\s*(.+)', stdout)
+            service_names.append(service_name)
+
+        return service_names
 
     def wrap_mitmproxy(self):
-        if not self.proxy_enabled_for_service(self.primary_service_name):
-            self.enable_proxy_for_service(self.primary_service_name)
+        with self.wrap_proxy():
+            cmd = ['mitmproxy', '-p', str(self.port)]
+            if self.extra_arguments:
+                cmd.extend(self.extra_arguments)
+            subprocess.check_call(cmd)
 
-        subprocess.check_call(['mitmproxy', '-p', str(self.port), '--palette', 'light'])
+    def wrap_honeyproxy(self):
+        with self.wrap_proxy():
+            popen = subprocess.Popen('honeyproxy.sh')
+            try:
+                popen.wait()
+            except KeyboardInterrupt:
+                popen.terminate()
 
-        if self.proxy_enabled_for_service(self.primary_service_name):
-            self.disable_proxy_for_service(self.primary_service_name)
+    @contextlib.contextmanager
+    def wrap_proxy(self):
+        connected_service_names = self.connected_service_names()
+        for service_name in connected_service_names:
+            if not self.proxy_enabled_for_service(service_name):
+                self.enable_proxy_for_service(service_name)
+        
+        yield
+
+        for service_name in connected_service_names:
+            if self.proxy_enabled_for_service(service_name):
+                self.disable_proxy_for_service(service_name)
+
+    @classmethod
+    def ensure_superuser(cls):
+        if os.getuid() != 0:
+            print 'Relaunching with sudo...'
+            os.execv('/usr/bin/sudo', ['/usr/bin/sudo'] + sys.argv)
 
     @classmethod
     def main(cls):
-        parser = argparse.ArgumentParser(description='Helper tool for OS X proxy configuration and mitmproxy')
+        parser = argparse.ArgumentParser(
+            description='Helper tool for OS X proxy configuration and mitmproxy.',
+            epilog='Any additional arguments will be passed on unchanged to mitmproxy.'
+        )
         parser.add_argument('-t', '--toggle', action='store_true', help='just toggle the proxy configuration')
+#         parser.add_argument('--honeyproxy', action='store_true', help='run honeyproxy instead of mitmproxy')
         parser.add_argument('-p', '--port', type=int, help='override the default port of 8080', default=8080)
-        args = parser.parse_args()
+        args, extra_arguments = parser.parse_known_args()
 
-        wrapper = cls(port=args.port)
+        wrapper = cls(port=args.port, extra_arguments=extra_arguments)
+        
         if args.toggle:
             wrapper.toggle_proxy()
+#         elif args.honeyproxy:
+#             wrapper.wrap_honeyproxy()
         else:
             wrapper.wrap_mitmproxy()
 
+
 if __name__ == '__main__':
+    Wrapper.ensure_superuser()
     Wrapper.main()
 

examples/modify_form.py

@@ -1,8 +1,6 @@
 
 def request(context, flow):
     if "application/x-www-form-urlencoded" in flow.request.headers["content-type"]:
-        frm = flow.request.get_form_urlencoded()
-        frm["mitmproxy"] = ["rocks"]
-        flow.request.set_form_urlencoded(frm)
-
-
+        form = flow.request.get_form_urlencoded()
+        form["mitmproxy"] = ["rocks"]
+        flow.request.set_form_urlencoded(form)

examples/modify_querystring.py

@@ -3,5 +3,4 @@ def request(context, flow):
     q = flow.request.get_query()
     if q:
         q["mitmproxy"] = ["rocks"]
-        flow.request.set_query(q)
-
+        flow.request.set_query(q)

examples/modify_response_body.py

@@ -0,0 +1,15 @@
+# Usage: mitmdump -s "modify_response_body.py mitmproxy bananas"
+# (this script works best with --anticache)
+from libmproxy.protocol.http import decoded
+
+
+def start(context, argv):
+    if len(argv) != 3:
+        raise ValueError('Usage: -s "modify-response-body.py old new"')
+    # You may want to use Python's argparse for more sophisticated argument parsing.
+    context.old, context.new = argv[1], argv[2]
+
+
+def response(context, flow):
+    with decoded(flow.response):  # automatically decode gzipped responses.
+        flow.response.content = flow.response.content.replace(context.old, context.new)

examples/nonblocking.py

@@ -0,0 +1,9 @@
+import time
+from libmproxy.script import concurrent
+
+
+@concurrent  # Remove this and see what happens
+def request(context, flow):
+    print "handle request: %s%s" % (flow.request.host, flow.request.path)
+    time.sleep(5)
+    print "start  request: %s%s" % (flow.request.host, flow.request.path)

examples/proxapp

@@ -1,47 +0,0 @@
-#!/usr/bin/env python
-"""
-This example shows how to graft a WSGI app onto mitmproxy. In this
-instance, we're using the Bottle framework (http://bottlepy.org/) to expose
-a single simplest-possible page.
-"""
-import bottle
-import os
-from libmproxy import proxy, flow
-
-@bottle.route('/')
-def index():
-    return 'Hi!'
-
-
-class MyMaster(flow.FlowMaster):
-    def run(self):
-        try:
-            flow.FlowMaster.run(self)
-        except KeyboardInterrupt:
-            self.shutdown()
-
-    def handle_request(self, r):
-        f = flow.FlowMaster.handle_request(self, r)
-        if f:
-            r.reply()
-        return f
-
-    def handle_response(self, r):
-        f = flow.FlowMaster.handle_response(self, r)
-        if f:
-            r.reply()
-        print f
-        return f
-
-
-config = proxy.ProxyConfig(
-    cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
-)
-state = flow.State()
-server = proxy.ProxyServer(config, 8080)
-# Register the app using the magic domain "proxapp" on port 80. Requests to
-# this domain and port combination will now be routed to the WSGI app instance.
-server.apps.add(bottle.app(), "proxapp", 80)
-m = MyMaster(server, state)
-m.run()
-

examples/proxapp.py

@@ -0,0 +1,24 @@
+"""
+This example shows how to graft a WSGI app onto mitmproxy. In this
+instance, we're using the Flask framework (http://flask.pocoo.org/) to expose
+a single simplest-possible page.
+"""
+from flask import Flask
+
+app = Flask("proxapp")
+
+
+@app.route('/')
+def hello_world():
+    return 'Hello World!'
+
+
+# Register the app using the magic domain "proxapp" on port 80. Requests to
+# this domain and port combination will now be routed to the WSGI app instance.
+def start(context, argv):
+    context.app_registry.add(app, "proxapp", 80)
+
+    # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design.
+    # mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set)
+    # but won't send any data.
+    context.app_registry.add(app, "example.com", 443)

examples/read_dumpfile

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+#
+# Simple script showing how to read a mitmproxy dump file
+#
+
+from libmproxy import flow
+import json, sys
+
+with open("logfile", "rb") as logfile:
+    freader = flow.FlowReader(logfile)
+    try:
+        for f in freader.stream():
+            print(f)
+            print(f.request.host)
+            json.dump(f.get_state(), sys.stdout, indent=4)
+            print ""
+    except flow.FlowReadError, v:
+        print "Flow file corrupted. Stopped loading."

examples/redirect_requests.py

@@ -1,19 +1,24 @@
-from libmproxy.flow import Response
+from libmproxy.protocol.http import HTTPResponse
 from netlib.odict import ODictCaseless
 
 """
 This example shows two ways to redirect flows to other destinations.
 """
 
+
 def request(context, flow):
-    if flow.request.host.endswith("example.com"):
-        resp = Response(flow.request,
-                        [1,1],
-                        200, "OK",
-                        ODictCaseless([["Content-Type","text/html"]]),
-                        "helloworld",
-                        None)
-        flow.request.reply(resp)
-    if flow.request.host.endswith("example.org"):
+    # pretty_host(hostheader=True) takes the Host: header of the request into account,
+    # which is useful in transparent mode where we usually only have the IP otherwise.
+
+    # Method 1: Answer with a locally generated response
+    if flow.request.pretty_host(hostheader=True).endswith("example.com"):
+        resp = HTTPResponse(
+            [1, 1], 200, "OK",
+            ODictCaseless([["Content-Type", "text/html"]]),
+            "helloworld")
+        flow.reply(resp)
+
+    # Method 2: Redirect the request to a different server
+    if flow.request.pretty_host(hostheader=True).endswith("example.org"):
         flow.request.host = "mitmproxy.org"
-        flow.request.headers["Host"] = ["mitmproxy.org"]
+        flow.request.update_host_header()

examples/stickycookies

@@ -1,12 +1,14 @@
 #!/usr/bin/env python
 """
 This example builds on mitmproxy's base proxying infrastructure to
-implement functionality similar to the "sticky cookies" option. This is at
-a lower level than the Flow mechanism, so we're dealing directly with
-request and response objects.
+implement functionality similar to the "sticky cookies" option.
+
+Heads Up: In the majority of cases, you want to use inline scripts.
 """
-from libmproxy import controller, proxy
 import os
+from libmproxy import controller, proxy
+from libmproxy.proxy.server import ProxyServer
+
 
 class StickyMaster(controller.Master):
     def __init__(self, server):
@@ -19,24 +21,22 @@ class StickyMaster(controller.Master):
         except KeyboardInterrupt:
             self.shutdown()
 
-    def handle_request(self, msg):
-        hid = (msg.host, msg.port)
-        if msg.headers["cookie"]:
-            self.stickyhosts[hid] = msg.headers["cookie"]
+    def handle_request(self, flow):
+        hid = (flow.request.host, flow.request.port)
+        if flow.request.headers["cookie"]:
+            self.stickyhosts[hid] = flow.request.headers["cookie"]
         elif hid in self.stickyhosts:
-            msg.headers["cookie"] = self.stickyhosts[hid]
-        msg.reply()
+            flow.request.headers["cookie"] = self.stickyhosts[hid]
+        flow.reply()
 
-    def handle_response(self, msg):
-        hid = (msg.request.host, msg.request.port)
-        if msg.headers["set-cookie"]:
-            self.stickyhosts[hid] = msg.headers["set-cookie"]
-        msg.reply()
+    def handle_response(self, flow):
+        hid = (flow.request.host, flow.request.port)
+        if flow.response.headers["set-cookie"]:
+            self.stickyhosts[hid] = flow.response.headers["set-cookie"]
+        flow.reply()
 
 
-config = proxy.ProxyConfig(
-    cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
-)
-server = proxy.ProxyServer(config, 8080)
+config = proxy.ProxyConfig(port=8080)
+server = ProxyServer(config)
 m = StickyMaster(server)
 m.run()

examples/stream.py

@@ -0,0 +1,5 @@
+def responseheaders(context, flow):
+    """
+    Enables streaming for all responses.
+    """
+    flow.response.stream = True

examples/stub.py

@@ -1,48 +1,63 @@
 """
     This is a script stub, with definitions for all events.
 """
-
-def start(ctx):
+def start(context, argv):
     """
         Called once on script startup, before any other events.
     """
-    ctx.log("start")
+    context.log("start")
 
-def clientconnect(ctx, client_connect):
+def clientconnect(context, conn_handler):
     """
         Called when a client initiates a connection to the proxy. Note that a
         connection can correspond to multiple HTTP requests
     """
-    ctx.log("clientconnect")
+    context.log("clientconnect")
 
-def request(ctx, flow):
+def serverconnect(context, conn_handler):
+    """
+        Called when the proxy initiates a connection to the target server. Note that a
+        connection can correspond to multiple HTTP requests
+    """
+    context.log("serverconnect")
+
+def request(context, flow):
     """
         Called when a client request has been received.
     """
-    ctx.log("request")
+    context.log("request")
 
-def response(ctx, flow):
+
+def responseheaders(context, flow):
+    """
+        Called when the response headers for a server response have been received,
+        but the response body has not been processed yet. Can be used to tell mitmproxy
+        to stream the response.
+    """
+    context.log("responseheaders")
+
+def response(context, flow):
     """
        Called when a server response has been received.
     """
-    ctx.log("response")
+    context.log("response")
 
-def error(ctx, flow):
+def error(context, flow):
     """
         Called when a flow error has occured, e.g. invalid server responses, or
         interrupted connections. This is distinct from a valid server HTTP error
         response, which is simply a response with an HTTP error code.
     """
-    ctx.log("error")
+    context.log("error")
 
-def clientdisconnect(ctx, client_disconnect):
+def clientdisconnect(context, conn_handler):
     """
         Called when a client disconnects from the proxy.
     """
-    ctx.log("clientdisconnect")
+    context.log("clientdisconnect")
 
-def done(ctx):
+def done(context):
     """
         Called once on script shutdown, after any other events.
     """
-    ctx.log("done")
+    context.log("done")

examples/upsidedownternet.py

@@ -1,8 +1,16 @@
-import Image, cStringIO
+import cStringIO
+from PIL import Image
+from libmproxy.protocol.http import decoded
+
 def response(context, flow):
-    if flow.response.headers["content-type"] == ["image/png"]:
-        s = cStringIO.StringIO(flow.response.content)
-        img = Image.open(s).rotate(180)
-        s2 = cStringIO.StringIO()
-        img.save(s2, "png")
-        flow.response.content = s2.getvalue()
+    if flow.response.headers.get_first("content-type", "").startswith("image"):
+        with decoded(flow.response):  # automatically decode gzipped responses.
+            try:
+                s = cStringIO.StringIO(flow.response.content)
+                img = Image.open(s).rotate(180)
+                s2 = cStringIO.StringIO()
+                img.save(s2, "png")
+                flow.response.content = s2.getvalue()
+                flow.response.headers["content-type"] = ["image/png"]
+            except:  # Unknown image types etc.
+                pass

libmproxy/app.py

@@ -1,7 +0,0 @@
-import flask
-
-mapp = flask.Flask(__name__)
-
-@mapp.route("/")
-def hello():
-    return "mitmproxy"

libmproxy/cmdline.py

@@ -1,24 +1,18 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import absolute_import
+import os
+import re
+import configargparse
+from netlib import http
+from . import filt, utils, version
+from .proxy import config
 
-import proxy
-import re, filt
-import argparse
+APP_HOST = "mitm.it"
+APP_PORT = 80
+
+
+class ParseException(Exception):
+    pass
 
-class ParseException(Exception): pass
-class OptionException(Exception): pass
 
 def _parse_hook(s):
     sep, rem = s[0], s[1:]
@@ -29,13 +23,15 @@ def _parse_hook(s):
     elif len(parts) == 3:
         patt, a, b = parts
     else:
-        raise ParseException("Malformed hook specifier - too few clauses: %s"%s)
+        raise ParseException(
+            "Malformed hook specifier - too few clauses: %s" % s
+        )
 
     if not a:
-        raise ParseException("Empty clause: %s"%str(patt))
+        raise ParseException("Empty clause: %s" % str(patt))
 
     if not filt.parse(patt):
-        raise ParseException("Malformed filter pattern: %s"%patt)
+        raise ParseException("Malformed filter pattern: %s" % patt)
 
     return patt, a, b
 
@@ -70,7 +66,7 @@ def parse_replace_hook(s):
     try:
         re.compile(regex)
     except re.error, e:
-        raise ParseException("Malformed replacement regex: %s"%str(e.message))
+        raise ParseException("Malformed replacement regex: %s" % str(e.message))
     return patt, regex, replacement
 
 
@@ -103,6 +99,27 @@ def parse_setheader(s):
     return _parse_hook(s)
 
 
+def parse_server_spec(url):
+    normalized_url = re.sub("^https?2", "", url)
+
+    p = http.parse_url(normalized_url)
+    if not p or not p[1]:
+        raise configargparse.ArgumentTypeError(
+            "Invalid server specification: %s" % url
+        )
+
+    if url.lower().startswith("https2http"):
+        ssl = [True, False]
+    elif url.lower().startswith("http2https"):
+        ssl = [False, True]
+    elif url.lower().startswith("https"):
+        ssl = [True, True]
+    else:
+        ssl = [False, False]
+
+    return ssl + list(p[1:3])
+
+
 def get_common_options(options):
     stickycookie, stickyauth = None, None
     if options.stickycookie_filt:
@@ -111,178 +128,300 @@ def get_common_options(options):
     if options.stickyauth_filt:
         stickyauth = options.stickyauth_filt
 
+    stream_large_bodies = utils.parse_size(options.stream_large_bodies)
+
     reps = []
     for i in options.replace:
         try:
             p = parse_replace_hook(i)
         except ParseException, e:
-            raise OptionException(e.message)
+            raise configargparse.ArgumentTypeError(e.message)
         reps.append(p)
     for i in options.replace_file:
         try:
             patt, rex, path = parse_replace_hook(i)
         except ParseException, e:
-            raise OptionException(e.message)
+            raise configargparse.ArgumentTypeError(e.message)
         try:
             v = open(path, "rb").read()
         except IOError, e:
-            raise OptionException("Could not read replace file: %s"%path)
+            raise configargparse.ArgumentTypeError(
+                "Could not read replace file: %s" % path
+            )
         reps.append((patt, rex, v))
 
-
     setheaders = []
     for i in options.setheader:
         try:
             p = parse_setheader(i)
         except ParseException, e:
-            raise OptionException(e.message)
+            raise configargparse.ArgumentTypeError(e.message)
         setheaders.append(p)
 
     return dict(
-        anticache = options.anticache,
-        anticomp = options.anticomp,
-        client_replay = options.client_replay,
-        eventlog = options.eventlog,
-        kill = options.kill,
-        no_server = options.no_server,
-        refresh_server_playback = not options.norefresh,
-        rheaders = options.rheaders,
-        rfile = options.rfile,
-        replacements = reps,
-        setheaders = setheaders,
-        server_replay = options.server_replay,
-        script = options.script,
-        stickycookie = stickycookie,
-        stickyauth = stickyauth,
-        showhost = options.showhost,
-        wfile = options.wfile,
-        verbosity = options.verbose,
-        nopop = options.nopop,
+        app=options.app,
+        app_host=options.app_host,
+        app_port=options.app_port,
+
+        anticache=options.anticache,
+        anticomp=options.anticomp,
+        client_replay=options.client_replay,
+        kill=options.kill,
+        no_server=options.no_server,
+        refresh_server_playback=not options.norefresh,
+        rheaders=options.rheaders,
+        rfile=options.rfile,
+        replacements=reps,
+        setheaders=setheaders,
+        server_replay=options.server_replay,
+        scripts=options.scripts,
+        stickycookie=stickycookie,
+        stickyauth=stickyauth,
+        stream_large_bodies=stream_large_bodies,
+        showhost=options.showhost,
+        outfile=options.outfile,
+        verbosity=options.verbose,
+        nopop=options.nopop,
+        replay_ignore_content = options.replay_ignore_content,
+        replay_ignore_params = options.replay_ignore_params,
+        replay_ignore_payload_params = options.replay_ignore_payload_params
     )
 
 
 def common_options(parser):
     parser.add_argument(
-        "-b",
-        action="store", type = str, dest="addr", default='',
-        help = "Address to bind proxy to (defaults to all interfaces)"
+        '--version',
+        action= 'version',
+        version= "%(prog)s" + " " + version.VERSION
+    )
+    parser.add_argument(
+        '--shortversion',
+        action= 'version',
+        help = "show program's short version number and exit",
+        version = version.VERSION
     )
     parser.add_argument(
         "--anticache",
         action="store_true", dest="anticache", default=False,
-        help="Strip out request headers that might cause the server to return 304-not-modified."
+
+        help="""
+            Strip out request headers that might cause the server to return
+            304-not-modified.
+        """
     )
     parser.add_argument(
-        "--confdir",
-        action="store", type = str, dest="confdir", default='~/.mitmproxy',
-        help = "Configuration directory. (~/.mitmproxy)"
-    )
-    parser.add_argument(
-        "-e",
-        action="store_true", dest="eventlog",
-        help="Show event log."
-    )
-    parser.add_argument(
-        "-n",
-        action="store_true", dest="no_server",
-        help="Don't start a proxy server."
-    )
-    parser.add_argument(
-        "-p",
-        action="store", type = int, dest="port", default=8080,
-        help = "Proxy service port."
-    )
-    parser.add_argument(
-        "-P",
-        action="store", dest="reverse_proxy", default=None,
-        help="Reverse proxy to upstream server: http[s]://host[:port]"
-    )
-    parser.add_argument(
-        "-q",
-        action="store_true", dest="quiet",
-        help="Quiet."
-    )
-    parser.add_argument(
-        "-r",
-        action="store", dest="rfile", default=None,
-        help="Read flows from file."
-    )
-    parser.add_argument(
-        "-s",
-        action="store", dest="script", default=None,
-        help="Run a script."
-    )
-    parser.add_argument(
-        "-t",
-        action="store", dest="stickycookie_filt", default=None, metavar="FILTER",
-        help="Set sticky cookie filter. Matched against requests."
-    )
-    parser.add_argument(
-        "-T",
-        action="store_true", dest="transparent_proxy", default=False,
-        help="Set transparent proxy mode."
-    )
-    parser.add_argument(
-        "-u",
-        action="store", dest="stickyauth_filt", default=None, metavar="FILTER",
-        help="Set sticky auth filter. Matched against requests."
-    )
-    parser.add_argument(
-        "-v",
-        action="count", dest="verbose", default=1,
-        help="Increase verbosity. Can be passed multiple times."
-    )
-    parser.add_argument(
-        "-w",
-        action="store", dest="wfile", default=None,
-        help="Write flows to file."
-    )
-    parser.add_argument(
-        "-z",
-        action="store_true", dest="anticomp", default=False,
-        help="Try to convince servers to send us un-compressed data."
-    )
-    parser.add_argument(
-        "-Z",
-        action="store", dest="body_size_limit", default=None,
-        metavar="SIZE",
-        help="Byte size limit of HTTP request and response bodies."\
-             " Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
+        "--cadir",
+        action="store", type=str, dest="cadir", default=config.CA_DIR,
+        help="Location of the default mitmproxy CA files. (%s)"%config.CA_DIR
     )
     parser.add_argument(
         "--host",
         action="store_true", dest="showhost", default=False,
         help="Use the Host header to construct URLs for display."
     )
-
     parser.add_argument(
-        "--no-upstream-cert", default=False,
-        action="store_true", dest="no_upstream_cert",
-        help="Don't connect to upstream server to look up certificate details."
+        "-q", "--quiet",
+        action="store_true", dest="quiet",
+        help="Quiet."
+    )
+    parser.add_argument(
+        "-r", "--read-flows",
+        action="store", dest="rfile", default=None,
+        help="Read flows from file."
+    )
+    parser.add_argument(
+        "-s", "--script",
+        action="append", type=str, dest="scripts", default=[],
+        metavar='"script.py --bar"',
+        help="""
+            Run a script. Surround with quotes to pass script arguments. Can be
+            passed multiple times.
+        """
+    )
+    parser.add_argument(
+        "-t", "--stickycookie",
+        action="store",
+        dest="stickycookie_filt",
+        default=None,
+        metavar="FILTER",
+        help="Set sticky cookie filter. Matched against requests."
+    )
+    parser.add_argument(
+        "-u", "--stickyauth",
+        action="store", dest="stickyauth_filt", default=None, metavar="FILTER",
+        help="Set sticky auth filter. Matched against requests."
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="store_const", dest="verbose", default=1, const=2,
+        help="Increase event log verbosity."
+    )
+    outfile = parser.add_mutually_exclusive_group()
+    outfile.add_argument(
+        "-w", "--wfile",
+        action="store", dest="outfile", type=lambda f: (f, "wb"),
+        help="Write flows to file."
+    )
+    outfile.add_argument(
+        "-a", "--afile",
+        action="store", dest="outfile", type=lambda f: (f, "ab"),
+        help="Append flows to file."
+    )
+    parser.add_argument(
+        "-z", "--anticomp",
+        action="store_true", dest="anticomp", default=False,
+        help="Try to convince servers to send us un-compressed data."
+    )
+    parser.add_argument(
+        "-Z", "--body-size-limit",
+        action="store", dest="body_size_limit", default=None,
+        metavar="SIZE",
+        help="Byte size limit of HTTP request and response bodies."
+             " Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
+    )
+    parser.add_argument(
+        "--stream",
+        action="store", dest="stream_large_bodies", default=None,
+        metavar="SIZE",
+        help="""
+            Stream data to the client if response body exceeds the given
+            threshold. If streamed, the body will not be stored in any way.
+            Understands k/m/g suffixes, i.e. 3m for 3 megabytes.
+         """
     )
 
-    group = parser.add_argument_group("Web App")
+    group = parser.add_argument_group("Proxy Options")
+    # We could make a mutually exclusive group out of -R, -U, -T, but we don't
+    # do that because  - --upstream-server should be in that group as well, but
+    # it's already in a different group.  - our own error messages are more
+    # helpful
     group.add_argument(
-        "-a",
-        action="store_true", dest="app", default=False,
-        help="Enable the mitmproxy web app."
+        "-b", "--bind-address",
+        action="store", type=str, dest="addr", default='',
+        help="Address to bind proxy to (defaults to all interfaces)"
+    )
+    group.add_argument(
+        "-I", "--ignore",
+        action="append", type=str, dest="ignore_hosts", default=[],
+        metavar="HOST",
+        help="""
+            Ignore host and forward all traffic without processing it. In
+            transparent mode, it is recommended to use an IP address (range),
+            not the hostname. In regular mode, only SSL traffic is ignored and
+            the hostname should be used. The supplied value is interpreted as a
+            regular expression and matched on the ip or the hostname. Can be
+            passed multiple times.
+        """
+    )
+    group.add_argument(
+        "--tcp",
+        action="append", type=str, dest="tcp_hosts", default=[],
+        metavar="HOST",
+        help="""
+            Generic TCP SSL proxy mode for all hosts that match the pattern.
+            Similar to --ignore, but SSL connections are intercepted. The
+            communication contents are printed to the event log in verbose mode.
+        """
+    )
+    group.add_argument(
+        "-n", "--no-server",
+        action="store_true", dest="no_server",
+        help="Don't start a proxy server."
+    )
+    group.add_argument(
+        "-p", "--port",
+        action="store", type=int, dest="port", default=8080,
+        help="Proxy service port."
+    )
+    group.add_argument(
+        "-R", "--reverse",
+        action="store",
+        type=parse_server_spec,
+        dest="reverse_proxy",
+        default=None,
+        help="""
+            Forward all requests to upstream HTTP server:
+            http[s][2http[s]]://host[:port]
+        """
+    )
+    group.add_argument(
+        "--socks",
+        action="store_true", dest="socks_proxy", default=False,
+        help="Set SOCKS5 proxy mode."
+    )
+    group.add_argument(
+        "-T", "--transparent",
+        action="store_true", dest="transparent_proxy", default=False,
+        help="Set transparent proxy mode."
+    )
+    group.add_argument(
+        "-U", "--upstream",
+        action="store",
+        type=parse_server_spec,
+        dest="upstream_proxy",
+        default=None,
+        help="Forward all requests to upstream proxy server: http://host[:port]"
+    )
+
+    group = parser.add_argument_group(
+        "Advanced Proxy Options",
+        """
+            The following options allow a custom adjustment of the proxy
+            behavior. Normally, you don't want to use these options directly and
+            use the provided wrappers instead (-R, -U, -T).
+        """
+    )
+    group.add_argument(
+        "--http-form-in", dest="http_form_in", default=None,
+        action="store", choices=("relative", "absolute"),
+        help="Override the HTTP request form accepted by the proxy"
+    )
+    group.add_argument(
+        "--http-form-out", dest="http_form_out", default=None,
+        action="store", choices=("relative", "absolute"),
+        help="Override the HTTP request form sent upstream by the proxy"
+    )
+
+    group = parser.add_argument_group("Onboarding App")
+    group.add_argument(
+        "--noapp",
+        action="store_false", dest="app", default=True,
+        help="Disable the mitmproxy onboarding app."
+    )
+    group.add_argument(
+        "--app-host",
+        action="store", dest="app_host", default=APP_HOST, metavar="host",
+        help="""
+            Domain to serve the onboarding app from. For transparent mode, use
+            an IP when a DNS entry for the app domain is not present. Default:
+            %s
+        """ % APP_HOST
+    )
+    group.add_argument(
+        "--app-port",
+        action="store",
+        dest="app_port",
+        default=APP_PORT,
+        type=int,
+        metavar="80",
+        help="Port to serve the onboarding app from."
     )
 
     group = parser.add_argument_group("Client Replay")
     group.add_argument(
-        "-c",
+        "-c", "--client-replay",
         action="store", dest="client_replay", default=None, metavar="PATH",
         help="Replay client requests from a saved file."
     )
 
     group = parser.add_argument_group("Server Replay")
     group.add_argument(
-        "-S",
+        "-S", "--server-replay",
         action="store", dest="server_replay", default=None, metavar="PATH",
         help="Replay server responses from a saved file."
     )
     group.add_argument(
-        "-k",
+        "-k", "--kill",
         action="store_true", dest="kill", default=False,
         help="Kill extra requests during replay."
     )
@@ -290,19 +429,47 @@ def common_options(parser):
         "--rheader",
         action="append", dest="rheaders", type=str,
         help="Request headers to be considered during replay. "
-           "Can be passed multiple times."
+             "Can be passed multiple times."
     )
     group.add_argument(
         "--norefresh",
         action="store_true", dest="norefresh", default=False,
-        help= "Disable response refresh, "
-        "which updates times in cookies and headers for replayed responses."
+        help="""
+            Disable response refresh, which updates times in cookies and headers
+            for replayed responses.
+        """
     )
     group.add_argument(
         "--no-pop",
         action="store_true", dest="nopop", default=False,
         help="Disable response pop from response flow. "
-        "This makes it possible to replay same response multiple times."
+             "This makes it possible to replay same response multiple times."
+    )
+    payload = group.add_mutually_exclusive_group()
+    payload.add_argument(
+        "--replay-ignore-content",
+        action="store_true", dest="replay_ignore_content", default=False,
+        help="""
+            Ignore request's content while searching for a saved flow to replay
+        """
+    )
+    payload.add_argument(
+        "--replay-ignore-payload-param",
+        action="append", dest="replay_ignore_payload_params", type=str,
+        help="""
+            Request's payload parameters (application/x-www-form-urlencoded) to
+            be ignored while searching for a saved flow to replay.
+            Can be passed multiple times.
+        """
+    )
+
+    group.add_argument(
+        "--replay-ignore-param",
+        action="append", dest="replay_ignore_params", type=str,
+        help="""
+            Request's parameters to be ignored while searching for a saved flow
+            to replay. Can be passed multiple times.
+        """
     )
 
     group = parser.add_argument_group(
@@ -321,12 +488,14 @@ def common_options(parser):
     )
     group.add_argument(
         "--replace-from-file",
-        action="append", type=str, dest="replace_file", default=[],
-        metavar="PATH",
-        help="Replacement pattern, where the replacement clause is a path to a file."
+        action = "append", type=str, dest="replace_file", default=[],
+        metavar = "PATH",
+        help = """
+            Replacement pattern, where the replacement clause is a path to a
+            file.
+        """
     )
 
-
     group = parser.add_argument_group(
         "Set Headers",
         """
@@ -342,13 +511,11 @@ def common_options(parser):
         help="Header set pattern."
     )
 
-
     group = parser.add_argument_group(
         "Proxy Authentication",
         """
             Specify which users are allowed to access the proxy and the method
-            used for authenticating them. These options are ignored if the
-            proxy is in transparent or reverse proxy mode.
+            used for authenticating them.
         """
     )
     user_specification_group = group.add_mutually_exclusive_group()
@@ -362,13 +529,129 @@ def common_options(parser):
         "--singleuser",
         action="store", dest="auth_singleuser", type=str,
         metavar="USER",
-        help="Allows access to a a single user, specified in the form username:password."
+        help="""
+            Allows access to a a single user, specified in the form
+            username:password.
+        """
     )
     user_specification_group.add_argument(
         "--htpasswd",
-        action="store", dest="auth_htpasswd", type=argparse.FileType('r'),
+        action="store", dest="auth_htpasswd", type=str,
         metavar="PATH",
         help="Allow access to users specified in an Apache htpasswd file."
     )
 
-    proxy.certificate_option_group(parser)
+    config.ssl_option_group(parser)
+
+
+def mitmproxy():
+    # Don't import libmproxy.console for mitmdump, urwid is not available on all
+    # platforms.
+    from .console import palettes
+
+    parser = configargparse.ArgumentParser(
+        usage="%(prog)s [options]",
+        args_for_setting_config_path = ["--conf"],
+        default_config_files = [
+            os.path.join(config.CA_DIR, "common.conf"),
+            os.path.join(config.CA_DIR, "mitmproxy.conf")
+        ],
+        add_config_file_help = True,
+        add_env_var_help = True
+    )
+    common_options(parser)
+    parser.add_argument(
+        "--palette", type=str, default="dark",
+        action="store", dest="palette",
+        help="Select color palette: " + ", ".join(palettes.palettes.keys())
+    )
+    parser.add_argument(
+        "-e", "--eventlog",
+        action="store_true", dest="eventlog",
+        help="Show event log."
+    )
+    group = parser.add_argument_group(
+        "Filters",
+        "See help in mitmproxy for filter expression syntax."
+    )
+    group.add_argument(
+        "-i", "--intercept", action="store",
+        type=str, dest="intercept", default=None,
+        help="Intercept filter expression."
+    )
+    return parser
+
+
+def mitmdump():
+    parser = configargparse.ArgumentParser(
+        usage="%(prog)s [options] [filter]",
+        args_for_setting_config_path = ["--conf"],
+        default_config_files = [
+            os.path.join(config.CA_DIR, "common.conf"),
+            os.path.join(config.CA_DIR, "mitmdump.conf")
+        ],
+        add_config_file_help = True,
+        add_env_var_help = True
+    )
+
+    common_options(parser)
+    parser.add_argument(
+        "--keepserving",
+        action= "store_true", dest="keepserving", default=False,
+        help= """
+            Continue serving after client playback or file read. We exit by
+            default.
+        """
+    )
+    parser.add_argument(
+        "-d", "--detail",
+        action="count", dest="flow_detail", default=1,
+        help="Increase flow detail display level. Can be passed multiple times."
+    )
+    parser.add_argument('args', nargs="...")
+    return parser
+
+
+def mitmweb():
+    parser = configargparse.ArgumentParser(
+        usage="%(prog)s [options]",
+        args_for_setting_config_path = ["--conf"],
+        default_config_files = [
+            os.path.join(config.CA_DIR, "common.conf"),
+            os.path.join(config.CA_DIR, "mitmweb.conf")
+        ],
+        add_config_file_help = True,
+        add_env_var_help = True
+    )
+
+    group = parser.add_argument_group("Mitmweb")
+    group.add_argument(
+        "--wport",
+        action="store", type=int, dest="wport", default=8081,
+        metavar="PORT",
+        help="Mitmweb port."
+    )
+    group.add_argument(
+        "--wiface",
+        action="store", dest="wiface", default="127.0.0.1",
+        metavar="IFACE",
+        help="Mitmweb interface."
+    )
+    group.add_argument(
+        "--wdebug",
+        action="store_true", dest="wdebug",
+        help="Turn on mitmweb debugging"
+    )
+
+    common_options(parser)
+    group = parser.add_argument_group(
+        "Filters",
+        "See help in mitmproxy for filter expression syntax."
+    )
+    group.add_argument(
+        "-i", "--intercept", action="store",
+        type=str, dest="intercept", default=None,
+        help="Intercept filter expression."
+    )
+    return parser
+

libmproxy/console/__init__.py

@@ -1,23 +1,9 @@
-# Copyright (C) 2010  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
+from __future__ import absolute_import
 import mailcap, mimetypes, tempfile, os, subprocess, glob, time, shlex, stat
-import os.path, sys, weakref
+import os.path, sys, weakref, traceback
 import urwid
-from .. import controller, utils, flow
-import flowlist, flowview, help, common, grideditor, palettes, contentview, flowdetailview
+from .. import controller, utils, flow, script, proxy
+from . import flowlist, flowview, help, common, grideditor, palettes, contentview, flowdetailview
 
 EVENTLOG_SIZE = 500
 
@@ -92,7 +78,6 @@ class PathEdit(urwid.Edit, _PathCompleter):
 class ActionBar(common.WWrap):
     def __init__(self):
         self.message("")
-        self.expire = None
 
     def selectable(self):
         return True
@@ -144,6 +129,14 @@ class StatusBar(common.WWrap):
                 r.append(":%s in file]"%self.master.server_playback.count())
             else:
                 r.append(":%s to go]"%self.master.server_playback.count())
+        if self.master.get_ignore_filter():
+            r.append("[")
+            r.append(("heading_key", "I"))
+            r.append("gnore:%d]" % len(self.master.get_ignore_filter()))
+        if self.master.get_tcp_filter():
+            r.append("[")
+            r.append(("heading_key", "T"))
+            r.append("CP:%d]" % len(self.master.get_tcp_filter()))
         if self.master.state.intercept_txt:
             r.append("[")
             r.append(("heading_key", "i"))
@@ -160,10 +153,6 @@ class StatusBar(common.WWrap):
             r.append("[")
             r.append(("heading_key", "u"))
             r.append(":%s]"%self.master.stickyauth_txt)
-        if self.master.server.config.reverse_proxy:
-            r.append("[")
-            r.append(("heading_key", "P"))
-            r.append(":%s]"%utils.unparse_url(*self.master.server.config.reverse_proxy))
         if self.master.state.default_body_view.name != "Auto":
             r.append("[")
             r.append(("heading_key", "M"))
@@ -184,15 +173,23 @@ class StatusBar(common.WWrap):
             opts.append("no-upstream-cert")
         if self.master.state.follow_focus:
             opts.append("following")
+        if self.master.stream_large_bodies:
+            opts.append("stream:%s" % utils.pretty_size(self.master.stream_large_bodies.max_size))
 
         if opts:
             r.append("[%s]"%(":".join(opts)))
 
-        if self.master.script:
-            r.append("[script:%s]"%self.master.script.path)
-
-        if self.master.debug:
-            r.append("[lt:%0.3f]"%self.master.looptime)
+        if self.master.server.config.mode in ["reverse", "upstream"]:
+            dst = self.master.server.config.mode.dst
+            scheme = "https" if dst[0] else "http"
+            if dst[1] != dst[0]:
+                scheme += "2https" if dst[1] else "http"
+            r.append("[dest:%s]"%utils.unparse_url(scheme, *dst[2:]))
+        if self.master.scripts:
+            r.append("[")
+            r.append(("heading_key", "s"))
+            r.append("cripts:%s]"%len(self.master.scripts))
+        # r.append("[lt:%0.3f]"%self.master.looptime)
 
         if self.master.stream:
             r.append("[W:%s]"%self.master.stream_path)
@@ -204,18 +201,19 @@ class StatusBar(common.WWrap):
             self.message("")
 
         fc = self.master.state.flow_count()
-        if self.master.currentflow:
-            idx = self.master.state.view.index(self.master.currentflow) + 1
-            t = [
-                ('heading', ("[%s/%s]"%(idx, fc)).ljust(9))
-            ]
+        if self.master.state.focus is None:
+            offset = 0
         else:
-            t = [
-                ('heading', ("[%s]"%fc).ljust(9))
-            ]
+            offset = min(self.master.state.focus + 1, fc)
+        t = [
+            ('heading', ("[%s/%s]"%(offset, fc)).ljust(9))
+        ]
 
         if self.master.server.bound:
-            boundaddr = "[%s:%s]"%(self.master.server.address or "*", self.master.server.port)
+            host = self.master.server.address.host
+            if host == "0.0.0.0":
+                host = "*"
+            boundaddr = "[%s:%s]"%(host, self.master.server.address.port)
         else:
             boundaddr = ""
         t.extend(self.get_status())
@@ -263,7 +261,10 @@ class ConsoleState(flow.State):
         self.focus = None
         self.follow_focus = None
         self.default_body_view = contentview.get("Auto")
+
+        self.view_mode = common.VIEW_LIST
         self.view_flow_mode = common.VIEW_FLOW_REQUEST
+
         self.last_script = ""
         self.last_saveload = ""
         self.flowsettings = weakref.WeakKeyDictionary()
@@ -276,16 +277,16 @@ class ConsoleState(flow.State):
         d = self.flowsettings.get(flow, {})
         return d.get(key, default)
 
-    def add_request(self, req):
-        f = flow.State.add_request(self, req)
+    def add_flow(self, f):
+        super(ConsoleState, self).add_flow(f)
         if self.focus is None:
             self.set_focus(0)
         elif self.follow_focus:
             self.set_focus(len(self.view) - 1)
         return f
 
-    def add_response(self, resp):
-        f = flow.State.add_response(self, resp)
+    def update_flow(self, f):
+        super(ConsoleState, self).update_flow(f)
         if self.focus is None:
             self.set_focus(0)
         return f
@@ -308,6 +309,9 @@ class ConsoleState(flow.State):
                 idx = 0
             self.focus = idx
 
+    def set_focus_flow(self, f):
+        self.set_focus(self.view.index(f))
+
     def get_from_pos(self, pos):
         if len(self.view) <= pos or pos < 0:
             return None, None
@@ -320,6 +324,10 @@ class ConsoleState(flow.State):
         return self.get_from_pos(pos-1)
 
     def delete_flow(self, f):
+        if f in self.view and self.view.index(f) <= self.focus:
+            self.focus -= 1
+        if self.focus < 0:
+            self.focus = None
         ret = flow.State.delete_flow(self, f)
         self.set_focus(self.focus)
         return ret
@@ -328,10 +336,12 @@ class ConsoleState(flow.State):
 
 class Options(object):
     attributes = [
+        "app",
+        "app_domain",
+        "app_ip",
         "anticache",
         "anticomp",
         "client_replay",
-        "debug",
         "eventlog",
         "keepserving",
         "kill",
@@ -339,7 +349,7 @@ class Options(object):
         "no_server",
         "refresh_server_playback",
         "rfile",
-        "script",
+        "scripts",
         "showhost",
         "replacements",
         "rheaders",
@@ -347,6 +357,7 @@ class Options(object):
         "server_replay",
         "stickycookie",
         "stickyauth",
+        "stream_large_bodies",
         "verbosity",
         "wfile",
         "nopop",
@@ -395,6 +406,8 @@ class ConsoleMaster(flow.FlowMaster):
             print >> sys.stderr, "Sticky auth error:", r
             sys.exit(1)
 
+        self.set_stream_large_bodies(options.stream_large_bodies)
+
         self.refresh_server_playback = options.refresh_server_playback
         self.anticache = options.anticache
         self.anticomp = options.anticomp
@@ -412,13 +425,12 @@ class ConsoleMaster(flow.FlowMaster):
         if options.server_replay:
             self.server_playback_path(options.server_replay)
 
-        self.debug = options.debug
-
-        if options.script:
-            err = self.load_script(options.script)
-            if err:
-                print >> sys.stderr, "Script load error:", err
-                sys.exit(1)
+        if options.scripts:
+            for i in options.scripts:
+                err = self.load_script(i)
+                if err:
+                    print >> sys.stderr, "Script load error:", err
+                    sys.exit(1)
 
         if options.wfile:
             err = self.start_stream(options.wfile)
@@ -426,6 +438,9 @@ class ConsoleMaster(flow.FlowMaster):
                 print >> sys.stderr, "Script load error:", err
                 sys.exit(1)
 
+        if options.app:
+            self.start_app(self.options.app_host, self.options.app_port)
+
     def start_stream(self, path):
         path = os.path.expanduser(path)
         try:
@@ -435,42 +450,43 @@ class ConsoleMaster(flow.FlowMaster):
             return str(v)
         self.stream_path = path
 
-
     def _run_script_method(self, method, s, f):
         status, val = s.run(method, f)
         if val:
             if status:
-                self.add_event("Method %s return: %s"%(method, val))
+                self.add_event("Method %s return: %s"%(method, val), "debug")
             else:
-                self.add_event("Method %s error: %s"%(method, val[1]))
+                self.add_event("Method %s error: %s"%(method, val[1]), "error")
 
-    def run_script_once(self, path, f):
-        if not path:
+    def run_script_once(self, command, f):
+        if not command:
             return
-        self.add_event("Running script on flow: %s"%path)
-        ret = self.get_script(path)
-        if ret[0]:
+        self.add_event("Running script on flow: %s"%command, "debug")
+
+        try:
+            s = script.Script(command, self)
+        except script.ScriptError, v:
             self.statusbar.message("Error loading script.")
-            self.add_event("Error loading script:\n%s"%ret[0])
+            self.add_event("Error loading script:\n%s"%v.args[0], "error")
             return
-        s = ret[1]
+
         if f.request:
             self._run_script_method("request", s, f)
         if f.response:
             self._run_script_method("response", s, f)
         if f.error:
             self._run_script_method("error", s, f)
-        s.run("done")
+        s.unload()
         self.refresh_flow(f)
-        self.state.last_script = path
+        self.state.last_script = command
 
-    def set_script(self, path):
-        if not path:
+    def set_script(self, command):
+        if not command:
             return
-        ret = self.load_script(path)
+        ret = self.load_script(command)
         if ret:
             self.statusbar.message(ret)
-        self.state.last_script = path
+        self.state.last_script = command
 
     def toggle_eventlog(self):
         self.eventlog = not self.eventlog
@@ -500,7 +516,8 @@ class ConsoleMaster(flow.FlowMaster):
             self.start_server_playback(
                 ret,
                 self.killextra, self.rheaders,
-                False, self.nopop
+                False, self.nopop,
+                self.options.replay_ignore_params, self.options.replay_ignore_content
             )
 
     def spawn_editor(self, data):
@@ -564,48 +581,43 @@ class ConsoleMaster(flow.FlowMaster):
         self.palette = palettes.palettes[name]
 
     def run(self):
-        self.currentflow = None
-
         self.ui = urwid.raw_display.Screen()
         self.ui.set_terminal_properties(256)
         self.ui.register_palette(self.palette)
         self.flow_list_walker = flowlist.FlowListWalker(self, self.state)
-
         self.view = None
         self.statusbar = None
         self.header = None
         self.body = None
         self.help_context = None
-
         self.prompting = False
         self.onekey = False
 
         self.view_flowlist()
 
-        self.server.start_slave(controller.Slave, controller.Channel(self.masterq))
+        self.server.start_slave(controller.Slave, controller.Channel(self.masterq, self.should_exit))
 
         if self.options.rfile:
             ret = self.load_flows(self.options.rfile)
             if ret and self.state.flow_count():
-                self.add_event("File truncated or corrupted. Loaded as many flows as possible.")
+                self.add_event("File truncated or corrupted. Loaded as many flows as possible.","error")
             elif not self.state.flow_count():
                 self.shutdown()
                 print >> sys.stderr, "Could not load file:", ret
                 sys.exit(1)
 
-        self.ui.run_wrapper(self.loop)
-        # If True, quit just pops out to flow list view.
+        try:
+            self.ui.run_wrapper(self.loop)
+        except Exception:
+            self.ui.stop()
+            sys.stdout.flush()
+            print >> sys.stderr, traceback.format_exc()
+            print >> sys.stderr, "mitmproxy has crashed!"
+            print >> sys.stderr, "Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy"
         print >> sys.stderr, "Shutting down..."
         sys.stderr.flush()
         self.shutdown()
 
-    def focus_current(self):
-        if self.currentflow:
-            try:
-                self.flow_list_walker.set_focus(self.state.view.index(self.currentflow))
-            except (IndexError, ValueError):
-                pass
-
     def make_view(self):
         self.view = urwid.Frame(
                         self.body,
@@ -640,8 +652,6 @@ class ConsoleMaster(flow.FlowMaster):
             self.ui.clear()
         if self.state.follow_focus:
             self.state.set_focus(self.state.flow_count())
-        else:
-            self.focus_current()
 
         if self.eventlog:
             self.body = flowlist.BodyPile(self)
@@ -649,7 +659,7 @@ class ConsoleMaster(flow.FlowMaster):
             self.body = flowlist.FlowListBox(self)
         self.statusbar = StatusBar(self, flowlist.footer)
         self.header = None
-        self.currentflow = None
+        self.state.view_mode = common.VIEW_LIST
 
         self.make_view()
         self.help_context = flowlist.help_context
@@ -658,8 +668,8 @@ class ConsoleMaster(flow.FlowMaster):
         self.body = flowview.FlowView(self, self.state, flow)
         self.header = flowview.FlowViewHeader(self, flow)
         self.statusbar = StatusBar(self, flowview.footer)
-        self.currentflow = flow
-
+        self.state.set_focus_flow(flow)
+        self.state.view_mode = common.VIEW_FLOW
         self.make_view()
         self.help_context = flowview.help_context
 
@@ -705,7 +715,6 @@ class ConsoleMaster(flow.FlowMaster):
         f.close()
         if self.flow_list_walker:
             self.sync_list_view()
-            self.focus_current()
         return reterr
 
     def path_prompt(self, prompt, text, callback, *args):
@@ -758,7 +767,7 @@ class ConsoleMaster(flow.FlowMaster):
         self.prompt_done()
 
     def accept_all(self):
-        self.state.accept_all()
+        self.state.accept_all(self)
 
     def set_limit(self, txt):
         v = self.state.set_limit(txt)
@@ -771,17 +780,7 @@ class ConsoleMaster(flow.FlowMaster):
     def change_default_display_mode(self, t):
         v = contentview.get_by_shortcut(t)
         self.state.default_body_view = v
-        if self.currentflow:
-            self.refresh_flow(self.currentflow)
-
-    def set_reverse_proxy(self, txt):
-        if not txt:
-            self.server.config.reverse_proxy = None
-        else:
-            s = utils.parse_proxy_spec(txt)
-            if not s:
-                return "Invalid reverse proxy specification"
-            self.server.config.reverse_proxy = s
+        self.refresh_focus()
 
     def drawscreen(self):
         size = self.ui.get_cols_rows()
@@ -790,21 +789,38 @@ class ConsoleMaster(flow.FlowMaster):
         return size
 
     def pop_view(self):
-        if self.currentflow:
-            self.view_flow(self.currentflow)
+        if self.state.view_mode == common.VIEW_FLOW:
+            self.view_flow(self.state.view[self.state.focus])
         else:
             self.view_flowlist()
 
+    def edit_scripts(self, scripts):
+        commands = [x[0] for x in scripts]  # remove outer array
+        if commands == [s.command for s in self.scripts]:
+            return
+
+        self.unload_scripts()
+        for command in commands:
+            self.load_script(command)
+
+    def edit_ignore_filter(self, ignore):
+        patterns = (x[0] for x in ignore)
+        self.set_ignore_filter(patterns)
+
+    def edit_tcp_filter(self, tcp):
+        patterns = (x[0] for x in tcp)
+        self.set_tcp_filter(patterns)
+
     def loop(self):
         changed = True
         try:
-            while not controller.should_exit:
+            while not self.should_exit.is_set():
                 startloop = time.time()
                 if changed:
                     self.statusbar.redraw()
                     size = self.drawscreen()
-                changed = self.tick(self.masterq)
-                self.ui.set_input_timeouts(max_wait=0.1)
+                changed = self.tick(self.masterq, 0.01)
+                self.ui.set_input_timeouts(max_wait=0.01)
                 keys = self.ui.get_input()
                 if keys:
                     changed = True
@@ -851,6 +867,22 @@ class ConsoleMaster(flow.FlowMaster):
                                         self.setheaders.set
                                     )
                                 )
+                            elif k == "I":
+                                self.view_grideditor(
+                                    grideditor.HostPatternEditor(
+                                        self,
+                                        [[x] for x in self.get_ignore_filter()],
+                                        self.edit_ignore_filter
+                                    )
+                                )
+                            elif k == "T":
+                                self.view_grideditor(
+                                    grideditor.HostPatternEditor(
+                                        self,
+                                        [[x] for x in self.get_tcp_filter()],
+                                        self.edit_tcp_filter
+                                    )
+                                )
                             elif k == "i":
                                 self.prompt(
                                     "Intercept filter: ",
@@ -874,16 +906,6 @@ class ConsoleMaster(flow.FlowMaster):
                                     contentview.view_prompts,
                                     self.change_default_display_mode
                                 )
-                            elif k == "P":
-                                if self.server.config.reverse_proxy:
-                                    p = utils.unparse_url(*self.server.config.reverse_proxy)
-                                else:
-                                    p = ""
-                                self.prompt(
-                                    "Reverse proxy: ",
-                                    p,
-                                    self.set_reverse_proxy
-                                )
                             elif k == "R":
                                 self.view_grideditor(
                                     grideditor.ReplaceEditor(
@@ -893,14 +915,21 @@ class ConsoleMaster(flow.FlowMaster):
                                     )
                                 )
                             elif k == "s":
-                                if self.script:
-                                    self.load_script(None)
-                                else:
-                                    self.path_prompt(
-                                        "Set script: ",
-                                        self.state.last_script,
-                                        self.set_script
+                                self.view_grideditor(
+                                    grideditor.ScriptEditor(
+                                        self,
+                                        [[i.command] for i in self.scripts],
+                                        self.edit_scripts
                                     )
+                                )
+                                #if self.scripts:
+                                #    self.load_script(None)
+                                #else:
+                                #    self.path_prompt(
+                                #        "Set script: ",
+                                #        self.state.last_script,
+                                #        self.set_script
+                                #    )
                             elif k == "S":
                                 if not self.server_playback:
                                     self.path_prompt(
@@ -966,7 +995,7 @@ class ConsoleMaster(flow.FlowMaster):
         if a == "h":
             self.showhost = not self.showhost
             self.sync_list_view()
-            self.refresh_flow(self.currentflow)
+            self.refresh_focus()
         elif a == "k":
             self.killextra = not self.killextra
         elif a == "n":
@@ -997,6 +1026,10 @@ class ConsoleMaster(flow.FlowMaster):
         self.state.delete_flow(f)
         self.sync_list_view()
 
+    def refresh_focus(self):
+        if self.state.view:
+            self.refresh_flow(self.state.view[self.state.focus])
+
     def refresh_flow(self, c):
         if hasattr(self.header, "refresh_flow"):
             self.header.refresh_flow(c)
@@ -1005,11 +1038,11 @@ class ConsoleMaster(flow.FlowMaster):
         if hasattr(self.statusbar, "refresh_flow"):
             self.statusbar.refresh_flow(c)
 
-    def process_flow(self, f, r):
-        if self.state.intercept and f.match(self.state.intercept) and not f.request.is_replay():
-            f.intercept()
+    def process_flow(self, f):
+        if self.state.intercept and f.match(self.state.intercept) and not f.request.is_replay:
+            f.intercept(self)
         else:
-            r.reply()
+            f.reply()
         self.sync_list_view()
         self.refresh_flow(f)
 
@@ -1017,35 +1050,34 @@ class ConsoleMaster(flow.FlowMaster):
         self.eventlist[:] = []
 
     def add_event(self, e, level="info"):
-        if level == "info":
-            e = urwid.Text(str(e))
-        elif level == "error":
-            e = urwid.Text(("error", str(e)))
+        needed = dict(error=0, info=1, debug=2).get(level, 1)
+        if self.options.verbosity < needed:
+            return
 
+        if level == "error":
+            e = urwid.Text(("error", str(e)))
+        else:
+            e = urwid.Text(str(e))
         self.eventlist.append(e)
         if len(self.eventlist) > EVENTLOG_SIZE:
             self.eventlist.pop(0)
         self.eventlist.set_focus(len(self.eventlist)-1)
 
     # Handlers
-    def handle_log(self, l):
-        self.add_event(l.msg)
-        l.reply()
-
-    def handle_error(self, r):
-        f = flow.FlowMaster.handle_error(self, r)
+    def handle_error(self, f):
+        f = flow.FlowMaster.handle_error(self, f)
         if f:
-            self.process_flow(f, r)
+            self.process_flow(f)
         return f
 
-    def handle_request(self, r):
-        f = flow.FlowMaster.handle_request(self, r)
+    def handle_request(self, f):
+        f = flow.FlowMaster.handle_request(self, f)
         if f:
-            self.process_flow(f, r)
+            self.process_flow(f)
         return f
 
-    def handle_response(self, r):
-        f = flow.FlowMaster.handle_response(self, r)
+    def handle_response(self, f):
+        f = flow.FlowMaster.handle_response(self, f)
         if f:
-            self.process_flow(f, r)
+            self.process_flow(f)
         return f

libmproxy/console/common.py

@@ -1,23 +1,13 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
+from __future__ import absolute_import
 import urwid
 import urwid.util
-from .. import utils, flow
+from .. import utils
+from ..protocol.http import CONTENT_MISSING
 
 
+VIEW_LIST = 0
+VIEW_FLOW = 1
+
 
 VIEW_FLOW_REQUEST = 0
 VIEW_FLOW_RESPONSE = 1
@@ -118,7 +108,7 @@ def raw_format_flow(f, focus, extended, padding):
 
     preamble = sum(i[1] for i in req) + len(req) -1
 
-    if f["intercepting"] and not f["req_acked"]:
+    if f["intercepted"] and not f["acked"]:
         uc = "intercept"
     elif f["resp_code"] or f["err_msg"]:
         uc = "text"
@@ -148,7 +138,7 @@ def raw_format_flow(f, focus, extended, padding):
         if f["resp_is_replay"]:
             resp.append(fcol(SYMBOL_REPLAY, "replay"))
         resp.append(fcol(f["resp_code"], ccol))
-        if f["intercepting"] and f["resp_code"] and not f["resp_acked"]:
+        if f["intercepted"] and f["resp_code"] and not f["acked"]:
             rc = "intercept"
         else:
             rc = "text"
@@ -156,6 +146,8 @@ def raw_format_flow(f, focus, extended, padding):
         if f["resp_ctype"]:
             resp.append(fcol(f["resp_ctype"], rc))
         resp.append(fcol(f["resp_clen"], rc))
+        resp.append(fcol(f["resp_rate"], rc))
+
     elif f["err_msg"]:
         resp.append(fcol(SYMBOL_RETURN, "error"))
         resp.append(
@@ -179,13 +171,13 @@ flowcache = FlowCache()
 
 def format_flow(f, focus, extended=False, hostheader=False, padding=2):
     d = dict(
-        intercepting = f.intercepting,
+        intercepted = f.intercepted,
+        acked = f.reply.acked,
 
         req_timestamp = f.request.timestamp_start,
-        req_is_replay = f.request.is_replay(),
+        req_is_replay = f.request.is_replay,
         req_method = f.request.method,
-        req_acked = f.request.reply.acked,
-        req_url = f.request.get_url(hostheader=hostheader),
+        req_url = f.request.pretty_url(hostheader=hostheader),
 
         err_msg = f.error.msg if f.error else None,
         resp_code = f.response.code if f.response else None,
@@ -193,15 +185,23 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
     if f.response:
         if f.response.content:
             contentdesc = utils.pretty_size(len(f.response.content))
-        elif f.response.content == flow.CONTENT_MISSING:
+        elif f.response.content == CONTENT_MISSING:
             contentdesc = "[content missing]"
         else:
             contentdesc = "[no content]"
+
+        if f.response.timestamp_end:
+            delta = f.response.timestamp_end - f.response.timestamp_start
+        else:
+            delta = 0
+        size = f.response.size()
+        rate = utils.pretty_size(size / ( delta if delta > 0 else 1 ) )
+
         d.update(dict(
             resp_code = f.response.code,
-            resp_is_replay = f.response.is_replay(),
-            resp_acked = f.response.reply.acked,
-            resp_clen = contentdesc
+            resp_is_replay = f.response.is_replay,
+            resp_clen = contentdesc,
+            resp_rate = "{0}/s".format(rate),
         ))
         t = f.response.headers["content-type"]
         if t:

libmproxy/console/contentview.py

@@ -1,24 +1,32 @@
-import re, cStringIO, traceback, json
-import urwid
-
-try: from PIL import Image
-except ImportError: import Image
-
-try: from PIL.ExifTags import TAGS
-except ImportError: from ExifTags import TAGS
+from __future__ import absolute_import
+import logging, subprocess, re, cStringIO, traceback, json, urwid
+from PIL import Image
+from PIL.ExifTags import TAGS
 
 import lxml.html, lxml.etree
 import netlib.utils
-import common
+from . import common
 from .. import utils, encoding, flow
 from ..contrib import jsbeautifier, html2text
-import subprocess
+from ..contrib.wbxml.ASCommandResponse import ASCommandResponse
 try:
     import pyamf
     from pyamf import remoting, flex
 except ImportError: # pragma nocover
     pyamf = None
 
+try:
+    import cssutils
+except ImportError: # pragma nocover
+    cssutils = None
+else:
+    cssutils.log.setLevel(logging.CRITICAL)
+
+    cssutils.ser.prefs.keepComments = True
+    cssutils.ser.prefs.omitLastSemicolon = False
+    cssutils.ser.prefs.indentClosingBrace = False
+    cssutils.ser.prefs.validOnly = False
+
 VIEW_CUTOFF = 1024*50
 
 
@@ -318,7 +326,23 @@ class ViewJavaScript:
         opts = jsbeautifier.default_options()
         opts.indent_size = 2
         res = jsbeautifier.beautify(content[:limit], opts)
-        return "JavaScript", _view_text(res, len(content), limit)
+        return "JavaScript", _view_text(res, len(res), limit)
+
+class ViewCSS:
+    name = "CSS"
+    prompt = ("css", "c")
+    content_types = [
+        "text/css"
+    ]
+
+    def __call__(self, hdrs, content, limit):
+        if cssutils:
+            sheet = cssutils.parseString(content)
+            beautified = sheet.cssText
+        else:
+            beautified = content
+
+        return "CSS", _view_text(beautified, len(beautified), limit)
 
 
 class ViewImage:
@@ -371,7 +395,10 @@ class ViewProtobuf:
 
     name = "Protocol Buffer"
     prompt = ("protobuf", "p")
-    content_types = ["application/x-protobuf"]
+    content_types = [
+        "application/x-protobuf",
+        "application/x-protobuffer",
+    ]
 
     @staticmethod
     def is_available():
@@ -400,15 +427,35 @@ class ViewProtobuf:
         txt = _view_text(decoded[:limit], len(decoded), limit)
         return "Protobuf", txt
 
+class ViewWBXML:
+    name = "WBXML"
+    prompt = ("wbxml", "w")
+    content_types = [
+        "application/vnd.wap.wbxml",
+        "application/vnd.ms-sync.wbxml"
+    ]
+
+    def __call__(self, hdrs, content, limit):
+        
+        try:
+            parser = ASCommandResponse(content)
+            parsedContent = parser.xmlString
+            txt = _view_text(parsedContent, len(parsedContent), limit)
+            return "WBXML", txt
+        except:
+        	return None
+
 views = [
     ViewAuto(),
     ViewRaw(),
     ViewHex(),
     ViewJSON(),
     ViewXML(),
+    ViewWBXML(),
     ViewHTML(),
     ViewHTMLOutline(),
     ViewJavaScript(),
+    ViewCSS(),
     ViewURLEncoded(),
     ViewMultipart(),
     ViewImage(),
@@ -460,10 +507,10 @@ def get_content_view(viewmode, hdrItems, content, limit, logfunc):
     try:
         ret = viewmode(hdrs, content, limit)
     # Third-party viewers can fail in unexpected ways...
-    except Exception, e:
+    except Exception:
         s = traceback.format_exc()
         s = "Content viewer failed: \n"  + s
-        logfunc(s)
+        logfunc(s, "error")
         ret = None
     if not ret:
         ret = get("Raw")(hdrs, content, limit)

libmproxy/console/flowdetailview.py

@@ -1,20 +1,7 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
+from __future__ import absolute_import
 import urwid
-import common
+from . import common
+from .. import utils
 
 footer = [
     ('heading_key', "q"), ":back ",
@@ -48,8 +35,17 @@ class FlowDetailsView(urwid.ListBox):
         title = urwid.AttrWrap(title, "heading")
         text.append(title)
 
-        if self.flow.response:
-            c = self.flow.response.cert
+        if self.flow.server_conn:
+            text.append(urwid.Text([("head", "Server Connection:")]))
+            sc = self.flow.server_conn
+            parts = [
+                ["Address", "%s:%s" % sc.address()],
+                ["Start time", utils.format_timestamp(sc.timestamp_start)],
+                ["End time", utils.format_timestamp(sc.timestamp_end) if sc.timestamp_end else "active"],
+            ]
+            text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
+
+            c = self.flow.server_conn.cert
             if c:
                 text.append(urwid.Text([("head", "Server Certificate:")]))
                 parts = [
@@ -58,19 +54,13 @@ class FlowDetailsView(urwid.ListBox):
                     ["Valid to", str(c.notafter)],
                     ["Valid from", str(c.notbefore)],
                     ["Serial", str(c.serial)],
-                ]
-
-                parts.append(
                     [
                         "Subject",
                         urwid.BoxAdapter(
                             urwid.ListBox(common.format_keyvals(c.subject, key="highlight", val="text")),
                             len(c.subject)
                         )
-                    ]
-                )
-
-                parts.append(
+                    ],
                     [
                         "Issuer",
                         urwid.BoxAdapter(
@@ -78,7 +68,7 @@ class FlowDetailsView(urwid.ListBox):
                             len(c.issuer)
                         )
                     ]
-                )
+                ]
 
                 if c.altnames:
                     parts.append(
@@ -89,13 +79,14 @@ class FlowDetailsView(urwid.ListBox):
                     )
                 text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
 
-        if self.flow.request.client_conn:
+        if self.flow.client_conn:
             text.append(urwid.Text([("head", "Client Connection:")]))
-            cc = self.flow.request.client_conn
+            cc = self.flow.client_conn
             parts = [
-                ["Address", "%s:%s"%tuple(cc.address)],
-                ["Requests", "%s"%cc.requestcount],
-                ["Closed", "%s"%cc.close],
+                ["Address", "%s:%s" % cc.address()],
+                ["Start time", utils.format_timestamp(cc.timestamp_start)],
+                # ["Requests", "%s"%cc.requestcount],
+                ["End time", utils.format_timestamp(cc.timestamp_end) if cc.timestamp_end else "active"],
             ]
             text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
 

libmproxy/console/flowlist.py

@@ -1,20 +1,6 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
+from __future__ import absolute_import
 import urwid
-import common
+from . import common
 
 def _mkhelp():
     text = []
@@ -83,7 +69,7 @@ class BodyPile(urwid.Pile):
             else:
                 self.widget_list[1].header = self.inactive_header
             key = None
-        elif key == "v":
+        elif key == "e":
             self.master.toggle_eventlog()
             key = None
 
@@ -134,13 +120,15 @@ class ConnectionItem(common.WWrap):
             self.master.start_server_playback(
                 [i.copy() for i in self.master.state.view],
                 self.master.killextra, self.master.rheaders,
-                False, self.master.nopop
+                False, self.master.nopop,
+                self.master.options.replay_ignore_params, self.master.options.replay_ignore_content
             )
         elif k == "t":
             self.master.start_server_playback(
                 [self.flow.copy()],
                 self.master.killextra, self.master.rheaders,
-                False, self.master.nopop
+                False, self.master.nopop,
+                self.master.options.replay_ignore_params, self.master.options.replay_ignore_content
             )
         else:
             self.master.path_prompt(
@@ -152,7 +140,7 @@ class ConnectionItem(common.WWrap):
     def keypress(self, (maxcol,), key):
         key = common.shortcuts(key)
         if key == "a":
-            self.flow.accept_intercept()
+            self.flow.accept_intercept(self.master)
             self.master.sync_list_view()
         elif key == "d":
             self.flow.kill(self.master)
@@ -160,10 +148,8 @@ class ConnectionItem(common.WWrap):
             self.master.sync_list_view()
         elif key == "D":
             f = self.master.duplicate_flow(self.flow)
-            self.master.currentflow = f
-            self.master.focus_current()
+            self.master.view_flow(f)
         elif key == "r":
-            self.flow.backup()
             r = self.master.replay_request(self.flow)
             if r:
                 self.master.statusbar.message(r)

libmproxy/console/flowview.py

@@ -1,22 +1,13 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import os, sys
+from __future__ import absolute_import
+import os, sys, copy
 import urwid
-import common, grideditor, contentview
+from . import common, grideditor, contentview
 from .. import utils, flow, controller
+from ..protocol.http import HTTPResponse, CONTENT_MISSING, decoded
+
+
+class SearchError(Exception): pass
+
 
 def _mkhelp():
     text = []
@@ -78,6 +69,9 @@ def _mkhelp():
         ("tab", "toggle request/response view"),
         ("space", "next flow"),
         ("|", "run script on this flow"),
+        ("/", "search in response body (case sensitive)"),
+        ("n", "repeat search forward"),
+        ("N", "repeat search backwards"),
     ]
     text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
     return text
@@ -124,8 +118,12 @@ class FlowView(common.WWrap):
         ("options", "o"),
         ("edit raw", "e"),
     ]
+
+    highlight_color = "focusfield"
+
     def __init__(self, master, state, flow):
         self.master, self.state, self.flow = master, state, flow
+        self.last_displayed_body = None
         if self.state.view_flow_mode == common.VIEW_FLOW_RESPONSE:
             self.view_response()
         else:
@@ -144,57 +142,84 @@ class FlowView(common.WWrap):
             limit = sys.maxint
         else:
             limit = contentview.VIEW_CUTOFF
-        return cache.callback(
+        description, text_objects = cache.callback(
                     self, "_cached_content_view",
                     viewmode,
                     tuple(tuple(i) for i in conn.headers.lst),
                     conn.content,
                     limit
                 )
+        return (description, text_objects)
 
-    def conn_text(self, conn):
-        txt = common.format_keyvals(
-                [(h+":", v) for (h, v) in conn.headers.lst],
-                key = "header",
-                val = "text"
-            )
-        if conn.content is not None:
-            override = self.state.get_flow_setting(
-                self.flow,
-                (self.state.view_flow_mode, "prettyview"),
-            )
-            viewmode = self.state.default_body_view if override is None else override
-
-            if conn.content == flow.CONTENT_MISSING:
+    def cont_view_handle_missing(self, conn, viewmode):
+            if conn.content == CONTENT_MISSING:
                 msg, body = "", [urwid.Text([("error", "[content missing]")])]
             else:
                 msg, body = self.content_view(viewmode, conn)
 
-            cols = [
-                urwid.Text(
-                    [
-                        ("heading", msg),
-                    ]
+            return (msg, body)
+
+    def viewmode_get(self, override):
+        return self.state.default_body_view if override is None else override
+
+    def override_get(self):
+        return self.state.get_flow_setting(self.flow,
+                (self.state.view_flow_mode, "prettyview"))
+
+    def conn_text_raw(self, conn):
+        """
+            Based on a request/response, conn, returns the elements for
+            display.
+        """
+        headers = common.format_keyvals(
+                [(h+":", v) for (h, v) in conn.headers.lst],
+                key = "header",
+                val = "text"
+            )
+        override = self.override_get()
+        viewmode = self.viewmode_get(override)
+        msg, body = self.cont_view_handle_missing(conn, viewmode)
+        return headers, msg, body
+
+    def conn_text_merge(self, headers, msg, body):
+        """
+            Grabs what is returned by conn_text_raw and merges them all
+            toghether, mainly used by conn_text and search
+        """
+        override = self.override_get()
+        viewmode = self.viewmode_get(override)
+
+        cols = [urwid.Text(
+                [
+                    ("heading", msg),
+                ]
+            )
+        ]
+
+        if override is not None:
+            cols.append(urwid.Text([
+                        " ",
+                        ('heading', "["),
+                        ('heading_key', "m"),
+                        ('heading', (":%s]"%viewmode.name)),
+                    ],
+                    align="right"
                 )
-            ]
-            if override is not None:
-                cols.append(
-                    urwid.Text(
-                        [
-                            " ",
-                            ('heading', "["),
-                            ('heading_key', "m"),
-                            ('heading', (":%s]"%viewmode.name)),
-                        ],
-                        align="right"
-                    )
-                )
-            title = urwid.AttrWrap(urwid.Columns(cols), "heading")
-            txt.append(title)
-            txt.extend(body)
-        elif conn.content == flow.CONTENT_MISSING:
-            pass
-        return urwid.ListBox(txt)
+            )
+
+        title = urwid.AttrWrap(urwid.Columns(cols), "heading")
+        headers.append(title)
+        headers.extend(body)
+
+        return headers
+
+    def conn_text(self, conn):
+        """
+        Same as conn_text_raw, but returns result wrapped in a listbox ready for usage.
+        """
+        headers, msg, body = self.conn_text_raw(conn)
+        merged = self.conn_text_merge(headers, msg, body)
+        return urwid.ListBox(merged)
 
     def _tab(self, content, attr):
         p = urwid.Text(content)
@@ -205,7 +230,7 @@ class FlowView(common.WWrap):
     def wrap_body(self, active, body):
         parts = []
 
-        if self.flow.intercepting and not self.flow.request.reply.acked:
+        if self.flow.intercepted and not self.flow.reply.acked and not self.flow.response:
             qt = "Request intercepted"
         else:
             qt = "Request"
@@ -214,7 +239,7 @@ class FlowView(common.WWrap):
         else:
             parts.append(self._tab(qt, "heading_inactive"))
 
-        if self.flow.intercepting and self.flow.response and not self.flow.response.reply.acked:
+        if self.flow.intercepted and not self.flow.reply.acked and self.flow.response:
             st = "Response intercepted"
         else:
             st = "Response"
@@ -230,6 +255,211 @@ class FlowView(common.WWrap):
                 )
         return f
 
+    def search_wrapped_around(self, last_find_line, last_search_index, backwards):
+        """
+            returns true if search wrapped around the bottom.
+        """
+
+        current_find_line = self.state.get_flow_setting(self.flow,
+                "last_find_line")
+        current_search_index = self.state.get_flow_setting(self.flow,
+                "last_search_index")
+
+        if not backwards:
+            message = "search hit BOTTOM, continuing at TOP"
+            if current_find_line <= last_find_line:
+                return True, message
+            elif current_find_line == last_find_line:
+                if current_search_index <= last_search_index:
+                    return True, message
+        else:
+            message = "search hit TOP, continuing at BOTTOM"
+            if current_find_line >= last_find_line:
+                return True, message
+            elif current_find_line == last_find_line:
+                if current_search_index >= last_search_index:
+                    return True, message
+
+        return False, ""
+
+    def search_again(self, backwards=False):
+        """
+            runs the previous search again, forwards or backwards.
+        """
+        last_search_string = self.state.get_flow_setting(self.flow, "last_search_string")
+        if last_search_string:
+            message = self.search(last_search_string, backwards)
+            if message:
+                self.master.statusbar.message(message)
+        else:
+            message = "no previous searches have been made"
+            self.master.statusbar.message(message)
+
+        return message
+
+    def search(self, search_string, backwards=False):
+        """
+            similar to view_response or view_request, but instead of just
+            displaying the conn, it highlights a word that the user is
+            searching for and handles all the logic surrounding that.
+        """
+
+        if not search_string:
+            search_string = self.state.get_flow_setting(self.flow,
+                    "last_search_string")
+            if not search_string:
+                return
+
+        if self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
+            text = self.flow.request
+            const = common.VIEW_FLOW_REQUEST
+        else:
+            text = self.flow.response
+            const = common.VIEW_FLOW_RESPONSE
+            if not self.flow.response:
+                return "no response to search in"
+
+        last_find_line = self.state.get_flow_setting(self.flow,
+                "last_find_line")
+        last_search_index = self.state.get_flow_setting(self.flow,
+                "last_search_index")
+
+        # generate the body, highlight the words and get focus
+        headers, msg, body = self.conn_text_raw(text)
+        try:
+            body, focus_position = self.search_highlight_text(body, search_string, backwards=backwards)
+        except SearchError:
+            return "Search not supported in this view."
+
+        if focus_position == None:
+            # no results found.
+            return "no matches for '%s'" % search_string
+
+        # UI stuff.
+        merged = self.conn_text_merge(headers, msg, body)
+        list_box = urwid.ListBox(merged)
+        list_box.set_focus(focus_position + 2)
+        self.w = self.wrap_body(const, list_box)
+        self.master.statusbar.redraw()
+
+        self.last_displayed_body = list_box
+
+        wrapped, wrapped_message = self.search_wrapped_around(last_find_line, last_search_index, backwards)
+
+        if wrapped:
+            return wrapped_message
+
+    def search_get_start(self, search_string):
+        start_line = 0
+        start_index = 0
+        last_search_string = self.state.get_flow_setting(self.flow, "last_search_string")
+        if search_string == last_search_string:
+            start_line = self.state.get_flow_setting(self.flow, "last_find_line")
+            start_index = self.state.get_flow_setting(self.flow,
+                    "last_search_index")
+
+            if start_index == None:
+                start_index = 0
+            else:
+                start_index += len(search_string)
+
+            if start_line == None:
+                start_line = 0
+
+        else:
+            self.state.add_flow_setting(self.flow, "last_search_string",
+                    search_string)
+
+        return (start_line, start_index)
+
+    def search_get_range(self, len_text_objects, start_line, backwards):
+        if not backwards:
+            loop_range = xrange(start_line, len_text_objects)
+        else:
+            loop_range = xrange(start_line, -1, -1)
+
+        return loop_range
+
+    def search_find(self, text, search_string, start_index, backwards):
+            if backwards == False:
+                find_index = text.find(search_string, start_index)
+            else:
+                if start_index != 0:
+                    start_index -= len(search_string)
+                else:
+                    start_index = None
+
+                find_index = text.rfind(search_string, 0, start_index)
+
+            return find_index
+
+    def search_highlight_text(self, text_objects, search_string, looping = False, backwards = False):
+        start_line, start_index = self.search_get_start(search_string)
+        i = start_line
+
+        found = False
+        text_objects = copy.deepcopy(text_objects)
+        loop_range = self.search_get_range(len(text_objects), start_line, backwards)
+        for i in loop_range:
+            text_object = text_objects[i]
+
+            try:
+                text, style = text_object.get_text()
+            except AttributeError:
+                raise SearchError()
+
+            if i != start_line:
+                start_index = 0
+
+            find_index = self.search_find(text, search_string, start_index, backwards)
+
+            if find_index != -1:
+                new_text = self.search_highlight_object(text, find_index, search_string)
+                text_objects[i] = new_text
+
+                found = True
+                self.state.add_flow_setting(self.flow, "last_search_index",
+                        find_index)
+                self.state.add_flow_setting(self.flow, "last_find_line", i)
+
+                break
+
+        # handle search WRAP
+        if found:
+            focus_pos = i
+        else :
+            if looping:
+                focus_pos = None
+            else:
+                if not backwards:
+                    self.state.add_flow_setting(self.flow, "last_search_index", 0)
+                    self.state.add_flow_setting(self.flow, "last_find_line", 0)
+                else:
+                    self.state.add_flow_setting(self.flow, "last_search_index", None)
+                    self.state.add_flow_setting(self.flow, "last_find_line", len(text_objects) - 1)
+
+                text_objects, focus_pos = self.search_highlight_text(text_objects,
+                        search_string, looping=True, backwards=backwards)
+
+        return text_objects, focus_pos
+
+    def search_highlight_object(self, text_object, find_index, search_string):
+        """
+            just a little abstraction
+        """
+        before = text_object[:find_index]
+        after = text_object[find_index+len(search_string):]
+
+        new_text = urwid.Text(
+            [
+                before,
+                (self.highlight_color, search_string),
+                after,
+            ]
+        )
+
+        return new_text
+
     def view_request(self):
         self.state.view_flow_mode = common.VIEW_FLOW_REQUEST
         body = self.conn_text(self.flow.request)
@@ -295,7 +525,9 @@ class FlowView(common.WWrap):
 
     def set_url(self, url):
         request = self.flow.request
-        if not request.set_url(str(url)):
+        try:
+            request.url = str(url)
+        except ValueError:
             return "Invalid URL."
         self.master.refresh_flow(self.flow)
 
@@ -338,23 +570,27 @@ class FlowView(common.WWrap):
 
     def edit(self, part):
         if self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
-            conn = self.flow.request
+            message = self.flow.request
         else:
             if not self.flow.response:
-                self.flow.response = flow.Response(
-                    self.flow.request, 
+                self.flow.response = HTTPResponse(
                     self.flow.request.httpversion,
-                    200, "OK", flow.ODictCaseless(), "", None
+                    200, "OK", flow.ODictCaseless(), ""
                 )
                 self.flow.response.reply = controller.DummyReply()
-            conn = self.flow.response
+            message = self.flow.response
 
         self.flow.backup()
         if part == "r":
-            c = self.master.spawn_editor(conn.content or "")
-            conn.content = c.rstrip("\n") # what?
+            with decoded(message):
+                # Fix an issue caused by some editors when editing a request/response body.
+                # Many editors make it hard to save a file without a terminating newline on the last
+                # line. When editing message bodies, this can cause problems. For now, I just
+                # strip the newlines off the end of the body when we return from an editor.
+                c = self.master.spawn_editor(message.content or "")
+                message.content = c.rstrip("\n")
         elif part == "f":
-            if not conn.get_form_urlencoded() and conn.content:
+            if not message.get_form_urlencoded() and message.content:
                 self.master.prompt_onekey(
                     "Existing body is not a URL-encoded form. Clear and edit?",
                     [
@@ -362,26 +598,26 @@ class FlowView(common.WWrap):
                         ("no", "n"),
                     ],
                     self.edit_form_confirm,
-                    conn
+                    message
                 )
             else:
-                self.edit_form(conn)
+                self.edit_form(message)
         elif part == "h":
-            self.master.view_grideditor(grideditor.HeaderEditor(self.master, conn.headers.lst, self.set_headers, conn))
+            self.master.view_grideditor(grideditor.HeaderEditor(self.master, message.headers.lst, self.set_headers, message))
         elif part == "p":
-            p = conn.get_path_components()
+            p = message.get_path_components()
             p = [[i] for i in p]
-            self.master.view_grideditor(grideditor.PathEditor(self.master, p, self.set_path_components, conn))
+            self.master.view_grideditor(grideditor.PathEditor(self.master, p, self.set_path_components, message))
         elif part == "q":
-            self.master.view_grideditor(grideditor.QueryEditor(self.master, conn.get_query().lst, self.set_query, conn))
+            self.master.view_grideditor(grideditor.QueryEditor(self.master, message.get_query().lst, self.set_query, message))
         elif part == "u" and self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
-            self.master.prompt_edit("URL", conn.get_url(), self.set_url)
+            self.master.prompt_edit("URL", message.url, self.set_url)
         elif part == "m" and self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
             self.master.prompt_onekey("Method", self.method_options, self.edit_method)
         elif part == "c" and self.state.view_flow_mode == common.VIEW_FLOW_RESPONSE:
-            self.master.prompt_edit("Code", str(conn.code), self.set_resp_code)
+            self.master.prompt_edit("Code", str(message.code), self.set_resp_code)
         elif part == "m" and self.state.view_flow_mode == common.VIEW_FLOW_RESPONSE:
-            self.master.prompt_edit("Message", conn.msg, self.set_resp_msg)
+            self.master.prompt_edit("Message", message.msg, self.set_resp_msg)
         self.master.refresh_flow(self.flow)
 
     def _view_nextprev_flow(self, np, flow):
@@ -393,7 +629,7 @@ class FlowView(common.WWrap):
             new_flow, new_idx = self.state.get_next(idx)
         else:
             new_flow, new_idx = self.state.get_prev(idx)
-        if new_idx is None:
+        if new_flow is None:
             self.master.statusbar.message("No more flows!")
             return
         self.master.view_flow(new_flow)
@@ -414,7 +650,7 @@ class FlowView(common.WWrap):
 
     def delete_body(self, t):
         if t == "m":
-            val = flow.CONTENT_MISSING
+            val = CONTENT_MISSING
         else:
             val = None
         if self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
@@ -446,7 +682,7 @@ class FlowView(common.WWrap):
             # Why doesn't this just work??
             self.w.keypress(size, key)
         elif key == "a":
-            self.flow.accept_intercept()
+            self.flow.accept_intercept(self.master)
             self.master.view_flow(self.flow)
         elif key == "A":
             self.master.accept_all()
@@ -478,7 +714,6 @@ class FlowView(common.WWrap):
         elif key == "D":
             f = self.master.duplicate_flow(self.flow)
             self.master.view_flow(f)
-            self.master.currentflow = f
             self.master.statusbar.message("Duplicated.")
         elif key == "e":
             if self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
@@ -518,7 +753,7 @@ class FlowView(common.WWrap):
             self.master.statusbar.message("")
         elif key == "m":
             p = list(contentview.view_prompts)
-            p.insert(0, ("clear", "c"))
+            p.insert(0, ("Clear", "C"))
             self.master.prompt_onekey(
                 "Display mode",
                 p,
@@ -528,7 +763,6 @@ class FlowView(common.WWrap):
         elif key == "p":
             self.view_prev_flow(self.flow)
         elif key == "r":
-            self.flow.backup()
             r = self.master.replay_request(self.flow)
             if r:
                 self.master.statusbar.message(r)
@@ -577,7 +811,8 @@ class FlowView(common.WWrap):
                 self.flow.backup()
                 e = conn.headers.get_first("content-encoding", "identity")
                 if e != "identity":
-                    conn.decode()
+                    if not conn.decode():
+                        self.master.statusbar.message("Could not decode - invalid data?")
                 else:
                     self.master.prompt_onekey(
                         "Select encoding: ",
@@ -589,6 +824,16 @@ class FlowView(common.WWrap):
                         conn
                     )
                 self.master.refresh_flow(self.flow)
+        elif key == "/":
+            last_search_string = self.state.get_flow_setting(self.flow, "last_search_string")
+            search_prompt = "Search body ["+last_search_string+"]: " if last_search_string else "Search body: "
+            self.master.prompt(search_prompt,
+                    None,
+                    self.search)
+        elif key == "n":
+            self.search_again(backwards=False)
+        elif key == "N":
+            self.search_again(backwards=True)
         else:
             return key
 

libmproxy/console/grideditor.py

@@ -1,22 +1,8 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
+from __future__ import absolute_import
 import copy, re, os
 import urwid
-import common
-from .. import utils, filt
+from . import common
+from .. import utils, filt, script
 from netlib import http_uastrings
 
 
@@ -137,12 +123,13 @@ class GridWalker(urwid.ListWalker):
             except ValueError:
                 self.editor.master.statusbar.message("Invalid Python-style string encoding.", 1000)
                 return
-
         errors = self.lst[self.focus][1]
         emsg = self.editor.is_error(self.focus_col, val)
         if emsg:
             self.editor.master.statusbar.message(emsg, 1000)
             errors.add(self.focus_col)
+        else:
+            errors.discard(self.focus_col)
 
         row = list(self.lst[self.focus][0])
         row[self.focus_col] = val
@@ -334,9 +321,11 @@ class GridEditor(common.WWrap):
         elif key == "d":
             self.walker.delete_focus()
         elif key == "r":
-            self.master.path_prompt("Read file: ", "", self.read_file)
+            if self.walker.get_current_value() is not None:
+                self.master.path_prompt("Read file: ", "", self.read_file)
         elif key == "R":
-            self.master.path_prompt("Read unescaped file: ", "", self.read_file, True)
+            if self.walker.get_current_value() is not None:
+                self.master.path_prompt("Read unescaped file: ", "", self.read_file, True)
         elif key == "e":
             o = self.walker.get_current_value()
             if o is not None:
@@ -497,3 +486,25 @@ class PathEditor(GridEditor):
     columns = 1
     headings = ("Component",)
 
+
+class ScriptEditor(GridEditor):
+    title = "Editing scripts"
+    columns = 1
+    headings = ("Command",)
+    def is_error(self, col, val):
+        try:
+            script.Script.parse_command(val)
+        except script.ScriptError, v:
+            return str(v)
+
+
+class HostPatternEditor(GridEditor):
+    title = "Editing host patterns"
+    columns = 1
+    headings = ("Regex (matched on hostname:port / ip:port)",)
+
+    def is_error(self, col, val):
+        try:
+            re.compile(val, re.IGNORECASE)
+        except re.error as e:
+            return "Invalid regex: %s" % str(e)

libmproxy/console/help.py

@@ -1,20 +1,6 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
+from __future__ import absolute_import
 import urwid
-import common
+from . import common
 from .. import filt, version
 
 footer = [
@@ -38,7 +24,7 @@ class HelpView(urwid.ListBox):
 
         text.append(urwid.Text([("head", "\n\nMovement:\n")]))
         keys = [
-            ("j, k", "up, down"),
+            ("j, k", "down, up"),
             ("h, l", "left, right (in some contexts)"),
             ("space", "page down"),
             ("pg up/down", "page up/down"),
@@ -50,6 +36,7 @@ class HelpView(urwid.ListBox):
         keys = [
             ("c", "client replay"),
             ("H", "edit global header set patterns"),
+            ("I", "set ignore pattern"),
             ("i", "set interception pattern"),
             ("M", "change global default display mode"),
                 (None,
@@ -76,6 +63,10 @@ class HelpView(urwid.ListBox):
                     common.highlight_key("json", "s") +
                     [("text", ": JSON")]
                 ),
+                (None,
+                    common.highlight_key("css", "c") +
+                    [("text", ": CSS")]
+                ),
                 (None,
                     common.highlight_key("urlencoded", "u") +
                     [("text", ": URL-encoded data")]
@@ -88,6 +79,10 @@ class HelpView(urwid.ListBox):
                     common.highlight_key("xml", "x") +
                     [("text", ": XML")]
                 ),
+                (None,
+                    common.highlight_key("wbxml", "w") +
+                    [("text", ": WBXML")]
+                ),
                 (None,
                     common.highlight_key("amf", "f") +
                     [("text", ": AMF (requires PyAMF)")]
@@ -120,11 +115,11 @@ class HelpView(urwid.ListBox):
 
             ("q", "quit / return to flow list"),
             ("Q", "quit without confirm prompt"),
-            ("P", "set reverse proxy mode"),
             ("R", "edit replacement patterns"),
-            ("s", "set/unset script"),
+            ("s", "add/remove scripts"),
             ("S", "server replay"),
             ("t", "set sticky cookie expression"),
+            ("T", "set tcp proxying pattern"),
             ("u", "set sticky auth expression"),
         ]
         text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))

libmproxy/console/palettes.py

@@ -1,18 +1,3 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 palettes = {
 
 # Default palette for dark background

libmproxy/contrib/README

@@ -11,5 +11,4 @@ jsbeautifier, git checkout 25/03/12, MIT license
 
 html2text, git checkout 18/08/12, GPLv3
 
-md5crypt, PSF license, http://code.activestate.com/recipes/325204/
-
+WinDivert 1.1.4, LGPL license, http://reqrypt.org/windivert.html

libmproxy/contrib/jsbeautifier/unpackers/__init__.py

@@ -7,7 +7,7 @@
 
 import pkgutil
 import re
-from jsbeautifier.unpackers import evalbased
+from . import evalbased
 
 # NOTE: AT THE MOMENT, IT IS DEACTIVATED FOR YOUR SECURITY: it runs js!
 BLACKLIST = ['jsbeautifier.unpackers.evalbased']