Changelog for v0.12.1

Fixes #597

.coveragerc

@@ -2,5 +2,5 @@
 branch = True
 
 [report]
-omit = *contrib*, *tnetstring*, *platform*, *console*
+omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py
 include = *libmproxy*

.env

@@ -0,0 +1,5 @@
+DIR="${0%/*}"
+if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/../venv.mitmproxy/bin/activate" ]; then
+    echo "Activating mitmproxy virtualenv..."
+    source "$DIR/../venv.mitmproxy/bin/activate"
+fi

.gitattributes

@@ -0,0 +1,2 @@
+libmproxy/web/static/**/* -diff
+web/src/js/filt/filt.js -diff

.gitignore

@@ -1,14 +1,27 @@
+.DS_Store
 MANIFEST
 /build
 /dist
 /tmp
 /doc
 /venv
+/libmproxy/gui
+/release/build
 *.py[cdo]
 *.swp
 *.swo
+mitmproxy.egg-info/
 mitmproxyc
 mitmdumpc
 .coverage
+.idea
 netlib
+pathod
 libpathod
+
+# UI
+
+node_modules
+bower_components
+*.compiled.js
+*.map

.travis.yml

@@ -0,0 +1,25 @@
+language: python
+sudo: false
+python:
+  - "2.7"
+  - pypy
+# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
+install:
+  - "pip install --src . -r requirements.txt"
+# command to run tests, e.g. python setup.py test
+script: 
+  - "nosetests --with-cov --cov-report term-missing"
+after_success: 
+  - coveralls
+notifications:
+  irc:
+    channels:
+      - "irc.oftc.net#mitmproxy"
+    on_success: change
+    on_failure: always
+cache:
+  directories:
+  - /home/travis/virtualenv/python2.7.9/lib/python2.7/site-packages
+  - /home/travis/virtualenv/python2.7.9/bin
+  - /home/travis/virtualenv/pypy-2.5.0/site-packages
+  - /home/travis/virtualenv/pypy-2.5.0/bin

CHANGELOG

@@ -1,17 +1,179 @@
+3 June 2015: mitmproxy 0.12.1
 
-x January 2013: mitmproxy 0.9:
+    * mitmproxy console: mouse interaction - scroll in the flow list, click on
+      flow to view, click to switch between tabs.
+
+    * Update our crypto defaults: SHA256, 2048 bit RSA, 4096 bit DH parameters.
+
+    * BUGFIX: crash under some circumstances when copying to clipboard.
+
+    * BUGFIX: occasional crash when deleting flows.
+
+
+18 May 2015: mitmproxy 0.12
+
+    * mitmproxy console: Significant revamp of the UI. The major changes are
+      listed below, and in addition almost every aspect of the UI has
+      been tweaked, and performance has improved significantly.
+
+    * mitmproxy console: A new options screen has been created ("o" shortcut),
+      and many options that were previously manipulated directly via a
+      keybinding have been moved there.
+
+    * mitmproxy console: Big improvement in palettes. This includes improvements
+      to all colour schemes. Palettes now set the terminal background colour by
+      default, and a new --palette-transparent option has been added to disable
+      this.
+
+    * mitmproxy console: g/G shortcuts throughout mitmproxy console to jump
+      to the beginning/end of the current view.
+
+    * mitmproxy console: switch  palettes on the fly from the options screen.
+
+    * mitmproxy console: A cookie editor has been added for mitmproxy console
+      at long last.
+
+    * mitmproxy console: Various components of requests and responses can be
+      copied to the clipboard from mitmproxy - thanks to @marceloglezer.
+
+    * Support for creating new requests from scratch in mitmproxy console (@marceloglezer).
+
+    * SSLKEYLOGFILE environment variable to specify a logging location for TLS
+      master keys. This can be used with tools like Wireshark to allow TLS
+      decoding.
+
+    * Server facing SSL cipher suite specification (thanks to Jim Shaver).
+
+    * Official support for transparent proxying on FreeBSD - thanks to Mike C
+      (http://github.com/mike-pt).
+
+    * Many other small bugfixes and improvemenets throughout the project.
+
+
+29 Dec 2014: mitmproxy 0.11.2:
+
+    * Configuration files - mitmproxy.conf, mitmdump.conf, common.conf in the
+      .mitmproxy directory.
+    * Better handling of servers that reject connections that are not SNI.
+    * Many other small bugfixes and improvements.
+
+
+15 November 2014: mitmproxy 0.11.1:
+
+    * Bug fixes: connection leaks some crashes
+
+
+7 November 2014: mitmproxy 0.11:
+
+    * Performance improvements for mitmproxy console
+
+    * SOCKS5 proxy mode allows mitmproxy to act as a SOCKS5 proxy server
+
+    * Data streaming for response bodies exceeding a threshold
+      (bradpeabody@gmail.com)
+
+    * Ignore hosts or IP addresses, forwarding both HTTP and HTTPS traffic
+      untouched
+
+    * Finer-grained control of traffic replay, including options to ignore
+      contents or parameters when matching flows (marcelo.glezer@gmail.com)
+
+    * Pass arguments to inline scripts
+
+    * Configurable size limit on HTTP request and response bodies
+
+    * Per-domain specification of interception certificates and keys (see
+      --cert option)
+
+    * Certificate forwarding, relaying upstream SSL certificates verbatim (see
+      --cert-forward)
+
+    * Search and highlighting for HTTP request and response bodies in
+      mitmproxy console (pedro@worcel.com)
+
+    * Transparent proxy support on Windows
+
+    * Improved error messages and logging
+
+    * Support for FreeBSD in transparent mode, using pf (zbrdge@gmail.com)
+
+    * Content view mode for WBXML (davidshaw835@air-watch.com)
+
+    * Better documentation, with a new section on proxy modes
+
+    * Generic TCP proxy mode
+
+    * Countless bugfixes and other small improvements
+
+
+
+28 January 2014: mitmproxy 0.10:
+
+    * Support for multiple scripts and multiple script arguments
+
+    * Easy certificate install through the in-proxy web app, which is now
+      enabled by default
+
+    * Forward proxy mode, that forwards proxy requests to an upstream HTTP server
+
+    * Reverse proxy now works with SSL
+
+    * Search within a request/response using the "/" and "n" shortcut keys
+
+    * A view that beatifies CSS files if cssutils is available
+
+    * Bug fix, documentation improvements, and more.
+
+
+25 August 2013: mitmproxy 0.9.2:
+
+    * Improvements to the mitmproxywrapper.py helper script for OSX.
+
+    * Don't take minor version into account when checking for serialized file
+      compatibility.
+
+    * Fix a bug causing resource exhaustion under some circumstances for SSL
+      connections.
+
+    * Revamp the way we store interception certificates. We used to store these
+      on disk, they're now in-memory. This fixes a race condition related to
+      cert handling, and improves compatibility with Windows, where the rules
+      governing permitted file names are weird, resulting in errors for some
+      valid IDNA-encoded names.
+
+    * Display transfer rates for responses in the flow list.
+
+    * Many other small bugfixes and improvements.
+
+
+
+
+16 June 2013: mitmproxy 0.9.1:
+
+    * Use "correct" case for Content-Type headers added by mitmproxy.
+
+    * Make UTF environment detection more robust.
+
+    * Improved MIME-type detection for viewers.
+
+    * Always read files in binary mode (Windows compatibility fix).
+
+    * Some developer documentation.
+
+
+15 May 2013: mitmproxy 0.9:
 
     * Upstream certs mode is now the default.
 
     * Add a WSGI container that lets you host in-proxy web applications.
-    
+
     * Full transparent proxy support for Linux and OSX.
-    
+
     * Introduce netlib, a common codebase for mitmproxy and pathod
       (http://github.com/cortesi/netlib).
 
     * Full support for SNI.
-    
+
     * Color palettes for mitmproxy, tailored for light and dark terminal
       backgrounds.
 
@@ -22,12 +184,12 @@ x January 2013: mitmproxy 0.9:
       match asset flows (js, images, css).
 
     * Follow mode in mitmproxy ("F" shortcut) to "tail" flows as they arrive.
-    
+
     * --dummy-certs option to specify and preserve the dummy certificate
       directory.
 
     * Server replay from the current captured buffer.
-    
+
     * Huge improvements in content views. We now have viewers for AMF, HTML,
       JSON, Javascript, images, XML, URL-encoded forms, as well as hexadecimal
       and raw views.
@@ -36,7 +198,7 @@ x January 2013: mitmproxy 0.9:
       on flows, based on a matching pattern.
 
     * A graphical editor for path components in mitmproxy.
-    
+
     * A small set of standard user-agent strings, which can be used easily in
       the header editor.
 

CONTRIBUTING.md

@@ -0,0 +1,39 @@
+# Contributing
+
+Thank you for your interest in contributing to mitmproxy!
+
+# Bug Reports
+
+Bug Reports are very welcome - please file them on the GitHub [issue tracker](https://github.com/mitmproxy/mitmproxy/issues). 
+You can use the following template to structure your report:
+
+```
+##### Steps to reproduce the problem:
+1.
+2.
+3.
+
+##### What is the expected behavior?
+
+
+##### What went wrong?
+
+
+##### Any other comments?
+
+
+---
+mitmproxy version:
+Operating System:
+```
+
+# Feature Requests
+
+We're happy to hear what you'd like to see in mitmproxy. Please file feature requests on the GitHub [issue tracker](https://github.com/mitmproxy/mitmproxy/issues). 
+
+# Patches
+
+We're always happy to accept patches. Please submit them in the form of pull requests to the main [mitmproxy repository](https://github.com/mitmproxy/mitmproxy/).  
+If you're working on something cool, please do not hesistate and get in touch!
+
+Instructions for setting up a development environment can be found in the [README](README.mkd).  

CONTRIBUTORS

@@ -1,35 +1,88 @@
-   759	Aldo Cortesi
+  1067	Aldo Cortesi
+   542	Maximilian Hils
+    76	Marcelo Glezer
     18	Henrik Nordstrom
     13	Thomas Roth
+    12	Pedro Worcel
     11	Stephen Altamirano
+    11	Justus Wingert
+    11	Jim Shaver
     10	András Veres-Szentkirályi
+     9	Legend Tang
      8	Rouli
+     8	Jason A. Novak
      7	Alexis Hildebrandt
-     5	Maximilian Hils
-     4	Bryan Bishop
+     5	Matthias Urlichs
+     5	Brad Peabody
+     5	Tomaz Muraus
+     5	elitest
+     4	root
      4	Valtteri Virtanen
+     4	Wade 524
+     4	Bryan Bishop
+     4	Youhei Sakurai
+     4	Marc Liyanage
      3	Chris Neasbitt
-     2	Michael Frister
-     2	Heikki Hannikainen
-     2	Jim Lloyd
-     2	Mark E. Haase
+     3	Zack B
+     3	Kyle Manna
+     3	Eli Shvartsman
+     2	Choongwoo Han
      2	Rob Wills
-     2	alts
      2	israel
-     1	Jakub Nawalaniec
-     1	Paul
-     1	phil plante
-     1	Rory McCann
-     1	Henrik Nordström
-     1	Rune Halvorsen
-     1	Sahn Lam
-     1	Felix Wolfsteller
-     1	Eric Entzel
-     1	Ulrich Petri
-     1	Andy Smith
+     2	Mark E. Haase
+     2	Jaime Soriano Pastor
+     2	Jim Lloyd
+     2	Heikki Hannikainen
+     2	Krzysztof Bielicki
+     2	Bennett Blodinger
+     2	Michael Frister
+     2	alts
      1	Yuangxuan Wang
-     1	meeee
      1	capt8bit
+     1	davidpshaw
+     1	deployable
+     1	joebowbeer
+     1	meeee
+     1	michaeljau
+     1	peralta
+     1	phil plante
+     1	sentient07
+     1	vzvu3k6k
+     1	Andy Smith
+     1	Dan Wilbraham
+     1	David Shaw
+     1	Eric Entzel
+     1	Felix Wolfsteller
+     1	Gabriel Kirkpatrick
+     1	Henrik Nordström
+     1	Ivaylo Popov
+     1	JC
+     1	Jakub Nawalaniec
+     1	James Billingham
+     1	Jean Regisser
+     1	Kit Randel
+     1	Lucas Cimon
      1	Mathieu Mitchell
-     1	Jason A. Novak
+     1	Michael Bisbjerg
+     1	Mike C
+     1	Mikhail Korobov
+     1	Nick HS
+     1	Nick Raptis
      1	Nicolas Esteves
+     1	Oleksandr Sheremet
+     1	Paul
+     1	Rich Somerfield
+     1	Rory McCann
+     1	Rune Halvorsen
+     1	Ryo Onodera
+     1	Sahn Lam
+     1	Seppo Yli-Olli
+     1	Sergey Chipiga
+     1	Steve Phillips
+     1	Steven Van Acker
+     1	Suyash
+     1	Tarashish Mishra
+     1	Terry Long
+     1	Ulrich Petri
+     1	Vyacheslav Bakhmutov
+     1	Wade Catron

MANIFEST.in

@@ -1,10 +1,10 @@
-include LICENSE
-include CHANGELOG
-include CONTRIBUTORS
-include README.txt
+include mitmproxy mitmdump
+include LICENSE CHANGELOG CONTRIBUTORS README.txt
 exclude README.mkd
 recursive-include examples *
 recursive-include doc *
 recursive-include test *
-recursive-include libmproxy/resources *
-recursive-exclude test *.swo *.swp *.pyc
+recursive-include libmproxy *
+recursive-exclude * *.pyc *.pyo *.swo *.swp
+recursive-exclude netlib *
+recursive-exclude libpathod *

README.mkd

@@ -1,3 +1,9 @@
+[![Build Status](https://travis-ci.org/mitmproxy/mitmproxy.svg?branch=master)](https://travis-ci.org/mitmproxy/mitmproxy) [![Coverage Status](https://coveralls.io/repos/mitmproxy/mitmproxy/badge.svg?branch=master)](https://coveralls.io/r/mitmproxy/mitmproxy)
+[![Latest Version](https://pypip.in/version/mitmproxy/badge.svg?style=flat)](https://pypi.python.org/pypi/mitmproxy/)
+[![Supported Python versions](https://pypip.in/py_versions/mitmproxy/badge.svg?style=flat)](https://pypi.python.org/pypi/mitmproxy)
+[![Supported Python implementations](https://pypip.in/implementation/mitmproxy/badge.svg?style=flat)](https://pypi.python.org/pypi/mitmproxy/)
+
+
 __mitmproxy__ is an interactive, SSL-capable man-in-the-middle proxy for HTTP
 with a console interface.
 
@@ -10,6 +16,8 @@ mitmproxy.org website:
 
 [mitmproxy.org](http://mitmproxy.org).
 
+You can find complete directions for installing mitmproxy [here](http://mitmproxy.org/doc/install.html).
+
 
 Features
 --------
@@ -24,37 +32,68 @@ Features
 - SSL certificates for interception are generated on the fly.
 - And much, much more.
 
+__mitmproxy__ is tested and developed on OSX, Linux and OpenBSD. On Windows,
+only mitmdump is supported, which does not have a graphical user interface.
 
-Requirements
-------------
-
-* [Python](http://www.python.org) 2.7.x.
-* [netlib](http://pypi.python.org/pypi/netlib), version matching mitmproxy.
-* [PyOpenSSL](http://pypi.python.org/pypi/pyOpenSSL) 0.13 or newer.
-* [pyasn1](http://pypi.python.org/pypi/pyasn1) 0.1.2 or newer.
-* [urwid](http://excess.org/urwid/) version 1.1 or newer.
-* [PIL](http://www.pythonware.com/products/pil/) version 1.1 or newer.
-* [lxml](http://lxml.de/) version 2.3 or newer.
-* [flask](http://flask.pocoo.org/) version 0.9 or newer.
-
-Optional, for extended content decoding:
-
-* [PyAMF](http://www.pyamf.org/) version 0.6.1 or newer.
-* [protobuf](https://code.google.com/p/protobuf/) version 2.5.0 or newer.
-
-__mitmproxy__ is tested and developed on OSX, Linux and OpenBSD. Windows is not
-officially supported at the moment.
 
 
 Hacking
 -------
 
-The following components are needed if you plan to hack on mitmproxy:
+To get started hacking on mitmproxy, make sure you have
+[Python](http://www.python.org) 2.7.x. with
+[virtualenv](https://virtualenv.pypa.io/en/latest/) installed (you can find
+installation instructions for virtualenv
+[here](https://virtualenv.pypa.io/en/latest/installation.html)). Then do the
+following:
 
-* The test suite uses the [nose](http://readthedocs.org/docs/nose/en/latest/) unit testing
-  framework and requires [pathod](http://pathod.org) and [flask](http://flask.pocoo.org/).
-* Rendering the documentation requires [countershape](http://github.com/cortesi/countershape).
+```
+$ git clone https://github.com/mitmproxy/mitmproxy.git
+$ git clone https://github.com/mitmproxy/netlib.git
+$ git clone https://github.com/mitmproxy/pathod.git
+$ cd mitmproxy
+$ ./dev
+```
+
+The *dev* script will create a virtualenv environment in a directory called
+"venv.mitmproxy", and install all of mitmproxy's development requirements, plus
+all optional modules. The primary mitmproxy components - mitmproxy, netlib and
+pathod - are all installed "editable", so any changes to the source in the git
+checkouts will be reflected live in the virtualenv.
+
+To confirm that you're up and running, activate the virtualenv, and run the
+mitmproxy test suite:
+
+```shell
+$ source ../venv.mitmproxy/bin/activate # ..\venv.mitmproxy\Scripts\activate.bat on Windows
+$ nosetests ./test
+```
+Note that the main executables for the project - **mitmdump**, **mitmproxy** and
+**mitmweb** - are all created within the virtualenv. After activating the
+virtualenv, they will be on your $PATH, and you can run them like any other
+command:
+
+```$ mitmdump --version```
+
+For convenience, the project includes an
+[autoenv](https://github.com/kennethreitz/autoenv) file
+([.env](https://github.com/mitmproxy/mitmproxy/blob/master/.env)) that
+auto-activates the virtualenv when you cd into the mitmproxy directory.
+
+
+### Testing
+
+If you've followed the procedure above, you already have all the development
+requirements installed, and you can simply run the test suite:
+
+```nosetests ./test```
 
 Please ensure that all patches are accompanied by matching changes in the test
 suite. The project maintains 100% test coverage.
 
+
+### Docs
+
+Rendering the documentation requires [countershape](http://github.com/cortesi/countershape). After installation, you can render the documentation to the doc like this:
+
+`cshape doc-src doc`

check_coding_style.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+autopep8 -i -r -a -a .
+if [[ -n "$(git status -s)" ]]; then
+  echo "autopep8 yielded the following changes:"
+  git status -s
+  git --no-pager diff
+  exit 1
+fi
+
+autoflake -i -r --remove-all-unused-imports --remove-unused-variables .
+if [[ -n "$(git status -s)" ]]; then
+  echo "autoflake yielded the following changes:"
+  git status -s
+  git --no-pager diff
+  exit 1
+fi
+
+echo "Coding style seems to be ok."
+exit 0

dev

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+VENV=../venv.mitmproxy
+
+python -m virtualenv $VENV
+source $VENV/bin/activate
+pip install --src .. -r requirements.txt
+
+echo ""
+echo "* Created virtualenv environment in $VENV."
+echo "* Installed all dependencies into the virtualenv."
+echo "* Activated virtualenv environment."

dev.bat

@@ -0,0 +1,14 @@
+@echo off
+set VENV=..\venv.mitmproxy
+
+virtualenv %VENV%
+if %errorlevel% neq 0 exit /b %errorlevel%
+call %VENV%\Scripts\activate.bat
+if %errorlevel% neq 0 exit /b %errorlevel%
+pip install --src .. -r requirements.txt
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+echo.
+echo * Created virtualenv environment in %VENV%.
+echo * Installed all dependencies into the virtualenv.
+echo * Activated virtualenv environment.

doc-src/02-app.css

@@ -0,0 +1,43 @@
+.masthead {
+  text-align: center;
+  border-bottom: 0;
+}
+.frontpage .talks div {
+  margin-bottom: 10px;
+}
+.nav-sidebar {
+  background-color: #f0f0f0;
+  margin-bottom: 20px;
+}
+.nav-sidebar li {
+  line-height: 1.1;
+}
+.nav-sidebar li > a,
+.nav-sidebar .nav-header {
+  padding-left: 20px;
+}
+.nav-sidebar .nav-header {
+  margin-top: 1em;
+  font-size: 1.2em;
+  font-weight: bold;
+}
+.nav-sidebar .active > a,
+.nav-sidebar .active > a:hover,
+.nav-sidebar .active > a:focus {
+  color: #fff;
+  background-color: #428bca;
+}
+.tablenum {
+  font-weight: bold;
+}
+.nowrap {
+  white-space: nowrap;
+}
+.page-header {
+  margin: 0px 0 22px;
+}
+.page-header h1 {
+  margin-top: 0px;
+}
+
+/*# sourceMappingURL=02-app.css.map */

doc-src/02-docstyle.css

@@ -1,12 +0,0 @@
-body {
-    padding-top: 60px;
-    padding-bottom: 40px;
-}
-
-.tablenum {
-    font-weight: bold;
-} 
-
-.nowrap {
-     white-space: nowrap;
-}

doc-src/_layout.html

@@ -1,78 +1,44 @@
-<div class="navbar navbar-fixed-top">
-  <div class="navbar-inner">
-    <div class="container">
-      <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </a>
-      <a class="brand" href="@!urlTo(idxpath)!@">mitmproxy 0.9 docs</a>
-      </div><!--/.nav-collapse -->
-    </div>
-  </div>
-</div>
-
-<div class="container">
-  <div class="row">
-    <div class="span3">
-      <div class="well sidebar-nav">
-        <ul class="nav nav-list">
-            $!nav(idxpath, this, state)!$
-            $!nav("install.html", this, state)!$
-            $!nav("howmitmproxy.html", this, state)!$
-
-            <li class="nav-header">Tools</li>
-                $!nav("mitmproxy.html", this, state)!$
-                $!nav("mitmdump.html", this, state)!$
-
-            <li class="nav-header">Features</li>
-                $!nav("anticache.html", this, state)!$
-                $!nav("clientreplay.html", this, state)!$
-                $!nav("filters.html", this, state)!$
-                $!nav("proxyauth.html", this, state)!$
-                $!nav("replacements.html", this, state)!$
-                $!nav("serverreplay.html", this, state)!$
-                $!nav("setheaders.html", this, state)!$
-                $!nav("sticky.html", this, state)!$
-                $!nav("reverseproxy.html", this, state)!$
-                $!nav("upstreamcerts.html", this, state)!$
-
-            <li class="nav-header">Installing Certificates</li>
-                $!nav("ssl.html", this, state)!$
-                $!nav("certinstall/firefox.html", this, state)!$
-                $!nav("certinstall/osx.html", this, state)!$
-                $!nav("certinstall/windows7.html", this, state)!$
-                $!nav("certinstall/ios.html", this, state)!$
-                $!nav("certinstall/ios-simulator.html", this, state)!$
-                $!nav("certinstall/android.html", this, state)!$
-
-            <li class="nav-header">Transparent Proxying</li>
-                $!nav("transparent.html", this, state)!$
-                $!nav("transparent/linux.html", this, state)!$
-                $!nav("transparent/osx.html", this, state)!$
-
-             <li class="nav-header">Tutorials</li>
-                $!nav("tutorials/30second.html", this, state)!$
-                $!nav("tutorials/gamecenter.html", this, state)!$
-
-            <li class="nav-header">Scripting mitmproxy</li>
-                $!nav("scripting/inlinescripts.html", this, state)!$
-                $!nav("scripting/libmproxy.html", this, state)!$
-        </ul>
-      </div>
-    </div>
-
-    <div class="span9">
-        <div class="page-header">
-        <h1>@!this.title!@</h1>
+<!DOCTYPE html>
+<html>
+    <head>
+        <title>@!pageTitle!@</title>
+        $!header!$
+    </head>
+    <body>
+        <div class="navbar navbar-default navbar-static-top">
+            <div class="container">
+                <div class="navbar-header">
+                    <a class="navbar-brand" href="@!urlTo("/index.html")!@">
+                        <img height="20px" src="@!urlTo("mitmproxy-long.png")!@"/>
+                    </a>
+                </div>
+                <div class="navbar-header navbar-right">
+                    <a class="navbar-brand" hre="#">$!VERSION!$ docs</a>
+                </div>
+            </div>
         </div>
-        $!body!$
-    </div>
-  </div>
 
-  <hr>
 
-  <footer>
-    <p>@!copyright!@</p>
-  </footer>
-</div>
+        <div class="container">
+          <div class="row">
+            <div class="col-md-3">
+                $!navbar!$
+            </div>
+            <div class="col-md-9">
+                <div class="page-header">
+                <h1>@!this.title!@</h1>
+                </div>
+                $!body!$
+            </div>
+          </div>
+        </div>
+
+        <div class="container">
+          <hr>
+
+          <footer>
+            <p>@!copyright!@</p>
+          </footer>
+        </div>
+    </body>
+</html>

doc-src/_nav.html

@@ -0,0 +1,50 @@
+<ul class="nav nav-sidebar">
+    $!nav(idxpath, this, state)!$
+    $!nav("install.html", this, state)!$
+    $!nav("certinstall.html", this, state)!$
+    $!nav("howmitmproxy.html", this, state)!$
+    $!nav("modes.html", this, state)!$
+
+    <li class="nav-header">Tools</li>
+        $!nav("mitmproxy.html", this, state)!$
+        $!nav("mitmdump.html", this, state)!$
+        $!nav("config.html", this, state)!$
+
+    <li class="nav-header">Features</li>
+        $!nav("anticache.html", this, state)!$
+
+        $!nav("filters.html", this, state)!$
+        $!nav("replacements.html", this, state)!$
+        $!nav("clientreplay.html", this, state)!$
+        $!nav("serverreplay.html", this, state)!$
+        $!nav("setheaders.html", this, state)!$
+        $!nav("passthrough.html", this, state)!$
+        $!nav("proxyauth.html", this, state)!$
+        $!nav("reverseproxy.html", this, state)!$
+        $!nav("responsestreaming.html", this, state)!$
+        $!nav("socksproxy.html", this, state)!$
+        $!nav("sticky.html", this, state)!$
+        $!nav("tcpproxy.html", this, state)!$
+        $!nav("upstreamproxy.html", this, state)!$
+        $!nav("upstreamcerts.html", this, state)!$
+
+
+    <li class="nav-header">Transparent Proxying</li>
+        $!nav("transparent.html", this, state)!$
+        $!nav("transparent/linux.html", this, state)!$
+        $!nav("transparent/osx.html", this, state)!$
+
+    <li class="nav-header">Scripting</li>
+        $!nav("scripting/inlinescripts.html", this, state)!$
+        $!nav("scripting/libmproxy.html", this, state)!$
+
+    <li class="nav-header">Tutorials</li>
+        $!nav("tutorials/30second.html", this, state)!$
+        $!nav("tutorials/gamecenter.html", this, state)!$
+        $!nav("tutorials/transparent-dhcp.html", this, state)!$
+
+    <li class="nav-header">Hacking</li>
+        $!nav("dev/architecture.html", this, state)!$
+        $!nav("dev/testing.html", this, state)!$
+        $!nav("dev/sslkeylogfile.html", this, state)!$
+</ul>

doc-src/_websitelayout.html

@@ -1,83 +0,0 @@
-<div class="navbar navbar-fixed-top">
-  <div class="navbar-inner">
-    <div class="container">
-      <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </a>
-      <a class="brand" href="@!urlTo("/index.html")!@">mitmproxy</a>
-      <div class="nav">
-        <ul class="nav">
-          <li $!'class="active"' if this.match("/index.html", True) else ""!$> <a href="@!top!@/index.html">home</a> </li>
-          <li $!'class="active"' if this.under("/doc") else ""!$><a href="@!top!@/doc/index.html">docs</a></li>
-          <li $!'class="active"' if this.under("/about.html") else ""!$><a href="@!top!@/about.html">about</a></li>
-        </ul>
-      </div>
-    </div>
-  </div>
-</div>
-
-<div class="container">
-  <div class="row">
-
-    <div class="span3">
-      <div class="well sidebar-nav">
-        <ul class="nav nav-list">
-            $!nav(idxpath, this, state)!$
-            $!nav("install.html", this, state)!$
-            $!nav("howmitmproxy.html", this, state)!$
-
-            <li class="nav-header">Tools</li>
-                $!nav("mitmproxy.html", this, state)!$
-                $!nav("mitmdump.html", this, state)!$
-
-            <li class="nav-header">Features</li>
-                $!nav("anticache.html", this, state)!$
-                $!nav("clientreplay.html", this, state)!$
-                $!nav("filters.html", this, state)!$
-                $!nav("proxyauth.html", this, state)!$
-                $!nav("replacements.html", this, state)!$
-                $!nav("serverreplay.html", this, state)!$
-                $!nav("setheaders.html", this, state)!$
-                $!nav("sticky.html", this, state)!$
-                $!nav("reverseproxy.html", this, state)!$
-                $!nav("upstreamcerts.html", this, state)!$
-
-            <li class="nav-header">SSL interception</li>
-                $!nav("ssl.html", this, state)!$
-                $!nav("certinstall/firefox.html", this, state)!$
-                $!nav("certinstall/osx.html", this, state)!$
-                $!nav("certinstall/windows7.html", this, state)!$
-                $!nav("certinstall/ios.html", this, state)!$
-                $!nav("certinstall/android.html", this, state)!$
-
-            <li class="nav-header">Transparent Proxying</li>
-                $!nav("transparent.html", this, state)!$
-                $!nav("transparent/linux.html", this, state)!$
-                $!nav("transparent/osx.html", this, state)!$
-
-             <li class="nav-header">Tutorials</li>
-                $!nav("tutorials/30second.html", this, state)!$
-                $!nav("tutorials/gamecenter.html", this, state)!$
-
-            <li class="nav-header">Scripting mitmproxy</li>
-                $!nav("scripting/inlinescripts.html", this, state)!$
-                $!nav("scripting/libmproxy.html", this, state)!$
-        </ul>
-      </div>
-    </div>
-    <div class="span9">
-        <div class="page-header">
-        <h1>@!this.title!@</h1>
-        </div>
-        $!body!$
-    </div>
-  </div>
-
-  <hr>
-
-  <footer>
-    <p>@!copyright!@</p>
-  </footer>
-</div>

doc-src/certinstall.html

@@ -0,0 +1,151 @@
+## On This Page
+
+* [Introduction](#docIntro)
+* [Quick Setup](#docQuick)
+* [Installing the mitmproxy CA certificate manually](#docManual)
+* [More on mitmproxy certificates](#docMore)
+* [CA and cert files](#docCertfiles)
+* [Using a custom certificate](#docCustom)
+* [Using a client side certificate](#docClient)
+* [Using a custom certificate authority](#docCA)
+
+## <a id="docIntro"></a>Introduction
+
+Mitmproxy can decrypt encrypted traffic on the fly, as long as the client
+trusts its built-in certificate authority. Usually this means that the
+mitmproxy CA certificates have to be installed on the client device.
+
+## <a id="docQuick"></a>Quick Setup
+
+By far the easiest way to install the mitmproxy certificates is to use the
+built-in certificate installation app. To do this, just start mitmproxy and
+configure your target device with the correct proxy settings. Now start a
+browser on the device, and visit the magic domain **mitm.it**. You should see
+something like this:
+
+<img src="@!urlTo("certinstall-webapp.png")!@"></img>
+
+Click on the relevant icon, and follow the setup instructions for the platform
+you're on, and you are good to go.
+
+
+## <a id="docManual"></a>Installing the mitmproxy CA certificate manually
+
+Sometimes using the quick install app is not an option - Java or the IOS
+similator spring to mind - or you just need to do it manually for some other
+reason. Below is a list of pointers to manual certificate installation
+documentation for some common platforms:
+
+<table class="table">
+    <tr>
+        <td><a href="https://github.com/ADVTOOLS/ADVTrustStore#how-to-use-advtruststore"</a>iOS Simulator</td>
+        <td><a href="http://docs.oracle.com/cd/E19906-01/820-4916/geygn/index.html">Java</a></td>
+    </tr>
+    <tr>
+        <td><a href="http://kb.mit.edu/confluence/pages/viewpage.action?pageId=152600377">iOS</a></td>
+        <td><a href="http://wiki.cacert.org/FAQ/ImportRootCert#Android_Phones_.26_Tablets">Android/Android Simulator</a></td>
+    </tr>
+
+    <tr>
+        <td><a href="http://windows.microsoft.com/en-ca/windows/import-export-certificates-private-keys#1TC=windows-7">Windows</a></td>
+        <td><a href="https://support.apple.com/kb/PH7297?locale=en_US">Mac OS X</a></td>
+    </tr>
+    <tr>
+        <td><a href="http://askubuntu.com/questions/73287/how-do-i-install-a-root-certificate/94861#94861">Ubuntu/Debian</a></td>
+        <td><a href="https://wiki.mozilla.org/MozillaRootCertificate#Mozilla_Firefox">Firefox</a></td>
+    </tr>
+    <tr>
+        <td><a href="https://code.google.com/p/chromium/wiki/LinuxCertManagement">Chrome on Linux</a></td>
+    </tr>
+
+</table>
+
+## <a id="docMore"></a>More on mitmproxy certificates
+
+The first time __mitmproxy__ or __mitmdump__ is run, the mitmproxy Certificate
+Authority(CA) is created in the config directory (~/.mitmproxy by default).
+This CA is used for on-the-fly generation of dummy certificates for each of the
+SSL sites that your client visits. Since your browser won't trust the
+__mitmproxy__ CA out of the box , you will see an SSL certificate warning every
+time you visit a new SSL domain through __mitmproxy__. When you are testing a
+single site through a browser, just accepting the bogus SSL cert manually is
+not too much trouble, but there are a many circumstances where you will want to
+configure your testing system or browser to trust the __mitmproxy__ CA as a
+signing root authority.
+
+
+## <a id="docCertfiles"></a>CA and cert files
+
+The files created by mitmproxy in the .mitmproxy directory are as follows:
+
+<table class="table">
+    <tr>
+        <td class="nowrap">mitmproxy-ca.pem</td>
+        <td>The private key and certificate in PEM format.</td>
+    </tr>
+    <tr>
+        <td class="nowrap">mitmproxy-ca-cert.pem</td>
+        <td>The certificate in PEM format. Use this to distribute to most
+        non-Windows platforms.</td>
+    </tr>
+    <tr>
+        <td class="nowrap">mitmproxy-ca-cert.p12</td>
+        <td>The certificate in PKCS12 format. For use on Windows.</td>
+    </tr>
+    <tr>
+        <td class="nowrap">mitmproxy-ca-cert.cer</td>
+        <td>Same file as .pem, but with an extension expected by some Android
+        devices.</td>
+    </tr>
+</table>
+
+
+## <a id="docCustom"></a>Using a custom certificate
+
+You can use your own certificate by passing the <kbd>--cert</kbd> option to
+mitmproxy. mitmproxy then uses the provided certificate for interception of the
+specified domains instead of generating a certificate signed by its own CA.
+
+The certificate file is expected to be in the PEM format.  You can include
+intermediary certificates right below your leaf certificate, so that you PEM
+file roughly looks like this:
+
+<pre>
+-----BEGIN PRIVATE KEY-----
+&lt;private key&gt;
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+&lt;cert&gt;
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+&lt;intermediary cert (optional)&gt;
+-----END CERTIFICATE-----
+</pre>
+
+For example, you can generate a certificate in this format using these instructions:
+
+<pre class="terminal">
+$ openssl genrsa -out cert.key 2048
+$ openssl req -new -x509 -key cert.key -out cert.crt
+    (Specify the mitm domain as Common Name, e.g. *.google.com)
+$ cat cert.key cert.crt > cert.pem
+$ mitmproxy --cert=cert.pem
+</pre>
+
+## <a id="docClient"></a>Using a client side certificate
+
+You can use a client certificate by passing the <kbd>--client-certs
+DIRECTORY</kbd> option to mitmproxy.  If you visit example.org, mitmproxy looks
+for a file named example.org.pem in the specified directory and uses this as
+the client cert. The certificate file needs to be in the PEM format and should
+contain both the unencrypted private key and the certificate.
+
+
+## <a id="docCA"></a>Using a custom certificate authority
+
+By default, mitmproxy will use <samp>~/.mitmproxy/mitmproxy-ca.pem</samp> as
+the certificate authority to generate certificates for all domains for which no
+custom certificate is provided (see above). You can use your own certificate
+authority by passing the <kbd>--confdir</kbd> option to mitmproxy. Mitmproxy
+will then look for <samp>mitmproxy-ca.pem</samp> in the specified directory. If
+no such file exists, it will be generated automatically.

doc-src/certinstall/android.html

@@ -1,46 +0,0 @@
-
-The proxy situation on Android is [an
-embarrasment](http://code.google.com/p/android/issues/detail?id=1273).  It's
-scarcely credible, but Android didn't have a global proxy setting at all until
-quite recently, and it's still not supported on many common Android versions.
-In the meantime the app ecosystem has grown used to life without this basic
-necessity, and many apps merrily ignore it even if it's there. This situation
-is improving, but in many circumstances using [transparent
-mode](@!urlTo("transparent.html")!@) is mandatory for testing Android apps.
-
-We used an Asus Transformer Prime TF201 with Android 4.0.3 in the examples
-below - your device may differ, but the broad process should be similar. 
-
-
-## Getting the certificate onto the device
-
-First we need to get the __mitmproxy-ca-cert.cer__ file into the
-__/sdcard/Downloads__ folder on the device. There are a number of ways to do
-this. If you have the Android Developer Tools installed, you can use [__adb
-push__](http://developer.android.com/tools/help/adb.html) to accomplish this.
-Depending on your device, you could also transfer the file using external media
-like an SD Card. In this example, we're using wget from within a terminal
-emulator  to transfer the certificate from a local HTTP server: 
-
-<img src="android-shellwgetmitmproxyca.png"/>
-
-
-## Installing the certificate
-
-Once we have the certificate on the local disk, we need to import it into the
-list of trusted CAs. Go to Settings -&gt; Security -&gt; Credential Storage,
-and select "Install from storage":
-
-<img src="android-settingssecuritymenu.png"/>
-
-The certificate in /sdcard/Downloads is automatically located and offered for
-installation. Installing the cert will delete the download file from the local
-disk:
-
-<img src="android-settingssecurityinstallca.png"/>
-
-Afterwards, you should see the certificate listed in the Trusted Credentials
-store:
-
-<img src="android-settingssecurityuserinstalledca.png"/>
-

doc-src/certinstall/firefox.html

@@ -1,23 +0,0 @@
-
-How to install the __mitmproxy__ certificate authority in Firefox:
-
-<ol class="tlist">
-    <li> If needed, copy the ~/.mitmproxy/mitmproxy-ca-cert.pem file to the target. </li>
-
-    <li>Open preferences, click on "Advanced", then select"Encryption":
-        <img src="@!urlTo('firefox3.jpg')!@"/>
-    </li>
-
-    <li> Click "View Certificates", "Import", and select the certificate file: 
-        <img src="@!urlTo('firefox3-import.jpg')!@"/>
-    </li>
-
-    <li>Tick "Trust this CS to identify web sites", and click "Ok":
-        <img src="@!urlTo('firefox3-trust.jpg')!@"/>
-    </li>
-
-    <li> You should now see the mitmproxy certificate listed in the Authorities
-    tab.</li>
-
-</ol>
-

doc-src/certinstall/index.py

@@ -1,10 +0,0 @@
-from countershape import Page
-
-pages = [
-    Page("firefox.html", "Firefox"),
-    Page("osx.html", "OSX"),
-    Page("windows7.html", "Windows 7"),
-    Page("ios.html", "IOS"),
-    Page("ios-simulator.html", "IOS Simulator"),
-    Page("android.html", "Android"),
-]

doc-src/certinstall/ios-simulator.html

@@ -1,23 +0,0 @@
-
-How to install the __mitmproxy__ certificate authority in the IOS simulator:
-
-<ol class="tlist">
-
-    <li> First, check out the <a
-    href="https://github.com/ADVTOOLS/ADVTrustStore">ADVTrustStore</a> tool
-    from github.</li>
-
-    <li> Now, run the following command:
-
-    <pre class="terminal">./iosCertTrustManager.py -a ~/.mitmproxy/mitmproxy-ca-cert.pem</pre>
-
-    </li>
-
-</ol>
-
-
-Note that although the IOS simulator has its own certificate store, it shares
-the proxy settings of the host operating system. You will therefore to have
-configure your OSX host's proxy settings to use the mitmproxy instance you want
-to test with.
-

doc-src/certinstall/ios.html

@@ -1,21 +0,0 @@
-
-How to install the __mitmproxy__ certificate authority on IOS devices:
-
-<ol class="tlist">
-    <li>Set up the Mail app on the device to receive email.</li>
-
-    <li>Mail the mitmproxy-ca-cert.pem file to the device, and tap on the attachment.</li>
-
-    <li>You will be prompted to install a profile. Click "Install":
-
-    <img src="@!urlTo('ios-profile.png')!@"/></li>
-
-    <li>Accept the warning by clicking "Install" again:
-
-    <img src="@!urlTo('ios-warning.png')!@"/></li>
-
-    <li>The certificate should now be trusted:
-
-    <img src="@!urlTo('ios-installed.png')!@"/></li>
-
-</ol>

doc-src/certinstall/osx.html

@@ -1,16 +0,0 @@
-
-How to install the __mitmproxy__ certificate authority in OSX:
-
-<ol class="tlist">
-
-    <li>Open Finder, and double-click on the mitmproxy-ca-cert.pem file.</li>
-
-    <li>You will be prompted to add the certificate. Click "Always Trust": 
-
-        <img src="@!urlTo('osx-addcert-alwaystrust.png')!@"/>
-    </li>
-
-    <li> You may be prompted for your password. You should now see the
-    mitmproxy cert listed under "Certificates".</li>
-</ol>
-

doc-src/certinstall/windows7.html

@@ -1,32 +0,0 @@
-
-How to install the __mitmproxy__ certificate authority in Windows 7:
-
-<ol class="tlist">
-
-    <li> Copy the ~/.mitmproxy/mitmproxy-ca-cert.p12 file to the target system. </li>
-
-    <li> 
-        Double-click the certificate file. You should see a certificate import wizard: 
-
-        <img src="@!urlTo('win7-wizard.png')!@"/>
-    </li>
-
-    <li> 
-        Click "Next" until you're prompted for the certificate store:
-
-        <img src="@!urlTo('win7-certstore.png')!@"/>
-
-    </li>
-
-
-    <li> 
-        <p>Select "Place all certificates in the following store", and select "Trusted Root Certification Authorities":</p>
-
-        <img src="@!urlTo('win7-certstore-trustedroot.png')!@"/>
-
-    </li>
-
-    <li> Click "Next" and "Finish". </li>
-
-</ol>
-

doc-src/config.html

@@ -0,0 +1,86 @@
+
+
+Mitmproxy is configured through a set of files in the users ~/.mitmproxy
+directory. 
+
+<table class="table">
+	<tbody>
+		<tr>
+			<th>mitmproxy.conf</th> 
+			<td>Settings for the <b>mitmproxy</b>. This file can contain any options supported by mitmproxy.</td>
+		</tr>
+		<tr>
+			<th>mitmdump.conf</th> 
+			<td>Settings for the <b>mitmdump</b>. This file can contain any options supported by mitmdump.</td>
+		</tr>
+		<tr>
+			<th>common.conf</th>
+
+			<td>Settings shared between all command-line tools. Settings in
+			this file are over-ridden by those in the tool-specific
+			files. Only options shared by mitmproxy and mitmdump should be used in this file. </td>
+		</tr>
+	</tbody>
+</table>
+
+# Syntax
+
+## Comments
+
+<pre>
+# this is a comment
+; this is also a comment (.ini style)
+--- and this is a comment too (yaml style)
+</pre>
+
+## Key/Value pairs
+
+- Keys and values are case-sensitive
+- Whitespace is ignored
+- Lists are comma-delimited, and enclosed in square brackets
+
+<pre>
+name = value   # (.ini style) 
+name: value    # (yaml style)
+--name value   # (command-line option style)
+
+fruit = [apple, orange, lemon]
+indexes = [1, 12, 35 , 40]
+</pre>
+
+## Flags
+
+These are boolean options that take no value but true/false.
+
+<pre>
+name = true    # (.ini style)
+name
+--name 	 	   # (command-line option style)
+</pre>
+
+# Options
+
+The options available in the config files are precisely those available as
+command-line flags, with the key being the option's long name. To get a
+complete list of these, use the __--help__ option on each of the tools. Be
+careful to only specify common options in the __common.conf__ file -
+unsupported options in this file will be detected as an error on startup.
+
+# Examples
+
+## common.conf
+
+Note that __port__ is an option supported by all tools. 
+
+<pre class="code">
+port = 8080
+</pre>
+
+
+## mitmproxy.conf
+
+<pre class="code">
+palette = light
+</pre>
+
+

doc-src/custom-routing.txt

@@ -0,0 +1,24 @@
+# Adapted from http://tldp.org/HOWTO/TransparentProxy-6.html (6.2 Second method)
+# Note that the choice of firewall mark (3) and routing table (2) was fairly arbitrary.
+# If you are already using policy routing or firewall marking for some other purpose,
+# make sure you choose unique numbers here. Otherwise, don't worry about it.
+
+
+
+# On the router, run
+
+PROXY_IP=192.168.1.100
+TARGET_IP=192.168.1.110
+
+iptables -t mangle -A PREROUTING -j ACCEPT -p tcp -m multiport --dports 80,443 -s ! $TARGET_IP
+# Alternative to MITM the whole network:
+# iptables -t mangle -A PREROUTING -j ACCEPT -p tcp -m multiport --dports 80,443 -s  $PROXY_IP
+iptables -t mangle -A PREROUTING -j MARK --set-mark 3 -p tcp -m multiport --dports 80,443
+ip rule add fwmark 3 table 2
+ip route add default via $PROXY_IP dev br0 table 2
+
+
+
+# On the proxy machine, run
+
+iptables -A PREROUTING -t nat -i eth0 -p tcp -m multiport --dports 80,443 -j REDIRECT --to-port 8080

doc-src/dev/addingviews.html

@@ -0,0 +1,52 @@
+As discussed in [the Flow View section of the mitmproxy
+overview](@!urlTo("mitmproxy.html")!@), mitmproxy allows you to inspect and
+manipulate flows.  When inspecting a single flow, mitmproxy uses a number of
+heuristics to show a friendly view of various content types; if mitmproxy
+cannot show a friendly view, mitmproxy defaults to a __raw__ view.
+
+Each content type invokes a different flow viewer to parse the data and display
+the friendly view. Users can add custom content viewers by adding a view class
+to contentview.py, discussed below.
+
+## Adding a new View class to contentview.py
+
+The content viewers used by mitmproxy to present a friendly view of various
+content types are stored in contentview.py.  Reviewing this file shows a number
+of classes named ViewSomeDataType, each with the properties: __name__,
+__prompt__, and __content\_types__ and a function named __\_\_call\_\___.
+
+Adding a new content viewer to parse a data type is as simple as writing a new
+View class.  Your new content viewer View class should have the same properties
+as the other View classes: __name__, __prompt__, and __content\_types__ and a
+__\_\_call\_\___ function to parse the content of the request/response. 
+
+* The __name__ property should be a string describing the contents and new content viewer; 
+* The __prompt__ property should be a two item tuple:
+
+  - __1__: A string that will be used to display the new content viewer's type; and
+  - __2__: A one character string that will be the hotkey used to select the new content viewer from the Flow View screen; 
+
+* The __content\_types__ property should be a list of strings of HTTP Content\-Types that the new content viewer can parse.  
+  * Note that mitmproxy will use the content\_types to try and heuristically show a friendly view of content and that you can override the built-in views by populating content\_types with values for content\_types that are already parsed -- e.g. "image/png".
+
+After defining the __name__, __prompt__, and __content\_types__ properties of
+the class, you should write the __\_\_call\_\___ function, which will parse the
+request/response data and provide a friendly view of the data.  The
+__\_\_call\_\___ function should take the following arguments: __self__,
+__hdrs__, __content__, __limit__; __hdrs__ is a ODictCaseless object containing
+the headers of the request/response; __content__ is the content of the
+request/response, and __limit__ is an integer representing the amount of data
+to display in the view window.
+
+The __\_\_call\_\___ function returns two values: (1) a string describing the
+parsed data; and (2) the parsed data for friendly display.  The parsed data to
+be displayed should be a list of strings formatted for display.  You can use
+the __\_view\_text__ function in contentview.py to format text for display.
+Alternatively, you can display content as a series of key-value pairs; to do
+so, prepare a list of lists, where each list item is a two item list -- a key
+that describes the data, and then the data itself; after preparing the list of
+lists, use the __common.format\_keyvals__ function on it to prepare it as text
+for display.  
+
+If the new content viewer fails or throws an exception, mitmproxy will default
+to a __raw__ view.

doc-src/dev/architecture.html

@@ -0,0 +1,8 @@
+To give you a better understanding of how mitmproxy works, mitmproxy's
+high-level architecture is detailed in the following graphic:
+
+<img class="img-responsive" src="@!urlTo('schematics/architecture.png')!@">
+
+<a href="@!urlTo('schematics/architecture.pdf')!@">(architecture.pdf)</a>
+<p>Please don't refrain from asking any further
+questions on the mailing list, the IRC channel or the GitHub issue tracker.</p>

doc-src/dev/index.py

@@ -0,0 +1,8 @@
+from countershape import Page
+
+pages = [
+    Page("testing.html", "Testing"),
+    Page("architecture.html", "Architecture"),
+    Page("sslkeylogfile.html", "TLS Master Secrets"),
+    #    Page("addingviews.html", "Writing Content Views"),
+]

doc-src/dev/sslkeylogfile.html

@@ -0,0 +1,8 @@
+The SSL master keys can be logged by mitmproxy so that external programs can decrypt TLS connections both from and to the proxy.
+Key logging is enabled by setting the environment variable <samp>SSLKEYLOGFILE</samp> so that it points to a writable
+text file. Recent versions of WireShark can use these log files to decrypt packets.
+You can specify the key file path in WireShark via<br>
+<samp>Edit → Preferences → Protocols → SSL → (Pre)-Master-Secret log filename</samp>.
+
+ Note that <samp>SSLKEYLOGFILE</samp> is respected by other programs as well, e.g. Firefox and Chrome.
+If this creates any issues, you can set <samp>MITMPROXY_SSLKEYLOGFILE</samp> alternatively.

doc-src/dev/testing.html

@@ -0,0 +1,43 @@
+
+All the mitmproxy projects strive to maintain 100% code coverage. In general,
+patches and pull requests will be declined unless they're accompanied by a
+suitable extension to the test suite. 
+
+Our tests are written for the [nose](https://nose.readthedocs.org/en/latest/).
+At the point where you send your pull request, a command like this:
+
+<pre class="terminal">
+> nosetests --with-cov --cov-report term-missing ./test
+</pre>
+
+Should give output something like this:
+
+<pre class="terminal">
+> ---------- coverage: platform darwin, python 2.7.2-final-0 --
+> Name                   Stmts   Miss  Cover   Missing
+> ----------------------------------------------------
+> libmproxy/__init__         0      0   100%
+> libmproxy/app              4      0   100%
+> libmproxy/cmdline        100      0   100%
+> libmproxy/controller      69      0   100%
+> libmproxy/dump           150      0   100%
+> libmproxy/encoding        39      0   100%
+> libmproxy/filt           201      0   100%
+> libmproxy/flow           891      0   100%
+> libmproxy/proxy          427      0   100%
+> libmproxy/script          27      0   100%
+> libmproxy/utils          133      0   100%
+> libmproxy/version          4      0   100%
+> ----------------------------------------------------
+> TOTAL                   2045      0   100%
+> ----------------------------------------------------
+> Ran 251 tests in 11.864s
+</pre>
+
+
+There are exceptions to the coverage requirement - for instance, much of the
+console interface code can't sensibly be unit tested. These portions are
+excluded from coverage analysis either in the **.coveragerc** file, or using
+**#pragma no-cover** directives. To keep our coverage analysis relevant, we use
+these measures as sparingly as possible.
+

doc-src/features/index.py

@@ -4,11 +4,16 @@ pages = [
     Page("anticache.html", "Anticache"),
     Page("clientreplay.html", "Client-side replay"),
     Page("filters.html", "Filter expressions"),
+    Page("passthrough.html", "Ignore Domains"),
+    Page("proxyauth.html", "Proxy Authentication"),
+    Page("replacements.html", "Replacements"),
+    Page("responsestreaming.html", "Response Streaming"),
+    Page("reverseproxy.html", "Reverse proxy mode"),
+    Page("socksproxy.html", "SOCKS Mode"),
     Page("setheaders.html", "Set Headers"),
     Page("serverreplay.html", "Server-side replay"),
     Page("sticky.html", "Sticky cookies and auth"),
-    Page("proxyauth.html", "Proxy Authentication"),
-    Page("replacements.html", "Replacements"),
-    Page("reverseproxy.html", "Reverse proxy mode"),
+    Page("tcpproxy.html", "TCP Proxy"),
     Page("upstreamcerts.html", "Upstream Certs"),
+    Page("upstreamproxy.html", "Upstream proxy mode"),
 ]

doc-src/features/passthrough.html

@@ -0,0 +1,84 @@
+There are two main reasons why you may want to exempt some traffic from mitmproxy's interception mechanism:
+
+- **Certificate pinning:** Some traffic is is protected using
+  [certificate pinning](https://security.stackexchange.com/questions/29988/what-is-certificate-pinning) and mitmproxy's
+  interception leads to errors. For example, Windows Update or the Apple App Store fail to work if mitmproxy is active.
+- **Convenience:** You really don't care about some parts of the traffic and just want them to go away.
+
+If you want to peek into (SSL-protected) non-HTTP connections, check out the [tcp proxy](@!urlTo("tcpproxy.html")!@) feature.
+If you want to ignore traffic from mitmproxy's processing because of large response bodies, take a look at the
+[response streaming](@!urlTo("responsestreaming.html")!@) feature.
+
+## How it works
+
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>--ignore regex</td>
+        </tr>
+        <tr>
+            <th>mitmproxy shortcut</th> <td><b>I</b></td>
+        </tr>
+    </tbody>
+</table>
+
+
+mitmproxy allows you to specify a regex which is matched against a <code>host:port</code> string (e.g. "example.com:443")
+to determine hosts that should be excluded. 
+
+There are two important quirks to consider:
+
+- **In transparent mode, the ignore pattern is matched against the IP.** While we usually infer the hostname from the 
+  Host header if the --host argument is passed to mitmproxy, we do not have access to this information before the SSL
+  handshake.
+- In regular mode, explicit HTTP requests are never ignored.[^explicithttp] The ignore pattern is applied on CONNECT
+  requests, which initiate HTTPS or clear-text WebSocket connections.
+
+
+### Tutorial
+
+If you just want to ignore one specific domain, there's usually a bulletproof method to do so:
+
+1. Run mitmproxy or mitmdump in verbose mode (-v) and observe the host:port information in the serverconnect 
+messages. mitmproxy will filter on these.
+2. Take the host:port string, surround it with ^ and $, escape all dots (. becomes \\.) 
+and use this as your ignore pattern:
+
+<pre class="terminal">
+$ mitmdump -v
+127.0.0.1:50588: clientconnect
+127.0.0.1:50588: request
+  -> CONNECT example.com:443 HTTP/1.1
+127.0.0.1:50588: Set new server address: example.com:443
+127.0.0.1:50588: serverconnect
+  -> example.com:443
+^C
+$ mitmproxy --ignore ^example\.com:443$
+</pre>
+
+Here are some other examples for ignore patterns:
+<pre>
+# Exempt traffic from the iOS App Store (the regex is lax, but usually just works):
+--ignore apple.com:443  
+# "Correct" version without false-positives:
+--ignore '^(.+\.)?apple\.com:443$'
+    
+# Ignore example.com, but not its subdomains:
+--ignore '^example.com:'
+
+# Ignore everything but example.com and mitmproxy.org:
+--ignore '^(?!example\.com)(?!mitmproxy\.org)'
+    
+# Transparent mode:
+--ignore 17\.178\.96\.59:443
+# IP address range:
+--ignore 17\.178\.\d+\.\d+:443
+</pre>
+
+### See Also
+
+- [TCP Proxy](@!urlTo("tcpproxy.html")!@)
+- [Response Streaming](@!urlTo("responsestreaming.html")!@)
+
+[^explicithttp]: This stems from an limitation of explicit HTTP proxying: A single connection can be re-used for multiple target domains - a <code>GET http://example.com/</code> request may be followed by a <code>GET http://evil.com/</code> request on the same connection. If we start to ignore the connection after the first request, we would miss the relevant second one.

doc-src/features/responsestreaming.html

@@ -0,0 +1,58 @@
+By using mitmproxy's streaming feature, response contents can be passed to the client incrementally before they have been fully received by the proxy.
+This is especially useful for large binary files such as videos, where buffering the whole file slows down the client's browser.
+
+By default, mitmproxy will read the entire response, perform any indicated
+manipulations on it and then send the (possibly modified) response to
+the client. In some cases this is undesirable and you may wish to "stream"
+the reponse back to the client. When streaming is enabled, the response is
+not buffered on the proxy but directly sent back to the client instead.
+
+<h2>On the command-line</h2>
+
+Streaming can be enabled on the command line for all response bodies exceeding a certain size.  The SIZE argument understands
+k/m/g suffixes, e.g. 3m for 3 megabytes.
+
+<table class="table">
+    <tbody>
+    <tr>
+        <th width="20%">command-line</th>
+        <td>
+            --stream SIZE
+        </td>
+    </tr>
+    </tbody>
+</table>
+
+
+<h2>Caveats</h2>
+
+When response streaming is enabled, <strong>streamed response contents will not be
+    recorded or preserved in any way.</strong>
+
+When response streaming is enabled, the response body cannot be modified.
+
+<h2>Customizing Response Streaming</h2>
+
+You can also use an <a href="@!urlTo("scripting/inlinescripts.html")!@">inline script</a> to customize exactly
+which responses are streamed.
+
+Responses that should be tagged for streaming by setting their respective .stream attribute to True:
+
+$!example("examples/stream.py")!$
+
+<h2>Implementation Details</h2>
+
+When response streaming is enabled, portions of the code which would have otherwise performed changes
+on the response body will see an empty response body instead (<code>libmproxy.protocol.http.CONTENT_MISSING</code>). Any modifications will be ignored.
+
+Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a <code>Transfer-Encoding:
+    chunked</code> header, the response will be streamed one chunk at a time.
+
+<h2>Modifying streamed data</h2>
+If the <code>.stream</code> attribute is callable, .stream will work as a hook in chunk data processing.
+
+$!example("examples/stream_modify.py")!$
+
+### See Also
+
+- [Ignore Domains](@!urlTo("passthrough.html")!@)

doc-src/features/reverseproxy.html

@@ -1,17 +1,52 @@
 
-In reverse proxy mode, mitmproxy acts as a standard HTTP server and forwards
-all requests to the specified upstream server. Note that the displayed URL for
-flows in this mode will use the value of the __Host__ header field from the
-request, not the reverse proxy server.
-
+In reverse proxy mode, mitmproxy accepts standard HTTP requests and forwards
+them to the specified upstream server. This is in contrast to
+<a href="@!urlTo("upstreamproxy.html")!@">upstream proxy mode</a>, in which
+mitmproxy forwards HTTP proxy requests to an upstream proxy server.
 
 <table class="table">
     <tbody>
         <tr>
-            <th width="20%">command-line</th> <td>-P http[s]://hostname[:port]</td>
-        </tr>
-        <tr>
-            <th>mitmproxy shortcut</th> <td><b>P</b></td>
+            <th width="20%">command-line</th> <td>-R <i>schema</i>://hostname[:port]</td>
         </tr>
     </tbody>
 </table>
+
+Here, **schema** is one of http, https, http2https or https2http. The latter
+two extended schema specifications control the use of HTTP and HTTPS on
+mitmproxy and the upstream server. You can indicate that mitmproxy should use
+HTTP, and the upstream server uses HTTPS like this:
+
+	http2https://hostname:port
+
+And you can indicate that mitmproxy should use HTTPS while the upstream
+service uses HTTP like this:
+
+	https2http://hostname:port
+
+
+### Host Header
+
+In reverse proxy mode, mitmproxy does not rewrite the host header. While often useful, this
+may lead to issues with public web servers. For example, consider the following scenario:
+
+    $ python mitmdump -d -R http://example.com/ &
+    $ curl http://localhost:8080/
+
+    >> GET https://example.com/
+        Host: localhost:8080
+        User-Agent: curl/7.35.0
+        [...]
+
+    << 404 Not Found 345B
+
+Since the Host header doesn't match <samp>example.com</samp>, an error is returned.<br>
+There are two ways to solve this:
+<ol>
+    <li>Modify the hosts file of your OS so that example.com resolves to 127.0.0.1.</li>
+    <li>
+        Instruct mitmproxy to rewrite the host header by passing <kbd>&#8209;&#8209;setheader&nbsp;:~q:Host:example.com</kbd>.
+        However, keep in mind that absolute URLs within the returned document or HTTP redirects will cause the client application
+        to bypass the proxy.
+    </li>
+</ol>

doc-src/features/socksproxy.html

@@ -0,0 +1,10 @@
+
+In this mode, mitmproxy acts as a SOCKS5 proxy server.
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>--socks</td>
+        </tr>
+    </tbody>
+</table>

doc-src/features/tcpproxy.html

@@ -0,0 +1,30 @@
+WebSockets or other non-HTTP protocols are not supported by mitmproxy yet. However, you can exempt hostnames from
+processing, so that mitmproxy acts as a generic TCP forwarder. This feature is closely related to the
+[ignore domains](@!urlTo("passthrough.html")!@) functionality, but differs in two important aspects:
+
+- The raw TCP messages are printed to the event log.
+- SSL connections will be intercepted.
+
+Please note that message interception or modification are not possible yet.
+If you are not interested in the raw TCP messages, you should use the ignore domains feature.
+
+## How it works
+
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>--tcp HOST</td>
+        </tr>
+        <tr>
+            <th>mitmproxy shortcut</th> <td><b>T</b></td>
+        </tr>
+    </tbody>
+</table>
+
+For a detailed description on the structure of the hostname pattern, please refer to the [Ignore Domains](@!urlTo("passthrough.html")!@) feature.
+
+### See Also
+
+- [Ignore Domains](@!urlTo("passthrough.html")!@)
+- [Response Streaming](@!urlTo("responsestreaming.html")!@)

doc-src/features/upstreamproxy.html

@@ -0,0 +1,13 @@
+
+In this mode, mitmproxy accepts proxy requests and unconditionally forwards all
+requests to a specified upstream proxy server. This is in contrast to <a
+href="@!urlTo("reverseproxy.html")!@">reverse proxy mode</a>, in which
+mitmproxy forwards ordinary HTTP requests to an upstream server.
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>-U http://hostname[:port]</td>
+        </tr>
+    </tbody>
+</table>

doc-src/howmitmproxy.html

@@ -26,7 +26,7 @@ This is a proxy GET request - an extended form of the vanilla HTTP GET request
 that includes a schema and host specification, and it includes all the
 information mitmproxy needs to proceed.
 
-<img src="explicit.png"/>
+<img class="img-responsive" src="explicit.png"/>
 
 <table class="table">
     <tbody>
@@ -84,7 +84,7 @@ attempts to MITM an SSL connection for analysis. Our answer to this conundrum
 is to become a trusted Certificate Authority ourselves. Mitmproxy includes a
 full CA implementation that generates interception certificates on the fly. To
 get the client to trust these certificates, we [register mitmproxy as a trusted
-CA with the device manually](@!urlTo("ssl.html")!@).
+CA with the device manually](@!urlTo("certinstall.html")!@).
 
 ## Complication 1: What's the remote hostname?
 
@@ -158,7 +158,7 @@ handshake. Luckily, this is almost never an issue in practice.
 
 Lets put all of this together into the complete explicitly proxied HTTPS flow.
 
-<img src="explicit_https.png"/>
+<img class="img-responsive" src="explicit_https.png"/>
 
 <table class="table">
     <tbody>
@@ -246,11 +246,11 @@ mechanism has a different way of exposing this data, so this introduces the
 second component required for working transparent proxying: a host module that
 knows how to retrieve the original destination address from the router. In
 mitmproxy, this takes the form of a built-in set of
-[modules](https://github.com/cortesi/mitmproxy/tree/master/libmproxy/platform)
+[modules](https://github.com/mitmproxy/mitmproxy/tree/master/libmproxy/platform)
 that know how to talk to each platform's redirection mechanism.  Once we have
 this information, the process is fairly straight-forward.
 
-<img src="transparent.png"/>
+<img class="img-responsive" src="transparent.png"/>
 
 
 <table class="table">
@@ -296,7 +296,7 @@ transparently proxying HTTP, and explicitly proxying HTTPS. We use the routing
 mechanism to establish the upstream server address, and then proceed as for
 explicit HTTPS connections to establish the CN and SANs, and cope with SNI.
 
-<img src="transparent_https.png"/>
+<img class="img-responsive" src="transparent_https.png"/>
 
 
 <table class="table">

doc-src/index.html

@@ -1,4 +1,27 @@
 
-@!index_contents!@
+__mitmproxy__ is an interactive, SSL-capable man-in-the-middle proxy for HTTP
+with a console interface.
+
+__mitmdump__ is the command-line version of mitmproxy. Think tcpdump for HTTP.
+
+__libmproxy__ is the library that mitmproxy and mitmdump are built on.
+
+Documentation, tutorials and distribution packages can be found on the
+mitmproxy.org website:
+
+[mitmproxy.org](http://mitmproxy.org).
 
 
+Features
+--------
+
+- Intercept HTTP requests and responses and modify them on the fly.
+- Save complete HTTP conversations for later replay and analysis.
+- Replay the client-side of an HTTP conversations.
+- Replay HTTP responses of a previously recorded server.
+- Reverse proxy mode to forward traffic to a specified server.
+- Transparent proxy mode on OSX and Linux.
+- Make scripted changes to HTTP traffic using Python.
+- SSL certificates for interception are generated on the fly.
+- And much, much more.
+

doc-src/index.py

@@ -1,64 +1,49 @@
-import os, sys
+import os
+import sys
+import datetime
 import countershape
-from countershape import Page, Directory, PythonModule, markup, model
+from countershape import Page, Directory, markup
 import countershape.template
-sys.path.insert(0, "..")
-from libmproxy import filt
 
-MITMPROXY_SRC = "~/git/public/mitmproxy"
+MITMPROXY_SRC = os.path.abspath(
+    os.path.expanduser(os.environ.get("MITMPROXY_SRC", ".."))
+)
+sys.path.insert(0, MITMPROXY_SRC)
+from libmproxy import filt, version
+
+ns.VERSION = version.VERSION
 
 if ns.options.website:
     ns.idxpath = "doc/index.html"
-    this.layout = countershape.Layout("_websitelayout.html")
 else:
     ns.idxpath = "index.html"
-    this.layout = countershape.Layout("_layout.html")
 
+this.layout = countershape.layout.FileLayout("_layout.html")
 
 ns.title = countershape.template.Template(None, "<h1>@!this.title!@</h1>")
-this.titlePrefix = "mitmproxy 0.9 - "
+this.titlePrefix = "%s - " % version.NAMEVERSION
 this.markup = markup.Markdown(extras=["footnotes"])
 
 ns.docMaintainer = "Aldo Cortesi"
 ns.docMaintainerEmail = "aldo@corte.si"
-ns.copyright = u"\u00a9 mitmproxy project, 2013"
+ns.copyright = u"\u00a9 mitmproxy project, %s" % datetime.date.today().year
+
 
 def mpath(p):
     p = os.path.join(MITMPROXY_SRC, p)
     return os.path.expanduser(p)
 
-ns.index_contents = file(mpath("README.mkd")).read()
 
 def example(s):
     d = file(mpath(s)).read().rstrip()
     extemp = """<div class="example">%s<div class="example_legend">(%s)</div></div>"""
-    return extemp%(countershape.template.Syntax("py")(d), s)
+    return extemp % (countershape.template.Syntax("py")(d), s)
+
+
 ns.example = example
 
 
-filt_help = []
-for i in filt.filt_unary:
-    filt_help.append(
-        ("~%s"%i.code, i.help)
-    )
-for i in filt.filt_rex:
-    filt_help.append(
-        ("~%s regex"%i.code, i.help)
-    )
-for i in filt.filt_int:
-    filt_help.append(
-        ("~%s int"%i.code, i.help)
-    )
-filt_help.sort()
-filt_help.extend(
-    [
-        ("!", "unary not"),
-        ("&", "and"),
-        ("|", "or"),
-        ("(...)", "grouping"),
-    ]
-)
-ns.filt_help = filt_help
+ns.filt_help = filt.help
 
 
 def nav(page, current, state):
@@ -67,18 +52,23 @@ def nav(page, current, state):
     else:
         pre = "<li>"
     p = state.application.getPage(page)
-    return pre + '<a href="%s">%s</a></li>'%(model.UrlTo(page), p.title)
+    return pre + \
+        '<a href="%s">%s</a></li>' % (countershape.widgets.UrlTo(page), p.title)
 ns.nav = nav
+ns.navbar = countershape.template.File(None, "_nav.html")
+
 
 pages = [
     Page("index.html", "Introduction"),
     Page("install.html", "Installation"),
+    Page("certinstall.html", "About Certificates"),
+    Page("howmitmproxy.html", "How mitmproxy works"),
+    Page("modes.html", "Modes of Operation"),
+
     Page("mitmproxy.html", "mitmproxy"),
     Page("mitmdump.html", "mitmdump"),
-    Page("howmitmproxy.html", "How mitmproxy works"),
+    Page("config.html", "configuration"),
 
-    Page("ssl.html", "Overview"),
-    Directory("certinstall"),
     Directory("scripting"),
     Directory("tutorials"),
     Page("transparent.html", "Overview"),

doc-src/install.html

@@ -1,47 +1,33 @@
+## On This Page
 
-The preferred way to install mitmproxy - whether you're installing the latest
-release or from source - is to use [pip](http://www.pip-installer.org/). If you
-don't already have pip on your system, you can find installation instructions
-[here](http://www.pip-installer.org/en/latest/installing.html).
+* [Installation On Ubuntu](#docUbuntu)
+* [Installation On Mac OS X](#docOSX)
+* [Installation On Windows](#docWindows)
 
+## <a id=docUbuntu></a>Installation On Ubuntu
 
-## Installing the latest release
-
-A single command will download and install the latest release of mitmproxy,
-along with all its dependencies:
+Ubuntu comes with Python but we need to install pip, python-dev and several libraries.  This was tested on a fully patched installation of Ubuntu 14.04.
 
 <pre class="terminal">
-pip install mitmproxy
+$ sudo apt-get install python-pip python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev
+
+$ sudo pip install mitmproxy
 </pre>
 
+Once installation is complete you can run <a href="mitmproxy.html">mitmproxy</a> or <a href="mitmdump.html">mitmdump</a> from a terminal.
 
-## Installing from source
+### Installation From Source
 
-When installing from source, the easiest method is still to use pip. In this
-case run:
-    
-<pre class="terminal">
-pip install /path/to/source
-</pre>
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to get set up to contribute to the project,
+install the dependencies as you would for a regular mitmproxy installation (see previous section).
+Then see the <a href="https://github.com/mitmproxy/mitmproxy/blob/master/README.mkd#hacking">Hacking</a> section of the README on GitHub.
 
-Note that if you're installing current git master, you will also have to
-install the current git master of [netlib](http://github.com/cortesi/netlib) by
-hand.
 
-## OSX
+## <a id=docOSX></a>Installation On Mac OS X
 
-- If you're running a Python interpreter installed with homebrew (or similar),
-you may have to install some dependencies by hand. 
-- Make sure that XCode is installed from the App Store, and that the
-command-line tools have been downloaded (XCode/Preferences/Downloads).
-- Now use __pip__ to do the installation, as above.
+The easiest way to get up and running on OSX is to download the pre-built binary packages from [mitmproxy.org](http://mitmproxy.org).
 
-There are a few bits of customization you might want to do to make mitmproxy
-comfortable to use on OSX. The default color scheme is optimized for a dark
-background terminal, but you can select a palette for a light terminal
-background with the --palette option. You can use the OSX <b>open</b> program
-to create a simple and effective <b>~/.mailcap</b> file to view request and
-response bodies:
+There are a few bits of customization you might want to do to make mitmproxy comfortable to use on OSX. The default color scheme is optimized for a dark background terminal, but you can select a palette for a light terminal background with the --palette option. You can use the OSX <b>open</b> program to create a simple and effective <b>~/.mailcap</b> file to view request and response bodies:
 
 <pre class="terminal">
 application/*; /usr/bin/open -Wn %s
@@ -50,3 +36,44 @@ image/*; /usr/bin/open -Wn %s
 video/*; /usr/bin/open -Wn %s
 </pre>
 
+Once installation is complete you can run <a href="mitmproxy.html">mitmproxy</a> or <a href="mitmdump.html">mitmdump</a> from a terminal.
+
+### Installation From Source
+
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to get set up to contribute to the project, ithere are a few OS X specific things to keep in mind.
+
+- Make sure that XCode is installed from the App Store, and that the command-line tools have been downloaded (XCode/Preferences/Downloads).
+- If you're running a Python interpreter installed with homebrew (or similar), you may have to install some dependencies by hand. 
+
+Then see the <a href="https://github.com/mitmproxy/mitmproxy/blob/master/README.mkd#hacking">Hacking</a> section of the README on GitHub.
+
+## <a id=docWindows></a>Installation On Windows
+
+Please note that mitmdump is the only component of mitmproxy that is supported on Windows at the moment.
+There is no interactive user interface on Windows.
+
+
+First, install the latest version of Python 2.7 from the <a href="https://www.python.org/downloads/windows/">Python website</a>.
+If you already have an older version of Python 2.7 installed, make sure to install <a href="https://pip.pypa.io/en/latest/installing.html">pip</a>
+(pip is included in Python 2.7.9+ by default).
+
+Next, add Python and the Python Scripts directory to your <strong>PATH</strong> variable. You can do this easily by running the following in powershell:
+
+<pre class="terminal">
+[Environment]::SetEnvironmentVariable("Path", "$env:Path;C:\Python27\;C:\Python27\Scripts\", "User")
+</pre>
+
+
+Now, you can install mitmproxy by running
+
+<pre class="terminal">
+pip install mitmproxy
+</pre>
+
+Once the installation is complete, you can run <a href="mitmdump.html">mitmdump</a> from a command prompt.
+
+### Installation From Source
+
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to get set up to contribute to the project, install Python as outlined above, then see the <a href="https://github.com/mitmproxy/mitmproxy/blob/master/README.mkd#hacking">Hacking</a> section of the README on GitHub.
+
+

doc-src/mitmproxy.html

@@ -7,13 +7,13 @@ documentation from any __mitmproxy__ screen.
 
 ## Flow list
 
-The flow list shows an index of captured flows in chronological order. 
+The flow list shows an index of captured flows in chronological order.
 
-<img src="@!urlTo("screenshots/mitmproxy.png")!@"/>
+<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy.png')!@"/>
 
 - __1__: A GET request, returning a 302 Redirect response.
 - __2__: A GET request, returning 16.75kb of text/html data.
-- __3__: A replayed request. 
+- __3__: A replayed request.
 - __4__: Intercepted flows are indicated with orange text. The user may edit
 these flows, and then accept them (using the _a_ key) to continue. In this
 case, the request has been intercepted on the way to the server.
@@ -32,7 +32,7 @@ interfaces.
 
 The __Flow View__ lets you inspect and manipulate a single flow:
 
-<img src="@!urlTo("screenshots/mitmproxy-flowview.png")!@"/>
+<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy-flowview.png')!@"/>
 
 - __1__: Flow summary.
 - __2__: The Request/Response tabs, showing you which part of the flow you are
@@ -53,11 +53,11 @@ using the _m_ key.
 Much of the data that we'd like to interact with in mitmproxy is structured.
 For instance, headers, queries and form data can all be thought of as a list of
 key/value pairs. Mitmproxy has a built-in editor that lays this type of data
-out in a grid for easy manipulation. 
+out in a grid for easy manipulation.
 
 At the moment, the Grid Editor is used in four parts of mitmproxy:
 
-- Editing request or response headers (_e_ for edit, then _h_ for headers in flow view) 
+- Editing request or response headers (_e_ for edit, then _h_ for headers in flow view)
 - Editing a query string (_e_ for edit, then _q_ for query in flow view)
 - Editing a URL-encoded form (_e_ for edit, then _f_ for form in flow view)
 - Editing replacement patterns (_R_ globally)
@@ -65,13 +65,13 @@ At the moment, the Grid Editor is used in four parts of mitmproxy:
 If there is is no data, an empty editor will be started to let you add some.
 Here is the editor showing the headers from a request:
 
-<img src="@!urlTo("screenshots/mitmproxy-kveditor.png")!@"/>
+<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy-kveditor.png')!@"/>
 
 To edit, navigate to the key or value you want to modify using the arrow or vi
 navigation keys, and press enter. The background color will change to show that
 you are in edit mode for the specified field:
 
-<img src="@!urlTo("screenshots/mitmproxy-kveditor-editmode.png")!@"/>
+<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy-kveditor-editmode.png')!@"/>
 
 Modify the field as desired, then press escape to exit edit mode when you're
 done. You can also add a row (_a_ key), delete a row (_d_ key), spawn an
@@ -83,12 +83,12 @@ help (_?_ key) for more.
 
 __mitmproxy__'s interception functionality lets you pause an HTTP request or
 response, inspect and modify it, and then accept it to send it on to the server
-or client. 
+or client.
 
 
 ### 1: Set an interception pattern
 
-<img src="@!urlTo('mitmproxy-intercept-filt.png')!@"/>
+<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-filt.png')!@"/>
 
 We press _i_ to set an interception pattern. In this case, the __~q__ filter
 pattern tells __mitmproxy__ to intercept all requests. For complete filter
@@ -97,19 +97,19 @@ document, or the built-in help function in __mitmproxy__.
 
 ### 2: Intercepted connections are indicated with orange text:
 
-<img src="@!urlTo('mitmproxy-intercept-mid.png')!@"/>
+<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-mid.png')!@"/>
 
 ### 3: You can now view and modify the request:
 
-<img src="@!urlTo('mitmproxy-intercept-options.png')!@"/>
+<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-options.png')!@"/>
 
 In this case, we viewed the request by selecting it, pressed _e_ for "edit"
 and _m_ for "method" to change the HTTP request method.
 
 ### 4: Accept the intercept to continue:
 
-<img src="@!urlTo('mitmproxy-intercept-result.png')!@"/>
+<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-result.png')!@"/>
 
 Finally, we press _a_ to accept the modified request, which is then sent on to
 the server. In this case, we changed the request from an HTTP GET to
-OPTIONS, and Google's server has responded with a 405 "Method not allowed". 
+OPTIONS, and Google's server has responded with a 405 "Method not allowed".

doc-src/modes.html

@@ -0,0 +1,222 @@
+
+Mitmproxy has four modes of operation that allow you to use mitmproxy in a
+variety of scenarios:
+
+- **Regular** (the default)
+- **Transparent**
+- **Reverse Proxy**
+- **Upstream Proxy**
+
+Now, which one should you pick? Use this flow chart:
+
+<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"/>
+
+<div class="page-header">
+    <h1>Regular Proxy</h1>
+</div>
+
+Mitmproxy's regular mode is the simplest and the easiest to set up.
+
+1. Start mitmproxy.
+2. Configure your client to use mitmproxy. For instance on IOS, the settings might look like <a href="@!urlTo('screenshots/ios-manual.png')!@">this</a>.
+3. Quick Check: You should already be able to visit an unencrypted HTTP site
+through the proxy.
+4. Open the magic domain <strong>mitm.it</strong> and install the certificate for your device.
+
+<div class="well">
+<strong>Heads Up:</strong> Unfortunately, some applications bypass the
+system HTTP proxy settings - Android applications are a common example. In
+these cases, you need to use mitmproxy's transparent mode.
+</div>
+
+If you are proxying an external device, your network will probably look like this:
+
+<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-regular.png')!@">
+
+The square brackets signify the source and destination IP addresses. Your
+client explicitly connects to mitmproxy and mitmproxy explicitly connects
+to the target server.
+
+<div class="page-header">
+    <h1>Transparent Proxy</h1>
+</div>
+
+In transparent mode, traffic is directed into a proxy at the network layer,
+without any client configuration required. This makes transparent proxying
+ideal for situations where you can't change client behaviour. In the graphic
+below, a machine running mitmproxy has been inserted between the router and
+the internet:
+
+<a href="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
+</a>
+
+The square brackets signify the source and destination IP addresses. Round
+brackets mark the next hop on the *Ethernet/data link* layer. This distinction
+is important: when the packet arrives at the mitmproxy machine, it must still
+be addressed to the target server. This means that Network Address Translation
+should not be applied before the traffic reaches mitmproxy, since this would
+remove the target information, leaving mitmproxy unable to determine the real
+destination.
+
+<a href="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@">
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@"></a>
+
+<h2>Common Configurations</h2>
+
+There are many ways to configure your network for transparent proxying. We'll
+look at three common scenarios:
+
+1. Configuring the client to use a custom gateway/router/"next hop"
+2. Implementing custom routing on the router
+
+In most cases, the first option is recommended due to its ease of use.
+
+<h3>(a) Custom Gateway</h3>
+
+One simple way to get traffic to the mitmproxy machine with the destination IP
+intact, is to simply configure the client with the mitmproxy box as the
+default gateway.
+
+<a href="@!urlTo('schematics/proxy-modes-transparent-2.png')!@">
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-2.png')!@"></a>
+
+In this scenario, we would:
+
+- Configure the proxy machine for transparent mode. You can find instructions
+in the <em>Transparent Proxying</em> section of the mitmproxy docs.
+
+- Configure the client to use the proxy machine's IP as the default gateway.
+<a href="@!urlTo('screenshots/ios-gateway.png')!@">Here</a> is what this would
+look like on IOS.
+
+- Quick Check: At this point, you should already be able to visit an
+unencrypted HTTP site over the proxy.
+
+- Open the magic domain <strong>mitm.it</strong> and install the certificate
+for your device.
+
+Setting the custom gateway on clients can be automated by serving the settings
+out to clients over DHCP. This lets set up an interception network where all
+clients are proxied automatically, which can save time and effort.
+
+
+<div class="well">
+    <strong style="text-align: center; display: block">Troubleshooting Transparent Mode</strong>
+
+    <p>Incorrect transparent mode configurations are a frequent source of
+    error. If it doesn't work for you, try the following things:</p>
+
+    <ul>
+        <li>
+            Open mitmproxy's event log (press `e`) - do you see clientconnect
+            messages? If not, the packets are not arriving at the proxy. One common
+            cause is the occurrence of ICMP redirects, which means that your
+            machine is telling the client that there's a faster way to the
+            internet by contacting your router directly (see the
+            <em>Transparent Proxying</em> section on how to disable them). If in
+            doubt, <a href="https://wireshark.org/">Wireshark</a> may help you
+            to see whether something arrives at your machine or not.
+        </li>
+        <li>
+            Make sure you have not explicitly configured an HTTP proxy on the
+            client. This is not needed in transparent mode.
+        </li>
+        <li>
+            Re-check the instructions in the <em>Transparent Proxying</em> section. Anything you missed?
+        </li>
+    </ul>
+
+    If you encounter any other pitfalls that should be listed here, please let us know!
+
+</div>
+
+<h3>(b) Custom Routing</h3>
+
+In some cases, you may need more fine-grained control of which traffic reaches
+the mitmproxy instance, and which doesn't. You may, for instance, choose only
+to divert traffic to some hosts into the transparent proxy. There are a huge
+number of ways to accomplish this, and much will depend on the router or
+packet filter you're using. In most cases, the configuration will look like
+this:
+
+<a href="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
+</a>
+
+
+<div class="page-header">
+    <h1>Reverse Proxy</h1>
+</div>
+
+Mitmproxy is usually used with a client that uses the proxy to access the
+Internet. Using reverse proxy mode, you can use mitmproxy to act like a normal
+HTTP server:
+
+<a href="@!urlTo('schematics/proxy-modes-reverse.png')!@">
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-reverse.png')!@">
+</a>
+
+There are various use-cases:
+
+- Say you have an internal API running at http://example.local/. You could now
+set up mitmproxy in reverse proxy mode at http://debug.example.local/ and
+dynamically point clients to this new API endpoint, which provides clients
+with the same data and you with debug information. Similarly, you could move
+your real server to a different IP/port and set up mitmproxy at the original
+place to debug all sessions.
+
+- Say you're a web developer working on example.com (with a development
+version running on localhost:8000). You can modify your hosts file so that
+example.com points to 127.0.0.1 and then run mitmproxy in reverse proxy mode
+on port 80. You can test your app on the example.com domain and get all
+requests recorded in mitmproxy.
+
+- Say you have some toy project that should get SSL support. Simply set up
+mitmproxy with SSL termination and you're done (<code>mitmdump -p 443 -R
+https2http://localhost:80/</code>). There are better tools for this specific
+task, but mitmproxy is very quick and simple way to set up an SSL-speaking
+server.
+
+- Want to add a non-SSL-capable compression proxy in front of your server? You
+could even spawn a mitmproxy instance that terminates SSL (https2http://...),
+point it to the compression proxy and let the compression proxy point to a
+SSL-initiating mitmproxy (http2https://...), which then points to the real
+server. As you see, it's a fairly flexible thing.
+
+Note that mitmproxy supports either an HTTP or an HTTPS upstream server, not
+both at the same time. You can work around this by spawning a second mitmproxy
+instance.
+
+<div class="well">
+    <strong style="text-align: center; display: block">Caveat: Interactive Use</strong>
+
+
+One caveat is that reverse proxy mode is often not sufficient for interactive
+browsing. Consider trying to clone Google by using:
+
+<code>mitmproxy -R http://google.com/</code>
+
+This works for the initial request, but the HTML served to the client remains
+unchanged. As soon as the user clicks on an non-relative URL (or downloads a
+non-relative image resource), traffic no longer passes through mitmproxy, and
+the client connects to Google directly again.
+
+</div>
+
+
+
+<div class="page-header">
+    <h1>Upstream Proxy</h1>
+</div>
+
+If you want to chain proxies by adding mitmproxy in front of a different proxy
+appliance, you can use mitmproxy's upstream mode. In upstream mode, all
+requests are unconditionally transferred to an upstream proxy of your choice.
+
+<a href="@!urlTo('schematics/proxy-modes-upstream.png')!@">
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-upstream.png')!@"></a>
+
+mitmproxy supports both explicit HTTP and explicit HTTPS in upstream proxy
+mode. You could in theory chain multiple mitmproxy instances in a row, but
+that doesn't make any sense in practice (i.e. outside of our tests).

doc-src/scripting/index.py

@@ -2,5 +2,5 @@ from countershape import Page
 
 pages = [
     Page("inlinescripts.html", "Inline Scripts"),
-    Page("libmproxy.html", "libmproxy")
+    Page("libmproxy.html", "libmproxy"),
 ]

doc-src/scripting/inlinescripts.html

@@ -1,5 +1,5 @@
 __mitmproxy__ has a powerful scripting API that allows you to modify flows
-on-the-fly or rewrite previously saved flows locally. 
+on-the-fly or rewrite previously saved flows locally.
 
 The mitmproxy scripting API is event driven - a script is simply a Python
 module that exposes a set of event methods. Here's a complete mitmproxy script
@@ -21,42 +21,59 @@ We can now run this script using mitmdump or mitmproxy as follows:
 
 The new header will be added to all responses passing through the proxy.
 
+## Example Scripts
+
+mitmproxy comes with a variety of example inline scripts, which demonstrate
+many basic tasks. We encourage you to either browse them locally or in our
+[GitHub repo](https://github.com/mitmproxy/mitmproxy/tree/master/examples).
 
 
 ## Events
 
-### start(ScriptContext)
+### start(ScriptContext, argv)
 
 Called once on startup, before any other events.
 
 
-### clientconnect(ScriptContext, ClientConnect)
+### clientconnect(ScriptContext, ConnectionHandler)
 
 Called when a client initiates a connection to the proxy. Note that
 a connection can correspond to multiple HTTP requests.
 
+### serverconnect(ScriptContext, ConnectionHandler)
 
-### request(ScriptContext, Flow)
+Called when the proxy initiates a connection to the target server. Note that
+a connection can correspond to multiple HTTP requests.
 
-Called when a client request has been received. The __Flow__ object is
+### request(ScriptContext, HTTPFlow)
+
+Called when a client request has been received. The __HTTPFlow__ object is
 guaranteed to have a non-None __request__ attribute.
 
+### responseheaders(ScriptContext, HTTPFlow)
 
-### response(ScriptContext, Flow)
+Called when the headers of a server response have been received.
+This will always be called before the response hook.
+The __HTTPFlow__ object is guaranteed to have non-None __request__ and
+__response__ attributes.  __response.content__ will be None,
+as the response body has not been read yet.
 
-Called when a server response has been received. The __Flow__ object is
+### response(ScriptContext, HTTPFlow)
+
+Called when a server response has been received. The __HTTPFlow__ object is
 guaranteed to have non-None __request__ and __response__ attributes.
+Note that if response streaming is enabled for this response,
+__response.content__ will not contain the response body.
 
-
-### error(ScriptContext, Flow)
+### error(ScriptContext, HTTPFlow)
 
 Called when a flow error has occurred, e.g. invalid server responses, or
 interrupted connections. This is distinct from a valid server HTTP error
-response, which is simply a response with an HTTP error code. The __Flow__
+response, which is simply a response with an HTTP error code. The __HTTPFlow__
 object is guaranteed to have non-None __request__ and __error__ attributes.
 
 
-### clientdisconnect(ScriptContext, ClientDisconnect)
+### clientdisconnect(ScriptContext, ConnectionHandler)
 
 Called when a client disconnects from the proxy.
 
@@ -71,58 +88,75 @@ The main classes you will deal with in writing mitmproxy scripts are:
 
 <table class="table">
     <tr>
-        <th>libmproxy.flow.ClientConnection</th>
+        <th>libmproxy.proxy.server.ConnectionHandler</th>
+        <td>Describes a proxy client connection session. Always has a client_conn attribute, might have a server_conn
+            attribute.
+        </td>
+    </tr>
+    <tr>
+        <th>libmproxy.proxy.connection.ClientConnection</th>
         <td>Describes a client connection.</td>
     </tr>
     <tr>
-        <th>libmproxy.flow.ClientDisconnection</th>
-        <td>Describes a client disconnection.</td>
+        <th>libmproxy.proxy.connection.ServerConnection</th>
+        <td>Describes a server connection.</td>
     </tr>
     <tr>
-        <th>libmproxy.flow.Error</th>
-        <td>A communications error.</td>
-    </tr>
-    <tr>
-        <th>libmproxy.flow.Flow</th>
+        <th>libmproxy.protocol.http.HTTPFlow</th>
         <td>A collection of objects representing a single HTTP transaction.</td>
     </tr>
     <tr>
-        <th>libmproxy.flow.Headers</th>
-        <td>HTTP headers for a request or response.</td>
-    </tr>
-    <tr>
-        <th>libmproxy.flow.ODict</th>
-
-        <td>A dictionary-like object for managing sets of key/value data. There
-        is also a variant called CaselessODict that ignores key case for some
-        calls (used mainly for headers).</td>
-    </tr>
-    <tr>
-        <th>libmproxy.flow.Response</th>
+        <th>libmproxy.protocol.http.HTTPResponse</th>
         <td>An HTTP response.</td>
     </tr>
     <tr>
-        <th>libmproxy.flow.Request</th>
+        <th>libmproxy.protocol.http.HTTPRequest</th>
         <td>An HTTP request.</td>
     </tr>
     <tr>
-        <th>libmproxy.flow.ScriptContext</th>
-        <td> A handle for interacting with mitmproxy's from within scripts.  </td>
+        <th>libmproxy.protocol.primitives.Error</th>
+        <td>A communications error.</td>
     </tr>
     <tr>
-        <th>libmproxy.certutils.SSLCert</th>
+        <th>libmproxy.script.ScriptContext</th>
+        <td> A handle for interacting with mitmproxy's from within scripts.</td>
+    </tr>
+    <tr>
+        <th>netlib.odict.ODict</th>
+
+        <td>A dictionary-like object for managing sets of key/value data. There
+        is also a variant called ODictCaseless that ignores key case for some
+        calls (used mainly for headers).</td>
+    </tr>
+    <tr>
+        <th>netlib.certutils.SSLCert</th>
         <td>Exposes information SSL certificates.</td>
     </tr>
 </table>
 
-The canonical API documentation is the code. You can view the API documentation
-using pydoc (which is installed with Python by default), like this:
+The canonical API documentation is the code, which you can browse locally or in our
+[GitHub repo](https://github.com/mitmproxy/mitmproxy).
+You can view the API documentation using pydoc (which is installed with Python by default), like this:
 
 <pre class="terminal">
-> pydoc libmproxy.flow.Request
+> pydoc libmproxy.protocol.http.HTTPRequest
 </pre>
 
 
+## Running scripts in parallel
+
+We have a single flow primitive, so when a script is handling something, other requests block.
+While that's a very desirable behaviour under some circumstances, scripts can be run threaded by using the <code>libmproxy.script.concurrent</code> decorator.
+
+$!example("examples/nonblocking.py")!$
+
+## Make scripts configurable with arguments
+
+Sometimes, you want to pass runtime arguments to the inline script. This can be simply done by surrounding the script call with quotes, e.g.
+<code>mitmdump -s "script.py --foo 42"</code>. The arguments are then exposed in the start event:
+
+$!example("examples/modify_response_body.py")!$
+
 ## Running scripts on saved flows
 
 Sometimes, we want to run a script on __Flow__ objects that are already
@@ -132,6 +166,11 @@ flows from a file (see the "scripted data transformation" example on the
 one-shot script on a single flow through the _|_ (pipe) shortcut in mitmproxy.
 
 In this case, there are no client connections, and the events are run in the
-following order: __start__, __request__, __response__, __error__, __done__.  If
+following order: __start__, __request__, __responseheaders__, __response__, __error__, __done__.  If
 the flow doesn't have a __response__ or __error__ associated with it, the
-matching event will be skipped. 
+matching events will be skipped.
+
+## Spaces in the script path
+By default, spaces are interpreted as separator between the inline script and its arguments (e.g. <code>-s "foo.py
+    42"</code>). Consequently, the script path needs to be wrapped in a separate pair of quotes if it contains spaces:
+<code>-s "'./foo bar/baz.py' 42"</code>.

doc-src/scripting/libmproxy.html

@@ -1,3 +1,13 @@
+<div class="well">
+    <strong>Heads up!</strong> We strongly encourage you to use <a href="@!urlTo("scripting/inlinescripts.html")!@">inline scripts</a> rather than libmproxy
+    directly.<br><br>
+    <ul>
+        <li>Inline Scripts are equally powerful and provide an easier syntax.</li>
+        <li>Most examples are written as inline scripts.</li>
+        <li>Multiple inline scripts can be combined and used together.</li>
+        <li>Inline Scripts can either be executed headless with mitmdump or within the mitmproxy UI.</li>
+    </ul>
+</div>
 
 All of mitmproxy's basic functionality is exposed through the __libmproxy__
 library. The example below shows a simple implementation of the "sticky cookie"

doc-src/ssl.html

@@ -1,46 +0,0 @@
-
-The first time __mitmproxy__ or __mitmdump__ is run, a set of certificate files
-for the mitmproxy Certificate Authority are created in the config directory
-(~/.mitmproxy by default). The files are as follows: 
-
-<table class="table">
-    <tr>
-        <td class="nowrap">mitmproxy-ca.pem</td>
-        <td>The private key and certificate in PEM format.</td>
-    </tr>
-    <tr>
-        <td class="nowrap">mitmproxy-ca-cert.pem</td>
-        <td>The certificate in PEM format. Use this to distribute to most
-        non-Windows platforms.</td>
-    </tr>
-    <tr>
-        <td class="nowrap">mitmproxy-ca-cert.p12</td>
-        <td>The certificate in PKCS12 format. For use on Windows.</td>
-    </tr>
-    <tr>
-        <td class="nowrap">mitmproxy-ca-cert.cer</td>
-        <td>Same file as .pem, but with an extension expected by some Android
-        devices.</td>
-    </tr>
-</table>
-    
-This CA is used for on-the-fly generation of dummy certificates for SSL
-interception. Since your browser won't trust the __mitmproxy__ CA out of the
-box (and rightly so), you will see an SSL cert warning every time you visit a
-new SSL domain through __mitmproxy__. When you're testing a single site through
-a browser, just accepting the bogus SSL cert manually is not too much trouble,
-but there are a many circumstances where you will want to configure your
-testing system or browser to trust the __mitmproxy__ CA as a signing root
-authority.
-
-
-Installing the mitmproxy CA
----------------------------
-
-* [Firefox](@!urlTo("certinstall/firefox.html")!@)
-* [OSX](@!urlTo("certinstall/osx.html")!@)
-* [Windows 7](@!urlTo("certinstall/windows7.html")!@)
-* [iPhone/iPad](@!urlTo("certinstall/ios.html")!@)
-* [IOS Simulator](@!urlTo("certinstall/ios-simulator.html")!@)
-* [Android](@!urlTo("certinstall/android.html")!@)
-

doc-src/transparent/linux.html

@@ -3,7 +3,7 @@ achieve transparent mode.
 
 <ol class="tlist">
 
-    <li> <a href="@!urlTo("ssl.html")!@">Install the mitmproxy
+    <li> <a href="@!urlTo('certinstall.html')!@">Install the mitmproxy
     certificates on the test device</a>. </li>
 
     <li> Enable IP forwarding:
@@ -15,13 +15,23 @@ achieve transparent mode.
 
     </li>
 
+    <li> If your target machine is on the same physical network and you configured it to use a custom gateway,
+    disable ICMP redirects:
+
+    <pre class="terminal">echo 0 | sudo tee /proc/sys/net/ipv4/conf/*/send_redirects</pre>
+
+    You may also want to consider enabling this permanently in
+    <b>/etc/sysctl.conf</b> as demonstrated <a href="http://unix.stackexchange.com/a/58081">here</a>.
+
+    </li>
+
     <li> Create an iptables ruleset that redirects the desired traffic to the
     mitmproxy port. Details will differ according to your setup, but the
     ruleset should look something like this:
 
 <pre class="terminal">iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080
 iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080</pre>
-    
+
     </li>
 
     <li> Fire up mitmproxy. You probably want a command like this:
@@ -38,3 +48,6 @@ iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8
     running as the default gateway.</li>
 
 </ol>
+
+
+For a detailed walkthrough, have a look at the <a href="@!urlTo('tutorials/transparent-dhcp.html')!@"><i>Transparently proxify virtual machines</i></a> tutorial.

doc-src/transparent/osx.html

@@ -7,7 +7,7 @@ OSX.
 
 <ol class="tlist">
 
-    <li> <a href="@!urlTo("ssl.html")!@">Install the mitmproxy
+    <li> <a href="@!urlTo('certinstall.html')!@">Install the mitmproxy
     certificates on the test device</a>. </li>
 
     <li> Enable IP forwarding:
@@ -67,3 +67,12 @@ rdr on en2 inet proto tcp to any port 443 -&gt; 127.0.0.1 port 8080
 
 
 </ol>
+
+Note that the **rdr** rules in the pf.conf given above only apply to inbound
+traffic. This means that they will NOT redirect traffic coming from the box
+running pf itself. We can't distinguish between an outbound connection from a
+non-mitmproxy app, and an outbound connection from mitmproxy itself - if you
+want to intercept your OSX traffic, you should use an external host to run
+mitmproxy. None the less, pf is flexible to cater for a range of creative
+possibilities, like intercepting traffic emanating from VMs.  See the
+**pf.conf** man page for more.

doc-src/tutorials/gamecenter.html

@@ -2,11 +2,9 @@
 ## The setup
 
 In this tutorial, I'm going to show you how simple it is to creatively
-interfere with Apple Game Center traffic using mitmproxy. To set things up, I
-registered my mitmproxy CA certificate with my iPhone - there's a [step by step
-set of instructions](@!urlTo("certinstall/ios.html")!@) elsewhere in this manual. I then
-started mitmproxy on my desktop, and configured the iPhone to use it as a
-proxy. 
+interfere with Apple Game Center traffic using mitmproxy. To set things up,
+install the [mitmproxy root certificate](@!urlTo("certinstall.html")!@). Then
+start mitmproxy on your desktop, and confige the iPhone to use it as a proxy.
 
 
 ## Taking a look at the Game Center traffic
@@ -14,17 +12,17 @@ proxy.
 Lets take a first look at the Game Center traffic. The game I'll use in this
 tutorial is [Super Mega
 Worm](http://itunes.apple.com/us/app/super-mega-worm/id388541990?mt=8) - a
-great little retro-apocalyptic sidescroller for the iPhone: 
+great little retro-apocalyptic sidescroller for the iPhone:
 
 <center>
-    <img src="@!urlTo("tutorials/supermega.png")!@"/>
+    <img class="img-responsive" src="@!urlTo('tutorials/supermega.png')!@"/>
 </center>
 
 After finishing a game (take your time), watch the traffic flowing through
 mitmproxy:
 
 <center>
-    <img src="@!urlTo("tutorials/one.png")!@"/>
+    <img class="img-responsive" src="@!urlTo('tutorials/one.png')!@"/>
 </center>
 
 We see a bunch of things we might expect - initialisation, the retrieval of
@@ -67,8 +65,8 @@ timestamp. Looks pretty simple to mess with.
 Lets edit the score submission. First, select it in mitmproxy, then press
 __enter__ to view it. Make sure you're viewing the request, not the response -
 you can use __tab__ to flick between the two. Now press __e__ for edit. You'll
-be prompted for the part of the request you want to change - press __b__ for
-body.  Your preferred editor (taken from the EDITOR environment variable) will
+be prompted for the part of the request you want to change - press __r__ for
+raw body.  Your preferred editor (taken from the EDITOR environment variable) will
 now fire up. Lets bump the score up to something a bit more ambitious:
 
 <!--(block|syntax("xml"))-->
@@ -91,7 +89,7 @@ now fire up. Lets bump the score up to something a bit more ambitious:
 </plist>
 <!--(end)-->
 
-Save the file and exit your editor. 
+Save the file and exit your editor.
 
 The final step is to replay this modified request. Simply press __r__ for
 replay.
@@ -99,11 +97,11 @@ replay.
 ## The glorious result and some intrigue
 
 <center>
-    <img src="@!urlTo("tutorials/leaderboard.png")!@"/>
+    <img class="img-responsive" src="@!urlTo('tutorials/leaderboard.png')!@"/>
 </center>
 
 And that's it - according to the records, I am the greatest Super Mega Worm
-player of all time. 
+player of all time.
 
 There's a curious addendum to this tale. When I first wrote this tutorial, all
 the top competitors' scores were the same: 2,147,483,647 (this is no longer the
@@ -119,4 +117,3 @@ phone, then 2^31-1 might well be the maximum score you could get. Then again,
 if the game itself stores its score in a signed 32-bit int, you could get the
 same score through perfect play, effectively beating the game. So, which is it
 in this case? I'll leave that for you to decide.
-

doc-src/tutorials/index.py

@@ -3,4 +3,5 @@ from countershape import Page
 pages = [
     Page("30second.html", "Client playback: a 30 second example"),
     Page("gamecenter.html", "Setting highscores on Apple's GameCenter"),
+    Page("transparent-dhcp.html", "Transparently proxify virtual machines")
 ]

doc-src/tutorials/transparent-dhcp.html

@@ -0,0 +1,53 @@
+This walkthrough illustrates how to set up transparent proxying with mitmproxy. We use VirtualBox VMs with an Ubuntu proxy machine in this example, but the general principle can be applied to other setups.
+
+1.  **Configure VirtualBox Network Adapters for the proxy machine**
+    The network setup is simple:  `internet <--> proxy vm <--> (virtual) internal network`.
+    For the proxy machine, *eth0* represents the outgoing network. *eth1* is connected to the internal network that will be proxified, using a static ip (192.168.3.1).
+    <hr>VirtualBox configuration:
+    <img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth0.png')!@"/><br><br>
+    <img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth1.png')!@"/>
+    <br>Proxy VM:
+    <img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step1_proxy.png')!@"/>
+    <hr>
+2.  **Configure DHCP and DNS**
+    We use dnsmasq to provide DHCP and DNS in our internal network.
+    Dnsmasq is a lightweight server designed to provide DNS (and optionally DHCP and TFTP) services to a small-scale
+    network.
+
+    -   Before we get to that, we need to fix some Ubuntu quirks:
+        **Ubuntu >12.04** runs an internal dnsmasq instance (listening on loopback only) by default
+        <a href="https://www.stgraber.org/2012/02/24/dns-in-ubuntu-12-04/">[1]</a>. For our use case, this needs to be
+        disabled by changing <br>`dns=dnsmasq` to `#dns=dnsmasq` in */etc/NetworkManager/NetworkManager.conf*
+        and running `sudo restart network-manager` afterwards.
+    -   Now, dnsmasq can be be installed and configured:
+        `sudo apt-get install dnsmasq`
+        Replace */etc/dnsmasq.conf* with the following configuration:
+        <pre>\# Listen for DNS requests on the internal network
+        interface=eth1
+        \# Act as a DHCP server, assign IP addresses to clients
+        dhcp-range=192.168.3.10,192.168.3.100,96h
+        \# Broadcast gateway and dns server information
+        dhcp-option=option:router,192.168.3.1
+        dhcp-option=option:dns-server,192.168.3.1
+        </pre>
+        Apply changes:
+        `sudo service dnsmasq restart`
+        <hr>
+        Your proxied machine's network settings should now look similar to this:
+        <img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step2_proxied_vm.png')!@"/>
+        <hr>
+
+3.  **Set up traffic redirection to mitmproxy**
+    To redirect traffic to mitmproxy, we need to add two iptables rules:
+    <pre class="terminal">
+    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 \
+        -j REDIRECT --to-port 8080
+    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 \
+        -j REDIRECT --to-port 8080
+    </pre>
+
+4.  If required, <a href="@!urlTo('certinstall.html')!@">install the mitmproxy
+    certificates on the test device</a>.
+
+5.  Finally, we can run <code>mitmproxy -T</code>.
+    The proxied machine cannot to leak any data outside of HTTP or DNS requests.

examples/README

@@ -1,9 +1,26 @@
-add_header.py           Simple script that just adds a header to every request.
-dup_and_replay.py       Duplicates each request, changes it, and then replays the modified request.
-flowbasic               Basic use of mitmproxy as a library.
-modify_form.py          Modify all form submissions to add a parameter.
-modify_querystring.py   Modify all query strings to add a parameters.
-proxapp                 How to embed a WSGI app in a mitmproxy server
-stub.py                 Script stub with a method definition for every event.
-stickycookies           An example of writing a custom proxy with libmproxy.
-upsidedownternet.py     Rewrites traffic to turn PNGs upside down.
+# inline script examples
+add_header.py             Simple script that just adds a header to every request.
+change_upstream_proxy.py  Dynamically change the upstream proxy
+dns_spoofing.py           Use mitmproxy in a DNS spoofing scenario.
+dup_and_replay.py         Duplicates each request, changes it, and then replays the modified request.
+filt.py                   Use mitmproxy's filter expressions in your script.
+flowwriter.py             Only write selected flows into a mitmproxy dumpfile.
+iframe_injector.py        Inject configurable iframe into pages.
+modify_form.py            Modify all form submissions to add a parameter.
+modify_querystring.py     Modify all query strings to add a parameters.
+modify_response_body.py   Replace arbitrary strings in all responses
+nonblocking.py            Demonstrate parallel processing with a blocking script.
+proxapp.py                How to embed a WSGI app in a mitmproxy server
+redirect_requests.py      Redirect requests or directly reply to them.
+stub.py                   Script stub with a method definition for every event.
+upsidedownternet.py       Rewrites traffic to turn images upside down.
+
+
+# libmproxy examples
+flowbasic                 Basic use of mitmproxy as a library.
+stickycookies             An example of writing a custom proxy with libmproxy.
+
+
+# misc
+read_dumpfile             Read a dumpfile generated by mitmproxy.
+mitmproxywrapper.py       Bracket mitmproxy run with proxy enable/disable on OS X

examples/change_upstream_proxy.py

@@ -0,0 +1,29 @@
+# This scripts demonstrates how mitmproxy can switch to a second/different upstream proxy
+# in upstream proxy mode.
+#
+# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s
+# "change_upstream_proxy.py host"
+from libmproxy.protocol.http import send_connect_request
+
+alternative_upstream_proxy = ("localhost", 8082)
+
+
+def should_redirect(flow):
+    return flow.request.host == "example.com"
+
+
+def request(context, flow):
+    if flow.live and should_redirect(flow):
+
+        # If you want to change the target server, you should modify flow.request.host and flow.request.port
+        # flow.live.change_server should only be used by inline scripts to change the upstream proxy,
+        # unless you are sure that you know what you are doing.
+        server_changed = flow.live.change_server(
+            alternative_upstream_proxy,
+            persistent_change=True)
+        if flow.request.scheme == "https" and server_changed:
+            send_connect_request(
+                flow.live.c.server_conn,
+                flow.request.host,
+                flow.request.port)
+            flow.live.c.establish_ssl(server=True)

examples/dns_spoofing.py

@@ -0,0 +1,37 @@
+"""
+This inline scripts makes it possible to use mitmproxy in scenarios where IP spoofing has been used to redirect
+connections to mitmproxy. The way this works is that we rely on either the TLS Server Name Indication (SNI) or the
+Host header of the HTTP request.
+Of course, this is not foolproof - if an HTTPS connection comes without SNI, we don't
+know the actual target and cannot construct a certificate that looks valid.
+Similarly, if there's no Host header or a spoofed Host header, we're out of luck as well.
+Using transparent mode is the better option most of the time.
+
+Usage:
+    mitmproxy
+        -p 80
+        -R http://example.com/  // Used as the target location if no Host header is present
+    mitmproxy
+        -p 443
+        -R https://example.com/ // Used as the target locaction if neither SNI nor host header are present.
+
+mitmproxy will always connect to the default location first, so it must be reachable.
+As a workaround, you can spawn an arbitrary HTTP server and use that for both endpoints, e.g.
+mitmproxy -p  80 -R       http://localhost:8000
+mitmproxy -p 443 -R https2http://localhost:8000
+"""
+
+
+def request(context, flow):
+    if flow.client_conn.ssl_established:
+        # TLS SNI or Host header
+        flow.request.host = flow.client_conn.connection.get_servername(
+        ) or flow.request.pretty_host(hostheader=True)
+
+        # If you use a https2http location as default destination, these
+        # attributes need to be corrected as well:
+        flow.request.port = 443
+        flow.request.scheme = "https"
+    else:
+        # Host header
+        flow.request.host = flow.request.pretty_host(hostheader=True)

examples/dup_and_replay.py

@@ -1,4 +1,4 @@
-def request(ctx, flow):
-   f = ctx.duplicate_flow(flow)
-   f.request.path = "/changed"
-   ctx.replay_request(f)
+def request(context, flow):
+    f = context.duplicate_flow(flow)
+    f.request.path = "/changed"
+    context.replay_request(f)

examples/filt.py

@@ -0,0 +1,16 @@
+# This scripts demonstrates how to use mitmproxy's filter pattern in inline scripts.
+# Usage: mitmdump -s "filt.py FILTER"
+
+from libmproxy import filt
+
+
+def start(context, argv):
+    if len(argv) != 2:
+        raise ValueError("Usage: -s 'filt.py FILTER'")
+    context.filter = filt.parse(argv[1])
+
+
+def response(context, flow):
+    if flow.match(context.filter):
+        print("Flow matches filter:")
+        print(flow)

examples/flowbasic

@@ -3,11 +3,15 @@
     This example shows how to build a proxy based on mitmproxy's Flow
     primitives.
 
+    Heads Up: In the majority of cases, you want to use inline scripts.
+
     Note that request and response messages are not automatically replied to,
     so we need to implement handlers to do this.
 """
 import os
-from libmproxy import proxy, flow
+from libmproxy import flow, proxy
+from libmproxy.proxy.server import ProxyServer
+
 
 class MyMaster(flow.FlowMaster):
     def run(self):
@@ -16,24 +20,26 @@ class MyMaster(flow.FlowMaster):
         except KeyboardInterrupt:
             self.shutdown()
 
-    def handle_request(self, r):
-        f = flow.FlowMaster.handle_request(self, r)
+    def handle_request(self, f):
+        f = flow.FlowMaster.handle_request(self, f)
         if f:
-            r.reply()
+            f.reply()
         return f
 
-    def handle_response(self, r):
-        f = flow.FlowMaster.handle_response(self, r)
+    def handle_response(self, f):
+        f = flow.FlowMaster.handle_response(self, f)
         if f:
-            r.reply()
-        print f
+            f.reply()
+        print(f)
         return f
 
 
 config = proxy.ProxyConfig(
-    cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
+    port=8080,
+    # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file.
+    cadir="~/.mitmproxy/"
 )
 state = flow.State()
-server = proxy.ProxyServer(config, 8080)
+server = ProxyServer(config)
 m = MyMaster(server, state)
 m.run()

examples/flowwriter.py

@@ -0,0 +1,20 @@
+import random
+import sys
+
+from libmproxy.flow import FlowWriter
+
+
+def start(context, argv):
+    if len(argv) != 2:
+        raise ValueError('Usage: -s "flowriter.py filename"')
+
+    if argv[1] == "-":
+        f = sys.stdout
+    else:
+        f = open(argv[1], "wb")
+    context.flow_writer = FlowWriter(f)
+
+
+def response(context, flow):
+    if random.choice([True, False]):
+        context.flow_writer.add(flow)

examples/har_extractor.py

@@ -0,0 +1,254 @@
+"""
+
+    This inline script utilizes harparser.HAR from
+    https://github.com/JustusW/harparser to generate a HAR log object.
+"""
+from harparser import HAR
+
+from datetime import datetime
+
+
+class _HARLog(HAR.log):
+    # The attributes need to be registered here for them to actually be
+    # available later via self. This is due to HAREncodable linking __getattr__
+    # to __getitem__. Anything that is set only in __init__ will just be added
+    # as key/value pair to self.__classes__.
+    __page_list__ = []
+    __page_count__ = 0
+    __page_ref__ = {}
+
+    def __init__(self, page_list):
+        self.__page_list__ = page_list
+        self.__page_count__ = 0
+        self.__page_ref__ = {}
+
+        HAR.log.__init__(self, {"version": "1.2",
+                                "creator": {"name": "MITMPROXY HARExtractor",
+                                            "version": "0.1",
+                                            "comment": ""},
+                                "pages": [],
+                                "entries": []})
+
+    def reset(self):
+        self.__init__(self.__page_list__)
+
+    def add(self, obj):
+        if isinstance(obj, HAR.pages):
+            self['pages'].append(obj)
+        if isinstance(obj, HAR.entries):
+            self['entries'].append(obj)
+
+    def create_page_id(self):
+        self.__page_count__ += 1
+        return "autopage_%s" % str(self.__page_count__)
+
+    def set_page_ref(self, page, ref):
+        self.__page_ref__[page] = ref
+
+    def get_page_ref(self, page):
+        return self.__page_ref__.get(page, None)
+
+    def get_page_list(self):
+        return self.__page_list__
+
+
+def start(context, argv):
+    """
+        On start we create a HARLog instance. You will have to adapt this to
+        suit your actual needs of HAR generation. As it will probably be
+        necessary to cluster logs by IPs or reset them from time to time.
+    """
+    context.dump_file = None
+    if len(argv) > 1:
+        context.dump_file = argv[1]
+    else:
+        raise ValueError(
+            'Usage: -s "har_extractor.py filename" '
+            '(- will output to stdout, filenames ending with .zhar '
+            'will result in compressed har)'
+        )
+    context.HARLog = _HARLog(['https://github.com'])
+    context.seen_server = set()
+
+
+def response(context, flow):
+    """
+       Called when a server response has been received. At the time of this
+       message both a request and a response are present and completely done.
+    """
+    # Values are converted from float seconds to int milliseconds later.
+    ssl_time = -.001
+    connect_time = -.001
+    if flow.server_conn not in context.seen_server:
+        # Calculate the connect_time for this server_conn. Afterwards add it to
+        # seen list, in order to avoid the connect_time being present in entries
+        # that use an existing connection.
+        connect_time = flow.server_conn.timestamp_tcp_setup - \
+            flow.server_conn.timestamp_start
+        context.seen_server.add(flow.server_conn)
+
+        if flow.server_conn.timestamp_ssl_setup is not None:
+            # Get the ssl_time for this server_conn as the difference between
+            # the start of the successful tcp setup and the successful ssl
+            # setup. If  no ssl setup has been made it is left as -1 since it
+            # doesn't apply to this connection.
+            ssl_time = flow.server_conn.timestamp_ssl_setup - \
+                flow.server_conn.timestamp_tcp_setup
+
+    # Calculate the raw timings from the different timestamps present in the
+    # request and response object. For lack of a way to measure it dns timings
+    # can not be calculated. The same goes for HAR blocked: MITMProxy will open
+    # a server connection as soon as it receives the host and port from the
+    # client connection. So the time spent waiting is actually spent waiting
+    # between request.timestamp_end and response.timestamp_start thus it
+    # correlates to HAR wait instead.
+    timings_raw = {
+        'send': flow.request.timestamp_end - flow.request.timestamp_start,
+        'wait': flow.response.timestamp_start - flow.request.timestamp_end,
+        'receive': flow.response.timestamp_end - flow.response.timestamp_start,
+        'connect': connect_time,
+        'ssl': ssl_time
+    }
+
+    # HAR timings are integers in ms, so we have to re-encode the raw timings to
+    # that format.
+    timings = dict([(key, int(1000 * value))
+                    for key, value in timings_raw.iteritems()])
+
+    # The full_time is the sum of all timings. Timings set to -1 will be ignored
+    # as per spec.
+    full_time = 0
+    for item in timings.values():
+        if item > -1:
+            full_time += item
+
+    started_date_time = datetime.fromtimestamp(
+        flow.request.timestamp_start,
+        tz=utc).isoformat()
+
+    request_query_string = [{"name": k, "value": v}
+                            for k, v in flow.request.get_query()]
+    request_http_version = ".".join([str(v) for v in flow.request.httpversion])
+    # Cookies are shaped as tuples by MITMProxy.
+    request_cookies = [{"name": k.strip(), "value": v[0]}
+                       for k, v in (flow.request.get_cookies() or {}).iteritems()]
+    request_headers = [{"name": k, "value": v} for k, v in flow.request.headers]
+    request_headers_size = len(str(flow.request.headers))
+    request_body_size = len(flow.request.content)
+
+    response_http_version = ".".join(
+        [str(v) for v in flow.response.httpversion])
+    # Cookies are shaped as tuples by MITMProxy.
+    response_cookies = [{"name": k.strip(), "value": v[0]}
+                        for k, v in (flow.response.get_cookies() or {}).iteritems()]
+    response_headers = [{"name": k, "value": v}
+                        for k, v in flow.response.headers]
+    response_headers_size = len(str(flow.response.headers))
+    response_body_size = len(flow.response.content)
+    response_body_decoded_size = len(flow.response.get_decoded_content())
+    response_body_compression = response_body_decoded_size - response_body_size
+    response_mime_type = flow.response.headers.get_first('Content-Type', '')
+    response_redirect_url = flow.response.headers.get_first('Location', '')
+
+    entry = HAR.entries(
+        {
+            "startedDateTime": started_date_time,
+            "time": full_time,
+            "request": {
+                "method": flow.request.method,
+                "url": flow.request.url,
+                "httpVersion": request_http_version,
+                "cookies": request_cookies,
+                "headers": request_headers,
+                "queryString": request_query_string,
+                "headersSize": request_headers_size,
+                "bodySize": request_body_size,
+            },
+            "response": {
+                "status": flow.response.code,
+                "statusText": flow.response.msg,
+                "httpVersion": response_http_version,
+                "cookies": response_cookies,
+                "headers": response_headers,
+                "content": {
+                    "size": response_body_size,
+                    "compression": response_body_compression,
+                    "mimeType": response_mime_type},
+                "redirectURL": response_redirect_url,
+                "headersSize": response_headers_size,
+                "bodySize": response_body_size,
+            },
+            "cache": {},
+            "timings": timings,
+        })
+
+    # If the current url is in the page list of context.HARLog or does not have
+    # a referrer we add it as a new pages object.
+    if flow.request.url in context.HARLog.get_page_list() or flow.request.headers.get(
+            'Referer',
+            None) is None:
+        page_id = context.HARLog.create_page_id()
+        context.HARLog.add(
+            HAR.pages({
+                "startedDateTime": entry['startedDateTime'],
+                "id": page_id,
+                "title": flow.request.url,
+            })
+        )
+        context.HARLog.set_page_ref(flow.request.url, page_id)
+        entry['pageref'] = page_id
+
+    # Lookup the referer in the page_ref of context.HARLog to point this entries
+    # pageref attribute to the right pages object, then set it as a new
+    # reference to build a reference tree.
+    elif context.HARLog.get_page_ref(flow.request.headers.get('Referer', (None, ))[0]) is not None:
+        entry['pageref'] = context.HARLog.get_page_ref(
+            flow.request.headers['Referer'][0]
+        )
+        context.HARLog.set_page_ref(
+            flow.request.headers['Referer'][0], entry['pageref']
+        )
+
+    context.HARLog.add(entry)
+
+
+def done(context):
+    """
+        Called once on script shutdown, after any other events.
+    """
+    from pprint import pprint
+    import json
+
+    json_dump = context.HARLog.json()
+    compressed_json_dump = context.HARLog.compress()
+
+    if context.dump_file == '-':
+        context.log(pprint.pformat(json.loads(json_dump)))
+    elif context.dump_file.endswith('.zhar'):
+        file(context.dump_file, "w").write(compressed_json_dump)
+    else:
+        file(context.dump_file, "w").write(json_dump)
+    context.log(
+        "HAR log finished with %s bytes (%s bytes compressed)" % (
+            len(json_dump), len(compressed_json_dump)
+        )
+    )
+    context.log(
+        "Compression rate is %s%%" % str(
+            100. * len(compressed_json_dump) / len(json_dump)
+        )
+    )
+
+
+def print_attributes(obj, filter_string=None, hide_privates=False):
+    """
+        Useful helper method to quickly get all attributes of an object and its
+        values.
+    """
+    for attr in dir(obj):
+        if hide_privates and "__" in attr:
+            continue
+        if filter_string is not None and filter_string not in attr:
+            continue
+        value = getattr(obj, attr)
+        print("%s.%s" % ('obj', attr), value, type(value))

examples/iframe_injector

@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-"""
-    Zap encoding in requests and inject iframe after body tag in html responses.
-    Usage:
-        iframe_injector http://someurl/somefile.html
-"""
-from libmproxy import controller, proxy
-import os
-import sys
-
-
-class InjectingMaster(controller.Master):
-    def __init__(self, server, iframe_url):
-        controller.Master.__init__(self, server)
-        self._iframe_url = iframe_url
-
-    def run(self):
-        try:
-            return controller.Master.run(self)
-        except KeyboardInterrupt:
-            self.shutdown()
-
-    def handle_request(self, msg):
-        if 'Accept-Encoding' in msg.headers:
-            msg.headers["Accept-Encoding"] = 'none'
-        msg.reply()
-
-    def handle_response(self, msg):
-        if msg.content:
-            c = msg.replace('<body>', '<body><iframe src="%s" frameborder="0" height="0" width="0"></iframe>' % self._iframe_url)
-            if c > 0:
-                print 'Iframe injected!'
-        msg.reply()
-
-
-def main(argv):
-    if len(argv) != 2:
-        print "Usage: %s IFRAME_URL" % argv[0]
-        sys.exit(1)
-    iframe_url = argv[1]
-    config = proxy.ProxyConfig(
-        cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
-    )
-    server = proxy.ProxyServer(config, 8080)
-    print 'Starting proxy...'
-    m = InjectingMaster(server, iframe_url)
-    m.run()
-
-if __name__ == '__main__':
-    main(sys.argv)

examples/iframe_injector.py

@@ -0,0 +1,27 @@
+# Usage: mitmdump -s "iframe_injector.py url"
+# (this script works best with --anticache)
+from bs4 import BeautifulSoup
+from libmproxy.protocol.http import decoded
+
+
+def start(context, argv):
+    if len(argv) != 2:
+        raise ValueError('Usage: -s "iframe_injector.py url"')
+    context.iframe_url = argv[1]
+
+
+def response(context, flow):
+    if flow.request.host in context.iframe_url:
+        return
+    with decoded(flow.response):  # Remove content encoding (gzip, ...)
+        html = BeautifulSoup(flow.response.content)
+        if html.body:
+            iframe = html.new_tag(
+                "iframe",
+                src=context.iframe_url,
+                frameborder=0,
+                height=0,
+                width=0)
+            html.body.insert(0, iframe)
+            flow.response.content = str(html)
+            context.log("Iframe inserted.")

examples/ignore_websocket.py

@@ -0,0 +1,36 @@
+# This script makes mitmproxy switch to passthrough mode for all HTTP
+# responses with "Connection: Upgrade" header. This is useful to make
+# WebSockets work in untrusted environments.
+#
+# Note: Chrome (and possibly other browsers), when explicitly configured
+# to use a proxy (i.e. mitmproxy's regular mode), send a CONNECT request
+# to the proxy before they initiate the websocket connection.
+# To make WebSockets work in these cases, supply
+# `--ignore :80$` as an additional parameter.
+# (see http://mitmproxy.org/doc/features/passthrough.html)
+
+from libmproxy.protocol.http import HTTPRequest
+from libmproxy.protocol.tcp import TCPHandler
+from libmproxy.protocol import KILL
+from libmproxy.script import concurrent
+
+
+def start(context, argv):
+    HTTPRequest._headers_to_strip_off.remove("Connection")
+    HTTPRequest._headers_to_strip_off.remove("Upgrade")
+
+
+def done(context):
+    HTTPRequest._headers_to_strip_off.append("Connection")
+    HTTPRequest._headers_to_strip_off.append("Upgrade")
+
+
+@concurrent
+def response(context, flow):
+    value = flow.response.headers.get_first("Connection", None)
+    if value and value.upper() == "UPGRADE":
+        # We need to send the response manually now...
+        flow.client_conn.send(flow.response.assemble())
+        # ...and then delegate to tcp passthrough.
+        TCPHandler(flow.live.c, log=False).handle_messages()
+        flow.reply(KILL)

examples/mitmproxywrapper.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python
+#
+# Helper tool to enable/disable OS X proxy and wrap mitmproxy
+#
+# Get usage information with:
+#
+# mitmproxywrapper.py -h
+#
+
+import subprocess
+import re
+import argparse
+import contextlib
+import os
+import sys
+
+
+class Wrapper(object):
+
+    def __init__(self, port, extra_arguments=None):
+        self.port = port
+        self.extra_arguments = extra_arguments
+
+    def run_networksetup_command(self, *arguments):
+        return subprocess.check_output(
+            ['sudo', 'networksetup'] + list(arguments))
+
+    def proxy_state_for_service(self, service):
+        state = self.run_networksetup_command(
+            '-getwebproxy',
+            service).splitlines()
+        return dict([re.findall(r'([^:]+): (.*)', line)[0] for line in state])
+
+    def enable_proxy_for_service(self, service):
+        print('Enabling proxy on {}...'.format(service))
+        for subcommand in ['-setwebproxy', '-setsecurewebproxy']:
+            self.run_networksetup_command(
+                subcommand, service, '127.0.0.1', str(
+                    self.port))
+
+    def disable_proxy_for_service(self, service):
+        print('Disabling proxy on {}...'.format(service))
+        for subcommand in ['-setwebproxystate', '-setsecurewebproxystate']:
+            self.run_networksetup_command(subcommand, service, 'Off')
+
+    def interface_name_to_service_name_map(self):
+        order = self.run_networksetup_command('-listnetworkserviceorder')
+        mapping = re.findall(
+            r'\(\d+\)\s(.*)$\n\(.*Device: (.+)\)$',
+            order,
+            re.MULTILINE)
+        return dict([(b, a) for (a, b) in mapping])
+
+    def run_command_with_input(self, command, input):
+        popen = subprocess.Popen(
+            command,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE)
+        (stdout, stderr) = popen.communicate(input)
+        return stdout
+
+    def primary_interace_name(self):
+        scutil_script = 'get State:/Network/Global/IPv4\nd.show\n'
+        stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script)
+        interface, = re.findall(r'PrimaryInterface\s*:\s*(.+)', stdout)
+        return interface
+
+    def primary_service_name(self):
+        return self.interface_name_to_service_name_map()[
+            self.primary_interace_name()]
+
+    def proxy_enabled_for_service(self, service):
+        return self.proxy_state_for_service(service)['Enabled'] == 'Yes'
+
+    def toggle_proxy(self):
+        new_state = not self.proxy_enabled_for_service(
+            self.primary_service_name())
+        for service_name in self.connected_service_names():
+            if self.proxy_enabled_for_service(service_name) and not new_state:
+                self.disable_proxy_for_service(service_name)
+            elif not self.proxy_enabled_for_service(service_name) and new_state:
+                self.enable_proxy_for_service(service_name)
+
+    def connected_service_names(self):
+        scutil_script = 'list\n'
+        stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script)
+        service_ids = re.findall(r'State:/Network/Service/(.+)/IPv4', stdout)
+
+        service_names = []
+        for service_id in service_ids:
+            scutil_script = 'show Setup:/Network/Service/{}\n'.format(
+                service_id)
+            stdout = self.run_command_with_input(
+                '/usr/sbin/scutil',
+                scutil_script)
+            service_name, = re.findall(r'UserDefinedName\s*:\s*(.+)', stdout)
+            service_names.append(service_name)
+
+        return service_names
+
+    def wrap_mitmproxy(self):
+        with self.wrap_proxy():
+            cmd = ['mitmproxy', '-p', str(self.port)]
+            if self.extra_arguments:
+                cmd.extend(self.extra_arguments)
+            subprocess.check_call(cmd)
+
+    def wrap_honeyproxy(self):
+        with self.wrap_proxy():
+            popen = subprocess.Popen('honeyproxy.sh')
+            try:
+                popen.wait()
+            except KeyboardInterrupt:
+                popen.terminate()
+
+    @contextlib.contextmanager
+    def wrap_proxy(self):
+        connected_service_names = self.connected_service_names()
+        for service_name in connected_service_names:
+            if not self.proxy_enabled_for_service(service_name):
+                self.enable_proxy_for_service(service_name)
+
+        yield
+
+        for service_name in connected_service_names:
+            if self.proxy_enabled_for_service(service_name):
+                self.disable_proxy_for_service(service_name)
+
+    @classmethod
+    def ensure_superuser(cls):
+        if os.getuid() != 0:
+            print('Relaunching with sudo...')
+            os.execv('/usr/bin/sudo', ['/usr/bin/sudo'] + sys.argv)
+
+    @classmethod
+    def main(cls):
+        parser = argparse.ArgumentParser(
+            description='Helper tool for OS X proxy configuration and mitmproxy.',
+            epilog='Any additional arguments will be passed on unchanged to mitmproxy.')
+        parser.add_argument(
+            '-t',
+            '--toggle',
+            action='store_true',
+            help='just toggle the proxy configuration')
+#         parser.add_argument('--honeyproxy', action='store_true', help='run honeyproxy instead of mitmproxy')
+        parser.add_argument(
+            '-p',
+            '--port',
+            type=int,
+            help='override the default port of 8080',
+            default=8080)
+        args, extra_arguments = parser.parse_known_args()
+
+        wrapper = cls(port=args.port, extra_arguments=extra_arguments)
+
+        if args.toggle:
+            wrapper.toggle_proxy()
+#         elif args.honeyproxy:
+#             wrapper.wrap_honeyproxy()
+        else:
+            wrapper.wrap_mitmproxy()
+
+
+if __name__ == '__main__':
+    Wrapper.ensure_superuser()
+    Wrapper.main()

examples/modify_form.py

@@ -1,8 +1,7 @@
 
 def request(context, flow):
-    if "application/x-www-form-urlencoded" in flow.request.headers["content-type"]:
-        frm = flow.request.get_form_urlencoded()
-        frm["mitmproxy"] = ["rocks"]
-        flow.request.set_form_urlencoded(frm)
-
-
+    if "application/x-www-form-urlencoded" in flow.request.headers[
+            "content-type"]:
+        form = flow.request.get_form_urlencoded()
+        form["mitmproxy"] = ["rocks"]
+        flow.request.set_form_urlencoded(form)

examples/modify_querystring.py

@@ -4,4 +4,3 @@ def request(context, flow):
     if q:
         q["mitmproxy"] = ["rocks"]
         flow.request.set_query(q)
-

examples/modify_response_body.py

@@ -0,0 +1,18 @@
+# Usage: mitmdump -s "modify_response_body.py mitmproxy bananas"
+# (this script works best with --anticache)
+from libmproxy.protocol.http import decoded
+
+
+def start(context, argv):
+    if len(argv) != 3:
+        raise ValueError('Usage: -s "modify-response-body.py old new"')
+    # You may want to use Python's argparse for more sophisticated argument
+    # parsing.
+    context.old, context.new = argv[1], argv[2]
+
+
+def response(context, flow):
+    with decoded(flow.response):  # automatically decode gzipped responses.
+        flow.response.content = flow.response.content.replace(
+            context.old,
+            context.new)

examples/nonblocking.py

@@ -0,0 +1,9 @@
+import time
+from libmproxy.script import concurrent
+
+
+@concurrent  # Remove this and see what happens
+def request(context, flow):
+    print("handle request: %s%s" % (flow.request.host, flow.request.path))
+    time.sleep(5)
+    print("start  request: %s%s" % (flow.request.host, flow.request.path))